@tinycloud/sdk-services 3.2.6-beta.2 → 3.3.0-beta.7

package/dist/index.cjs

@@ -122,6 +122,7 @@ __export(src_exports, {
   kvPrefixCovers: () => kvPrefixCovers,
   networkDiscoveryKey: () => networkDiscoveryKey,
   networkError: () => networkError,
+  nodeRetryError: () => nodeRetryError,
   notFoundError: () => notFoundError,
   ok: () => ok,
   openWrappedKey: () => openWrappedKey,
@@ -179,6 +180,14 @@ var ErrorCodes = {
   ABORTED: "ABORTED",
   INVALID_INPUT: "INVALID_INPUT",
   PERMISSION_DENIED: "PERMISSION_DENIED",
+  // Node retry signals (tinycloud-node #347). `meta.retryable` says whether a
+  // retry is safe; `meta.retryAfterMs` carries the node's Retry-After.
+  /** HTTP 503 `sql_busy:<stage>` / `unavailable_retryable`: nothing was applied; retry after `meta.retryAfterMs`. */
+  NODE_BUSY: "NODE_BUSY",
+  /** HTTP 503 `request_timeout`: the node's deadline passed before any write began; nothing was applied. */
+  REQUEST_TIMEOUT: "REQUEST_TIMEOUT",
+  /** HTTP 500 `outcome_unknown`: a write may have been applied; read back before retrying a non-idempotent operation. */
+  OUTCOME_UNKNOWN: "OUTCOME_UNKNOWN",
   // KV-specific errors
   KV_NOT_FOUND: "KV_NOT_FOUND",
   KV_WRITE_FAILED: "KV_WRITE_FAILED",
@@ -233,6 +242,8 @@ var TelemetryEvents = {
   REPLICATION_SYNC: "replication.sync",
   REPLICATION_STATE: "replication.state",
   REPLICATION_DIVERGENCE: "replication.divergence",
+  REPLICATION_GRANT: "replication.grant",
+  REPLICATION_LIFECYCLE: "replication.lifecycle",
   /** Emitted once, on the first write rejected because storage is full. */
   STORAGE_FULL: "storage.full"
 };
@@ -269,21 +280,27 @@ var SAFE_NUMBER_FIELDS = /* @__PURE__ */ new Set([
   "fetched",
   "pages",
   "lagMs",
-  "pendingCleared"
+  "pendingCleared",
+  "partitions",
+  "remaining",
+  "expiryMs"
 ]);
-var SAFE_BOOLEAN_FIELDS = /* @__PURE__ */ new Set(["authenticated", "ok", "persisted"]);
+var SAFE_BOOLEAN_FIELDS = /* @__PURE__ */ new Set(["authenticated", "ok", "persisted", "purgePending"]);
 var SAFE_REPLICATION_FIELDS = {
   source: ["replica", "network", "none"],
-  reason: ["hit", "absent", "deleted", "content_missing", "coverage_incomplete", "not_covered", "pending_write", "grant_missing", "grant_expired", "grant_revoked", "grant_not_yet_valid", "stale", "replica_error", "runtime_unsupported", "replica_unavailable", "unsupported_option", "network_cursor", "cursor_restart", "aborted", "REPLICA_BEHIND_OWN_WRITES", "REPLICA_UNPROVEN_SINCE_START", "NETWORK_REQUESTED"],
+  reason: ["hit", "absent", "deleted", "content_missing", "coverage_incomplete", "not_covered", "pending_write", "grant_missing", "grant_expired", "grant_revoked", "grant_not_yet_valid", "stale", "replica_error", "runtime_unsupported", "replica_unavailable", "unsupported_option", "network_cursor", "cursor_restart", "aborted", "REPLICA_BEHIND_OWN_WRITES", "REPLICA_UNPROVEN_SINCE_START", "NETWORK_REQUESTED", "replica_gated", "sign_out", "account_switch", "consent_expired", "startup"],
   op: ["get", "list", "put", "delete", "batchPut"],
-  trigger: ["start", "interval", "stale_read", "manual"],
+  trigger: ["start", "interval", "stale_read", "manual", "peer"],
+  origin: ["peer"],
   outcome: ["found", "not_found", "error", "committed", "failed", "ambiguous", "ok", "busy", "aborted"],
-  class: ["offline", "authority", "storage", "node"],
-  state: ["opened", "grant_installed", "grant_missing", "runtime_unsupported", "unavailable", "revoked", "recreated", "purged", "purge_failed", "pending_store_error", "pinned", "pending_cleared", "closed", "idle", "ready"],
+  class: ["offline", "authority", "storage", "node", "lifecycle"],
+  state: ["opened", "grant_installed", "grant_missing", "runtime_unsupported", "unavailable", "revoked", "recreated", "purged", "purge_failed", "pending_store_error", "pinned", "pending_cleared", "closed", "idle", "ready", "gated"],
+  action: ["minted", "renewed", "renewal_refused", "renewal_deferred", "capped", "expired", "consent_deadline_reached", "denied", "deny_stale", "deny_failed", "database_deleted", "marker_set", "marker_cleared", "unguarded", "drained", "erased", "erase_failed", "activation_blocked", "activated", "recovered"],
+  gate: ["not_established", "stamp_stale", "principal_mismatch", "purge_pending", "logout_pending", "gate_unavailable"],
   kind: ["value", "missing_local", "extra_local", "keys", "order"],
   coverage: ["empty", "bootstrapping", "complete"],
   authority: ["valid", "expired", "revoked", "not-yet-valid"],
-  code: ["NETWORK_ERROR", "TIMEOUT", "ABORTED", "REPLICA_BUSY", "REPLICA_BEHIND_OWN_WRITES", "REPLICA_UNPROVEN_SINCE_START", "NETWORK_REQUESTED", "INTEGRITY_ERROR", "STORAGE_ERROR", "REPLICA_CLOSED", "RUNTIME_UNSUPPORTED", "GRANT_EXPIRED", "GRANT_REVOKED", "GRANT_NOT_YET_VALID", "GRANT_MISSING", "GRANT_INVALID", "SOURCE_CHANGED", "PROTOCOL_ERROR", "SCOPE_VIOLATION", "NODE_ERROR", "CONTENT_MISMATCH", "STORAGE_FULL"],
+  code: ["NETWORK_ERROR", "TIMEOUT", "ABORTED", "REPLICA_BUSY", "REPLICA_BEHIND_OWN_WRITES", "REPLICA_UNPROVEN_SINCE_START", "NETWORK_REQUESTED", "INTEGRITY_ERROR", "STORAGE_ERROR", "REPLICA_CLOSED", "RUNTIME_UNSUPPORTED", "GRANT_EXPIRED", "GRANT_REVOKED", "GRANT_NOT_YET_VALID", "GRANT_MISSING", "GRANT_INVALID", "SOURCE_CHANGED", "PROTOCOL_ERROR", "SCOPE_VIOLATION", "NODE_ERROR", "CONTENT_MISMATCH", "STORAGE_FULL", "REPLICA_GATED", "RESET_REQUIRED", "STORAGE_BUDGET", "CONSENT_EXPIRED"],
   pendingState: ["in_flight", "committed", "ambiguous"],
   strategy: ["session", "minted", "installed"],
   verify: ["match", "diverged", "error", "aborted"]
@@ -1115,6 +1132,42 @@ var ServiceContext = class {
   }
 };
 
+// src/responseErrors.ts
+function parseServiceErrorBody(errorText) {
+  try {
+    return JSON.parse(errorText);
+  } catch {
+    return {};
+  }
+}
+function formatServiceResponseError(serviceLabel, operation, status, errorText, parsed) {
+  if (parsed.message) {
+    return compactMessage(parsed.message);
+  }
+  if (looksLikeHtml(errorText)) {
+    if (status === 524 || /524\s*[:-]/i.test(errorText) || /a timeout occurred/i.test(errorText)) {
+      return `${serviceLabel} ${operation} failed: upstream request timed out. Please retry.`;
+    }
+    return `${serviceLabel} ${operation} failed: upstream service returned an HTML error page (${status}).`;
+  }
+  return `${serviceLabel} ${operation} failed: ${status} - ${compactMessage(errorText)}`;
+}
+function responseErrorMeta(status, statusText, errorText) {
+  const meta = { status, statusText };
+  const snippet = compactMessage(errorText);
+  if (snippet) {
+    meta.responseSnippet = snippet.slice(0, 300);
+  }
+  return meta;
+}
+function looksLikeHtml(text) {
+  return /<!doctype html/i.test(text) || /<html[\s>]/i.test(text);
+}
+function compactMessage(message) {
+  const compact = message.replace(/\s+/g, " ").trim();
+  return compact.length > 500 ? `${compact.slice(0, 500)}...` : compact;
+}
+
 // src/errors.ts
 var import_bootstrap = require("@tinycloud/bootstrap");
 
@@ -1274,6 +1327,30 @@ function networkError(service, message, cause) {
     cause
   };
 }
+function nodeRetryError(service, message, response, errorText) {
+  const body = errorText.trim();
+  const [, token, stage] = /^([a-z_]+)(?::\s*([a-z_]+):)?/.exec(body) ?? [];
+  let code;
+  if (response.status === 503 && (token === "sql_busy" || token === "unavailable_retryable")) {
+    code = ErrorCodes.NODE_BUSY;
+  } else if (response.status === 503 && token === "request_timeout") {
+    code = ErrorCodes.REQUEST_TIMEOUT;
+  } else if (response.status === 500 && token === "outcome_unknown") {
+    code = ErrorCodes.OUTCOME_UNKNOWN;
+  } else {
+    return void 0;
+  }
+  const retryable = code !== ErrorCodes.OUTCOME_UNKNOWN;
+  const retryAfter = response.headers.get("retry-after")?.trim();
+  return serviceError(code, message, service, {
+    meta: {
+      ...responseErrorMeta(response.status, response.statusText, errorText),
+      retryable,
+      ...token === "sql_busy" && stage ? { stage } : {},
+      ...retryable && retryAfter && /^\d{1,9}$/.test(retryAfter) ? { retryAfterMs: Number(retryAfter) * 1e3 } : {}
+    }
+  });
+}
 var RequestTimeoutError = class extends Error {
   constructor(timeoutMs) {
     super(`Request timed out after ${timeoutMs}ms.`);
@@ -2612,9 +2689,12 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
+      const message = `Failed to get key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to get key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
@@ -2696,19 +2776,14 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
-      if (response.status === 503 && (options?.ifMatch !== void 0 || options?.ifNoneMatch !== void 0)) {
-        return err(serviceError(
-          ErrorCodes.KV_CONFLICT,
-          `Concurrent KV update conflicted for key ${JSON.stringify(key)}`,
-          "kv",
-          { meta: { status: response.status, statusText: response.statusText } }
-        ));
-      }
+      const message = `Failed to put key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       const quotaError = this.handleQuotaErrorResponse(response, errorText, key);
       if (quotaError) return quotaError;
       return err(serviceError(
         ErrorCodes.KV_WRITE_FAILED,
-        `Failed to put key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
@@ -3008,9 +3083,12 @@ var KVService = class extends BaseService {
         );
       }
       const errorText = await response.text();
+      const message = `Failed to list keys: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to list keys: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
@@ -3261,17 +3339,12 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
-      if (response.status === 503 && options?.ifMatch !== void 0) {
-        return err(serviceError(
-          ErrorCodes.KV_CONFLICT,
-          `Concurrent KV delete conflicted for key ${JSON.stringify(key)}`,
-          "kv",
-          { meta: { status: response.status, statusText: response.statusText } }
-        ));
-      }
+      const message = `Failed to delete key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to delete key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
@@ -3312,10 +3385,13 @@ var KVService = class extends BaseService {
             return await this.classifyNotFound(response, key);
           }
           const errorText = await response.text();
+          const message = `Failed to get metadata for key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+          const retryError = nodeRetryError("kv", message, response, errorText);
+          if (retryError) return err(retryError);
           return err(
             serviceError(
               ErrorCodes.NETWORK_ERROR,
-              `Failed to get metadata for key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+              message,
               "kv",
               { meta: { status: response.status, statusText: response.statusText } }
             )
@@ -3507,42 +3583,6 @@ function requiresSecretsOptIn(space, prefix) {
   return prefix.split("/", 1)[0] === "vault";
 }
 
-// src/responseErrors.ts
-function parseServiceErrorBody(errorText) {
-  try {
-    return JSON.parse(errorText);
-  } catch {
-    return {};
-  }
-}
-function formatServiceResponseError(serviceLabel, operation, status, errorText, parsed) {
-  if (parsed.message) {
-    return compactMessage(parsed.message);
-  }
-  if (looksLikeHtml(errorText)) {
-    if (status === 524 || /524\s*[:-]/i.test(errorText) || /a timeout occurred/i.test(errorText)) {
-      return `${serviceLabel} ${operation} failed: upstream request timed out. Please retry.`;
-    }
-    return `${serviceLabel} ${operation} failed: upstream service returned an HTML error page (${status}).`;
-  }
-  return `${serviceLabel} ${operation} failed: ${status} - ${compactMessage(errorText)}`;
-}
-function responseErrorMeta(status, statusText, errorText) {
-  const meta = { status, statusText };
-  const snippet = compactMessage(errorText);
-  if (snippet) {
-    meta.responseSnippet = snippet.slice(0, 300);
-  }
-  return meta;
-}
-function looksLikeHtml(text) {
-  return /<!doctype html/i.test(text) || /<html[\s>]/i.test(text);
-}
-function compactMessage(message) {
-  const compact = message.replace(/\s+/g, " ").trim();
-  return compact.length > 500 ? `${compact.slice(0, 500)}...` : compact;
-}
-
 // src/sql/SQLMigrations.ts
 var SQLMigrations = class {
   constructor(service, dbName) {
@@ -3958,6 +3998,8 @@ var SQLService = class extends BaseService {
     if (response.status === 413 && parseStorageRejection(errorText)) {
       return err(storageRejectionError("sql", 413, meta, errorText));
     }
+    const retryError = nodeRetryError("sql", `SQL ${operation} failed: ${response.status} - ${errorText}`, response, errorText);
+    if (retryError) return err(retryError);
     const errorBody = parseServiceErrorBody(errorText);
     const errorCode = this.mapHttpStatusToErrorCode(
       response.status,
@@ -4427,6 +4469,8 @@ var DuckDbService = class extends BaseService {
     if (response.status === 413 && parseStorageRejection(errorText)) {
       return err(storageRejectionError("duckdb", 413, meta, errorText));
     }
+    const retryError = nodeRetryError("duckdb", `DuckDB ${operation} failed: ${response.status} - ${errorText}`, response, errorText);
+    if (retryError) return err(retryError);
     const errorBody = parseServiceErrorBody(errorText);
     const errorCode = this.mapHttpStatusToErrorCode(
       response.status,
@@ -8468,6 +8512,7 @@ var import_decrypt_transport_response_error3 = require("@tinycloud/sdk-services/
   kvPrefixCovers,
   networkDiscoveryKey,
   networkError,
+  nodeRetryError,
   notFoundError,
   ok,
   openWrappedKey,