@tinycloud/sdk-services 3.2.6-beta.4 → 3.3.0-beta.25

package/dist/internal/decrypt-transport-response-error.d.cts

@@ -1,4 +1,4 @@
-import { P as PermissionHint } from '../types-gmdUiorz.cjs';
+import { P as PermissionHint } from '../types-eVS1Whuf.cjs';
 
 /** A reachable node returned an HTTP failure for a decrypt request. */
 declare class DecryptTransportResponseError extends Error {

package/dist/internal/decrypt-transport-response-error.d.ts

@@ -1,4 +1,4 @@
-import { P as PermissionHint } from '../types-gmdUiorz.js';
+import { P as PermissionHint } from '../types-eVS1Whuf.js';
 
 /** A reachable node returned an HTTP failure for a decrypt request. */
 declare class DecryptTransportResponseError extends Error {

package/dist/kv/index.cjs

@@ -48,6 +48,14 @@ var ErrorCodes = {
   ABORTED: "ABORTED",
   INVALID_INPUT: "INVALID_INPUT",
   PERMISSION_DENIED: "PERMISSION_DENIED",
+  // Node retry signals (tinycloud-node #347). `meta.retryable` says whether a
+  // retry is safe; `meta.retryAfterMs` carries the node's Retry-After.
+  /** HTTP 503 `sql_busy:<stage>` / `unavailable_retryable`/`read_audit_busy`: nothing was applied; retry after `meta.retryAfterMs`. */
+  NODE_BUSY: "NODE_BUSY",
+  /** HTTP 503 `request_timeout`: the node's deadline passed before any write began; nothing was applied. */
+  REQUEST_TIMEOUT: "REQUEST_TIMEOUT",
+  /** HTTP 500 `outcome_unknown`: a write may have been applied; read back before retrying a non-idempotent operation. */
+  OUTCOME_UNKNOWN: "OUTCOME_UNKNOWN",
   // KV-specific errors
   KV_NOT_FOUND: "KV_NOT_FOUND",
   KV_WRITE_FAILED: "KV_WRITE_FAILED",
@@ -83,11 +91,12 @@ var ErrorCodes = {
   DUCKDB_READONLY_VIOLATION: "DUCKDB_READONLY_VIOLATION"
 };
 var defaultRetryPolicy = {
-  maxAttempts: 3,
+  maxAttempts: 4,
   backoff: "exponential",
-  baseDelayMs: 1e3,
-  maxDelayMs: 1e4,
-  retryableErrors: [ErrorCodes.NETWORK_ERROR, ErrorCodes.TIMEOUT]
+  baseDelayMs: 250,
+  maxDelayMs: 4e3,
+  retryableErrors: [ErrorCodes.NODE_BUSY, ErrorCodes.REQUEST_TIMEOUT],
+  maxTotalMs: 12e3
 };
 var TelemetryEvents = {
   SPAN: "telemetry.span",
@@ -123,6 +132,20 @@ function serviceError(code, message, service, options) {
   };
 }
 
+// src/responseErrors.ts
+function responseErrorMeta(status, statusText, errorText) {
+  const meta = { status, statusText };
+  const snippet = compactMessage(errorText);
+  if (snippet) {
+    meta.responseSnippet = snippet.slice(0, 300);
+  }
+  return meta;
+}
+function compactMessage(message) {
+  const compact = message.replace(/\s+/g, " ").trim();
+  return compact.length > 500 ? `${compact.slice(0, 500)}...` : compact;
+}
+
 // src/errors.ts
 var import_bootstrap = require("@tinycloud/bootstrap");
 
@@ -216,6 +239,30 @@ function authRequiredError(service) {
     service
   };
 }
+function nodeRetryError(service, message, response, errorText) {
+  const body = errorText.trim();
+  const [, token, stage] = /^([a-z_]+)(?::\s*([a-z_]+):)?/.exec(body) ?? [];
+  let code;
+  if (response.status === 503 && (token === "sql_busy" || token === "unavailable_retryable" || token === "read_audit_busy")) {
+    code = ErrorCodes.NODE_BUSY;
+  } else if (response.status === 503 && token === "request_timeout") {
+    code = ErrorCodes.REQUEST_TIMEOUT;
+  } else if (response.status === 500 && token === "outcome_unknown") {
+    code = ErrorCodes.OUTCOME_UNKNOWN;
+  } else {
+    return void 0;
+  }
+  const retryable = code !== ErrorCodes.OUTCOME_UNKNOWN;
+  const retryAfter = response.headers.get("retry-after")?.trim();
+  return serviceError(code, message, service, {
+    meta: {
+      ...responseErrorMeta(response.status, response.statusText, errorText),
+      retryable,
+      ...token === "sql_busy" && stage ? { stage } : {},
+      ...retryable && retryAfter && /^\d{1,9}$/.test(retryAfter) ? { retryAfterMs: Number(retryAfter) * 1e3 } : {}
+    }
+  });
+}
 var RequestTimeoutError = class extends Error {
   constructor(timeoutMs) {
     super(`Request timed out after ${timeoutMs}ms.`);
@@ -627,6 +674,18 @@ var BaseService = class {
     }
     return controller.signal;
   }
+  /** POST a signed invocation with retry; `mint` re-signs per attempt, `init.headers` add to the signed ones. */
+  invokeWithRetry(url, mint, { headers: extra, ...init } = {}) {
+    const build = () => {
+      const signed = mint();
+      return {
+        method: "POST",
+        ...init,
+        headers: Array.isArray(signed) ? [...signed, ...Object.entries(extra ?? {})] : { ...signed, ...extra }
+      };
+    };
+    return this.context.invokeWithRetry ? this.context.invokeWithRetry(url, build) : this.context.fetch(url, build());
+  }
   /**
    * Create the abort signal for one request, bounded by the request timeout.
    *
@@ -994,15 +1053,6 @@ var KVService = class extends BaseService {
   get host() {
     return this.context.hosts[0];
   }
-  withJsonContentType(headers) {
-    if (Array.isArray(headers)) {
-      return [...headers, ["content-type", "application/json"]];
-    }
-    return {
-      ...headers,
-      "content-type": "application/json"
-    };
-  }
   /**
    * Execute an invoke operation.
    *
@@ -1013,20 +1063,11 @@ var KVService = class extends BaseService {
    * @returns Fetch response
    */
   async invokeOperation(path, action, body, signal, extraHeaders) {
-    const session = this.context.session;
-    const headers = this.context.invoke(
-      session,
-      "kv",
-      path,
-      action
+    return this.invokeWithRetry(
+      `${this.host}/invoke`,
+      () => this.context.invoke(this.context.session, "kv", path, action),
+      { headers: extraHeaders, body, signal }
     );
-    const requestHeaders = Array.isArray(headers) ? [...headers, ...Object.entries(extraHeaders ?? {})] : { ...headers, ...extraHeaders };
-    return this.context.fetch(`${this.host}/invoke`, {
-      method: "POST",
-      headers: requestHeaders,
-      body,
-      signal
-    });
   }
   /**
    * Serialize a single put value into a fetch body.
@@ -1232,28 +1273,28 @@ var KVService = class extends BaseService {
     }
     const request = this.createRequestSignal(options?.signal, options?.timeout);
     try {
-      const session = this.context.session;
-      const invocationHeaders = this.context.invokeAny(
-        session,
-        paths.map((path) => ({
-          spaceId: session.spaceId,
-          service: "kv",
-          path,
-          action
-        }))
-      );
       const limitHeaders = {};
       if (getOptions?.maxResponseBytes !== void 0) {
         limitHeaders["x-tinycloud-max-response-bytes"] = String(
           getOptions.maxResponseBytes
         );
       }
-      const headers = Array.isArray(invocationHeaders) ? [...invocationHeaders, ...Object.entries(limitHeaders)] : { ...invocationHeaders, ...limitHeaders };
-      const response = await this.context.fetch(`${this.host}/invoke`, {
-        method: "POST",
-        headers,
-        signal: request.signal
-      });
+      const response = await this.invokeWithRetry(
+        `${this.host}/invoke`,
+        () => {
+          const session = this.context.session;
+          return this.context.invokeAny(
+            session,
+            paths.map((path) => ({
+              spaceId: session.spaceId,
+              service: "kv",
+              path,
+              action
+            }))
+          );
+        },
+        { headers: limitHeaders, signal: request.signal }
+      );
       if (!response.ok) {
         const errorText = response.status === 401 || response.status === 403 ? await this.readAuthorizationText(response) : await response.text();
         if (response.status === 401 || response.status === 403) {
@@ -1299,10 +1340,10 @@ var KVService = class extends BaseService {
             result: err(serviceError(item.error.code, item.error.message, "kv"))
           };
         }
-        const headers2 = item.headers;
+        const headers = item.headers;
         const data = action === KVAction.HEAD ? void 0 : this.parseBatchValue(
           item.dataBase64,
-          headers2,
+          headers,
           getOptions?.raw,
           getOptions?.binary
         );
@@ -1310,7 +1351,7 @@ var KVService = class extends BaseService {
           key: keys[index],
           result: ok({
             data,
-            headers: this.createBatchResponseHeaders(headers2)
+            headers: this.createBatchResponseHeaders(headers)
           })
         };
       });
@@ -1534,15 +1575,18 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
+      const message = `Failed to get key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to get key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
     }
     const data = await this.parseResponse(response, options?.raw, options?.binary);
-    return ok({ data, headers: this.createResponseHeaders(response.headers) });
+    return ok({ data, headers: this.createResponseHeaders(response.headers), ...response.retry && { retry: response.retry } });
   }
   async batchGet(keys, options) {
     return this.withTelemetry(
@@ -1618,26 +1662,22 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
-      if (response.status === 503 && (options?.ifMatch !== void 0 || options?.ifNoneMatch !== void 0)) {
-        return err(serviceError(
-          ErrorCodes.KV_CONFLICT,
-          `Concurrent KV update conflicted for key ${JSON.stringify(key)}`,
-          "kv",
-          { meta: { status: response.status, statusText: response.statusText } }
-        ));
-      }
+      const message = `Failed to put key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       const quotaError = this.handleQuotaErrorResponse(response, errorText, key);
       if (quotaError) return quotaError;
       return err(serviceError(
         ErrorCodes.KV_WRITE_FAILED,
-        `Failed to put key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
     }
     return ok({
       data: void 0,
-      headers: this.createResponseHeaders(response.headers)
+      headers: this.createResponseHeaders(response.headers),
+      ...response.retry && { retry: response.retry }
     });
   }
   /**
@@ -1710,25 +1750,23 @@ var KVService = class extends BaseService {
           entries?.[index]?.body ?? this.serializeBatchPutValue(items[index])
         );
       }
-      const headers = this.context.invokeAny(
-        session,
-        paths.map((path) => ({
-          spaceId: session.spaceId,
-          service: "kv",
-          path,
-          action: KVAction.PUT
-        }))
+      const response = await this.invokeWithRetry(
+        `${this.host}/invoke`,
+        () => {
+          const headers = this.context.invokeAny(
+            session,
+            paths.map((path) => ({
+              spaceId: session.spaceId,
+              service: "kv",
+              path,
+              action: KVAction.PUT
+            }))
+          );
+          requestMayHaveDispatched = true;
+          return headers;
+        },
+        { body, signal }
       );
-      const url = `${this.host}/invoke`;
-      const init = {
-        method: "POST",
-        headers,
-        body,
-        signal
-      };
-      const fetchFn = this.context.fetch;
-      requestMayHaveDispatched = true;
-      const response = await fetchFn(url, init);
       if (!response.ok) {
         let errorText;
         try {
@@ -1930,9 +1968,12 @@ var KVService = class extends BaseService {
         );
       }
       const errorText = await response.text();
+      const message = `Failed to list keys: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to list keys: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
@@ -2183,24 +2224,20 @@ var KVService = class extends BaseService {
           { meta: { status: response.status, statusText: response.statusText } }
         ));
       }
-      if (response.status === 503 && options?.ifMatch !== void 0) {
-        return err(serviceError(
-          ErrorCodes.KV_CONFLICT,
-          `Concurrent KV delete conflicted for key ${JSON.stringify(key)}`,
-          "kv",
-          { meta: { status: response.status, statusText: response.statusText } }
-        ));
-      }
+      const message = `Failed to delete key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+      const retryError = nodeRetryError("kv", message, response, errorText);
+      if (retryError) return err(retryError);
       return err(serviceError(
         ErrorCodes.NETWORK_ERROR,
-        `Failed to delete key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+        message,
         "kv",
         { meta: { status: response.status, statusText: response.statusText } }
       ));
     }
     return ok({
       data: void 0,
-      headers: this.createResponseHeaders(response.headers)
+      headers: this.createResponseHeaders(response.headers),
+      ...response.retry && { retry: response.retry }
     });
   }
   /**
@@ -2234,10 +2271,13 @@ var KVService = class extends BaseService {
             return await this.classifyNotFound(response, key);
           }
           const errorText = await response.text();
+          const message = `Failed to get metadata for key ${JSON.stringify(key)}: ${response.status} - ${errorText}`;
+          const retryError = nodeRetryError("kv", message, response, errorText);
+          if (retryError) return err(retryError);
           return err(
             serviceError(
               ErrorCodes.NETWORK_ERROR,
-              `Failed to get metadata for key ${JSON.stringify(key)}: ${response.status} - ${errorText}`,
+              message,
               "kv",
               { meta: { status: response.status, statusText: response.statusText } }
             )
@@ -2271,12 +2311,6 @@ var KVService = class extends BaseService {
       }
       const path = this.getFullPath(key, options?.prefix);
       const session = this.context.session;
-      const headers = this.context.invoke(
-        session,
-        "kv",
-        path,
-        KVAction.GET
-      );
       const body = {
         space: session.spaceId,
         path,
@@ -2290,12 +2324,15 @@ var KVService = class extends BaseService {
       }
       const request = this.createRequestSignal(options?.signal, options?.timeout);
       try {
-        const response = await this.context.fetch(`${this.host}/signed/kv`, {
-          method: "POST",
-          headers: this.withJsonContentType(headers),
-          body: JSON.stringify(body),
-          signal: request.signal
-        });
+        const response = await this.invokeWithRetry(
+          `${this.host}/signed/kv`,
+          () => this.context.invoke(session, "kv", path, KVAction.GET),
+          {
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: request.signal
+          }
+        );
         if (!response.ok) {
           return await this.createSignedReadUrlError(response, key, path);
         }