@c3-oss/prosa 0.8.0 → 0.8.1

package/dist/cli/main.js

@@ -28394,8 +28394,9 @@ var CliUserError = class extends Error {
 import { request as httpRequest } from "http";
 import { request as httpsRequest } from "https";
 import { OBJECT_PACK_BINARY_CONTENT_TYPE, encodeBinaryObjectPack } from "@c3-oss/prosa-sync";
-var OBJECT_UPLOAD_MAX_ATTEMPTS = 4;
-var OBJECT_UPLOAD_BASE_BACKOFF_MS = 250;
+var OBJECT_UPLOAD_MAX_ATTEMPTS = 6;
+var OBJECT_UPLOAD_BASE_BACKOFF_MS = 500;
+var OBJECT_UPLOAD_MAX_BACKOFF_MS = 15e3;
 function trimTrailingSlash(url) {
   return url.endsWith("/") ? url.slice(0, -1) : url;
 }
@@ -28420,7 +28421,8 @@ __name(retryAfterMs, "retryAfterMs");
 function objectUploadBackoffMs(attempt, headers) {
   const retryAfter = headers ? retryAfterMs(headers) : void 0;
   if (retryAfter !== void 0) return retryAfter;
-  return OBJECT_UPLOAD_BASE_BACKOFF_MS * 2 ** attempt;
+  const exponential = Math.min(OBJECT_UPLOAD_MAX_BACKOFF_MS, OBJECT_UPLOAD_BASE_BACKOFF_MS * 2 ** attempt);
+  return Math.floor(exponential + Math.random() * Math.min(250, exponential));
 }
 __name(objectUploadBackoffMs, "objectUploadBackoffMs");
 function isRetryableObjectUploadStatus(status) {
@@ -28434,10 +28436,21 @@ function isRetryableNetworkError(err) {
   return code != null && [
     "ECONNRESET",
     "ETIMEDOUT",
-    "EPIPE"
+    "EPIPE",
+    "ECONNREFUSED",
+    "UND_ERR_SOCKET",
+    "UND_ERR_CONNECT_TIMEOUT"
   ].includes(code);
 }
 __name(isRetryableNetworkError, "isRetryableNetworkError");
+function networkErrorReason(err) {
+  if (err instanceof Error) {
+    const code = err.code ?? err.cause?.code;
+    return code ? `${err.message} (${code})` : err.message;
+  }
+  return String(err);
+}
+__name(networkErrorReason, "networkErrorReason");
 function isUnsupportedBinaryObjectPackResponse(status, text) {
   return status === 415 || status === 400 && /Unsupported Media Type|JSON body required/i.test(text);
 }
@@ -28535,12 +28548,16 @@ var ProsaApiClient = class {
   baseUrl;
   fetchFn;
   hasInjectedFetch;
+  onRetry;
+  onRequestSuccess;
   token;
   tenantId;
   constructor(opts) {
     this.baseUrl = trimTrailingSlash(opts.baseUrl);
     this.fetchFn = opts.fetch ?? globalThis.fetch;
     this.hasInjectedFetch = Boolean(opts.fetch);
+    this.onRetry = opts.onRetry;
+    this.onRequestSuccess = opts.onRequestSuccess;
     this.token = opts.token;
     this.tenantId = opts.tenantId;
   }
@@ -28571,6 +28588,90 @@ var ProsaApiClient = class {
     });
     return this.parseTrpc(path14, response);
   }
+  async trpcMutationRetriable(path14, input, opts) {
+    return this.retriableFetch({
+      operation: opts.operation,
+      retryHttpStatusBeforeParse: false,
+      retryParseErrorsOnRetryableStatus: true,
+      retryStructuredErrorsOnRetryableStatus: opts.retryStructuredErrorsOnRetryableStatus,
+      request: /* @__PURE__ */ __name(() => this.fetchFn(`${this.baseUrl}/trpc/${path14}`, {
+        method: "POST",
+        headers: this.headers({
+          "content-type": "application/json",
+          ...opts.headers
+        }),
+        body: JSON.stringify(input ?? {})
+      }), "request"),
+      parse: /* @__PURE__ */ __name((response) => this.parseTrpc(path14, response), "parse")
+    });
+  }
+  async retriableFetch(opts) {
+    const { operation: operation2, request: request2, parse: parse3 } = opts;
+    let lastError;
+    for (let attempt = 0; attempt < OBJECT_UPLOAD_MAX_ATTEMPTS; attempt += 1) {
+      let response;
+      try {
+        response = await request2();
+      } catch (err) {
+        lastError = err;
+        if (!isRetryableNetworkError(err) || attempt >= OBJECT_UPLOAD_MAX_ATTEMPTS - 1) {
+          throw this.wrapRetriedError(operation2, attempt + 1, err);
+        }
+        const delayMs = objectUploadBackoffMs(attempt);
+        this.onRetry?.({
+          operation: operation2,
+          attempt: attempt + 1,
+          maxAttempts: OBJECT_UPLOAD_MAX_ATTEMPTS,
+          delayMs,
+          reason: networkErrorReason(err)
+        });
+        await sleep(delayMs);
+        continue;
+      }
+      if (opts.retryHttpStatusBeforeParse !== false && isRetryableObjectUploadStatus(response.status) && attempt < OBJECT_UPLOAD_MAX_ATTEMPTS - 1) {
+        const delayMs = objectUploadBackoffMs(attempt, response.headers);
+        this.onRetry?.({
+          operation: operation2,
+          attempt: attempt + 1,
+          maxAttempts: OBJECT_UPLOAD_MAX_ATTEMPTS,
+          delayMs,
+          reason: `HTTP ${response.status}`
+        });
+        await response.arrayBuffer().catch(() => void 0);
+        await sleep(delayMs);
+        continue;
+      }
+      try {
+        const parsed = await parse3(response);
+        this.onRequestSuccess?.({
+          operation: operation2,
+          attempts: attempt + 1
+        });
+        return parsed;
+      } catch (err) {
+        if (opts.retryParseErrorsOnRetryableStatus && (opts.retryStructuredErrorsOnRetryableStatus !== false || !(err instanceof ProsaApiError)) && isRetryableObjectUploadStatus(response.status) && attempt < OBJECT_UPLOAD_MAX_ATTEMPTS - 1) {
+          const delayMs = objectUploadBackoffMs(attempt, response.headers);
+          this.onRetry?.({
+            operation: operation2,
+            attempt: attempt + 1,
+            maxAttempts: OBJECT_UPLOAD_MAX_ATTEMPTS,
+            delayMs,
+            reason: `HTTP ${response.status}: ${networkErrorReason(err)}`
+          });
+          await sleep(delayMs);
+          continue;
+        }
+        throw this.wrapRetriedError(operation2, attempt + 1, err);
+      }
+    }
+    throw this.wrapRetriedError(operation2, OBJECT_UPLOAD_MAX_ATTEMPTS, lastError);
+  }
+  wrapRetriedError(operation2, attempts, err) {
+    if (err instanceof ProsaApiError) return err;
+    if (attempts <= 1) return err instanceof Error ? err : new Error(String(err));
+    const reason = networkErrorReason(err);
+    return new CliUserError(`${operation2} failed after ${attempts} attempts: ${reason}`);
+  }
   async parseTrpc(path14, response) {
     const text = await response.text();
     if (!text) throw new CliUserError(`${path14}: empty response (status ${response.status})`);
@@ -28680,14 +28781,18 @@ var ProsaApiClient = class {
     return this.trpcMutation("sync.planUpload", input);
   }
   async syncCommitUpload(input, opts = {}) {
-    return this.trpcMutation("sync.commitUpload", input, {
+    return this.trpcMutationRetriable("sync.commitUpload", input, {
+      operation: "sync.commitUpload",
       headers: opts.idempotencyKey ? {
         "idempotency-key": opts.idempotencyKey
-      } : void 0
+      } : void 0,
+      retryStructuredErrorsOnRetryableStatus: false
     });
   }
   async syncVerifyPromotion(input) {
-    return this.trpcMutation("sync.verifyPromotion", input);
+    return this.trpcMutationRetriable("sync.verifyPromotion", input, {
+      operation: "sync.verifyPromotion"
+    });
   }
   async syncAckCleanup(input) {
     return this.trpcMutation("sync.ackCleanup", input);
@@ -28698,21 +28803,9 @@ var ProsaApiClient = class {
     } : void 0);
   }
   async uploadObjectBytes(input) {
-    let lastError;
-    for (let attempt = 0; attempt < OBJECT_UPLOAD_MAX_ATTEMPTS; attempt += 1) {
-      try {
-        return await this.uploadObjectBytesOnce(input, attempt);
-      } catch (err) {
-        lastError = err;
-        if (!isRetryableNetworkError(err) || attempt >= OBJECT_UPLOAD_MAX_ATTEMPTS - 1) {
-          throw err;
-        }
-        await sleep(objectUploadBackoffMs(attempt));
-      }
-    }
-    throw lastError instanceof Error ? lastError : new CliUserError("object upload failed");
+    return this.uploadObjectBytesOnce(input);
   }
-  async uploadObjectBytesOnce(input, attempt) {
+  async uploadObjectBytesOnce(input) {
     const url = new URL(`${this.baseUrl}/objects/${input.objectId}`);
     url.searchParams.set("batchId", input.batchId);
     url.searchParams.set("hash", input.hash);
@@ -28720,55 +28813,65 @@ var ProsaApiClient = class {
     url.searchParams.set("uncompressed", String(input.uncompressedSize));
     url.searchParams.set("compression", input.compression ?? "zstd");
     if (input.transportHash) url.searchParams.set("transportHash", input.transportHash);
-    const response = await this.fetchFn(url.toString(), {
-      method: "PUT",
-      headers: this.headers({
-        "content-type": "application/octet-stream"
-      }),
-      body: input.bytes
+    return this.retriableFetch({
+      operation: "object PUT upload",
+      request: /* @__PURE__ */ __name(() => this.fetchFn(url.toString(), {
+        method: "PUT",
+        headers: this.headers({
+          "content-type": "application/octet-stream"
+        }),
+        body: input.bytes
+      }), "request"),
+      parse: /* @__PURE__ */ __name(async (response) => {
+        const text = await response.text();
+        if (response.status >= 400) {
+          throw new CliUserError(`object upload failed: ${response.status} ${text}`);
+        }
+        const parsed = JSON.parse(text);
+        return {
+          alreadyExisted: Boolean(parsed.alreadyExisted)
+        };
+      }, "parse")
     });
-    const text = await response.text();
-    if (response.status >= 400) {
-      if (isRetryableObjectUploadStatus(response.status) && attempt < OBJECT_UPLOAD_MAX_ATTEMPTS - 1) {
-        await sleep(objectUploadBackoffMs(attempt, response.headers));
-        return this.uploadObjectBytesOnce(input, attempt + 1);
-      }
-      throw new CliUserError(`object upload failed: ${response.status} ${text}`);
-    }
-    const parsed = JSON.parse(text);
-    return {
-      alreadyExisted: Boolean(parsed.alreadyExisted)
-    };
   }
   async uploadObjectPack(input) {
     const url = new URL(`${this.baseUrl}/object-packs`);
     url.searchParams.set("batchId", input.batchId);
     const prepared = this.prepareObjectPackUpload(input.objects);
-    const binaryResponse = await this.fetchFn(url.toString(), {
-      method: "POST",
-      headers: this.headers({
-        "content-type": OBJECT_PACK_BINARY_CONTENT_TYPE
-      }),
-      body: encodeBinaryObjectPack({
-        entries: prepared.entries,
-        payload: prepared.payload
-      })
+    const binary = await this.retriableFetch({
+      operation: "object pack binary upload",
+      request: /* @__PURE__ */ __name(() => this.fetchFn(url.toString(), {
+        method: "POST",
+        headers: this.headers({
+          "content-type": OBJECT_PACK_BINARY_CONTENT_TYPE
+        }),
+        body: encodeBinaryObjectPack({
+          entries: prepared.entries,
+          payload: prepared.payload
+        })
+      }), "request"),
+      parse: /* @__PURE__ */ __name(async (response) => ({
+        status: response.status,
+        text: await response.text()
+      }), "parse")
     });
-    const binaryText = await binaryResponse.text();
-    if (!isUnsupportedBinaryObjectPackResponse(binaryResponse.status, binaryText)) {
-      return this.parseObjectPackUploadResponse(binaryResponse.status, binaryText);
-    }
-    const fallbackResponse = await this.fetchFn(url.toString(), {
-      method: "POST",
-      headers: this.headers({
-        "content-type": "application/json"
-      }),
-      body: JSON.stringify({
-        bytesBase64: prepared.payload.toString("base64"),
-        entries: prepared.entries
-      })
+    if (!isUnsupportedBinaryObjectPackResponse(binary.status, binary.text)) {
+      return this.parseObjectPackUploadResponse(binary.status, binary.text);
+    }
+    return this.retriableFetch({
+      operation: "object pack JSON upload",
+      request: /* @__PURE__ */ __name(() => this.fetchFn(url.toString(), {
+        method: "POST",
+        headers: this.headers({
+          "content-type": "application/json"
+        }),
+        body: JSON.stringify({
+          bytesBase64: prepared.payload.toString("base64"),
+          entries: prepared.entries
+        })
+      }), "request"),
+      parse: /* @__PURE__ */ __name(async (response) => this.parseObjectPackUploadResponse(response.status, await response.text()), "parse")
     });
-    return this.parseObjectPackUploadResponse(fallbackResponse.status, await fallbackResponse.text());
   }
   prepareObjectPackUpload(objects) {
     let offset = 0;
@@ -53667,6 +53770,46 @@ __name(runReadUploadPipeline, "runReadUploadPipeline");
 // src/cli/sync/promotion.ts
 import { rm as rm4 } from "fs/promises";
 import path11 from "path";
+var AdaptiveUploadConcurrencyController = class {
+  static {
+    __name(this, "AdaptiveUploadConcurrencyController");
+  }
+  ceiling;
+  onChange;
+  value;
+  successStreak = 0;
+  constructor(ceiling, onChange) {
+    this.ceiling = ceiling;
+    this.onChange = onChange;
+    this.value = Math.max(1, ceiling);
+  }
+  current() {
+    return this.value;
+  }
+  recordRetry() {
+    const previous = this.value;
+    this.value = Math.max(1, Math.floor(this.value / 2));
+    this.successStreak = 0;
+    if (this.value !== previous) this.onChange?.({
+      previous,
+      current: this.value,
+      reason: "retry"
+    });
+  }
+  recordSuccess() {
+    if (this.value >= this.ceiling) return;
+    this.successStreak += 1;
+    if (this.successStreak < 10) return;
+    this.successStreak = 0;
+    const previous = this.value;
+    this.value = Math.min(this.ceiling, this.value + 1);
+    if (this.value !== previous) this.onChange?.({
+      previous,
+      current: this.value,
+      reason: "success"
+    });
+  }
+};
 var BLAKE3_HEX_RE = /^[0-9a-f]{64}$/i;
 var OBJECT_PACK_ENTRY_LIMIT = 1024;
 var DEFAULT_OBJECT_PACK_MAX_BYTES = 8 * 1024 * 1024;
@@ -53729,9 +53872,10 @@ async function uploadObjectPut(client, batchId, object) {
   });
 }
 __name(uploadObjectPut, "uploadObjectPut");
-async function uploadMissingCasObjects({ client, batchId, missingObjects, objectConcurrency, maxObjectPackBytes = DEFAULT_OBJECT_PACK_MAX_BYTES }) {
+async function uploadMissingCasObjects({ client, batchId, missingObjects, objectConcurrency, uploadConcurrency, maxObjectPackBytes = DEFAULT_OBJECT_PACK_MAX_BYTES }) {
   const { packs, putObjects } = splitMissingObjectUploads(missingObjects, maxObjectPackBytes);
-  await mapConcurrent(packs, objectConcurrency, async (pack) => {
+  const concurrency = /* @__PURE__ */ __name(() => uploadConcurrency?.current() ?? objectConcurrency, "concurrency");
+  await mapConcurrent(packs, concurrency(), async (pack) => {
     await client.uploadObjectPack({
       batchId,
       objects: pack.map(({ entry, bytes }) => ({
@@ -53740,7 +53884,7 @@ async function uploadMissingCasObjects({ client, batchId, missingObjects, object
       }))
     });
   });
-  await mapConcurrent(putObjects, objectConcurrency, async (object) => {
+  await mapConcurrent(putObjects, concurrency(), async (object) => {
     await uploadObjectPut(client, batchId, object);
   });
   return {
@@ -54012,6 +54156,7 @@ async function uploadMissingCasObjectsWithReadPipeline(opts) {
       batchId: opts.batchId,
       missingObjects: batch,
       objectConcurrency: opts.objectConcurrency,
+      uploadConcurrency: opts.uploadConcurrency,
       ...opts.maxObjectPackBytes ? {
         maxObjectPackBytes: opts.maxObjectPackBytes
       } : {}
@@ -54721,7 +54866,7 @@ function readArtifactChunk(bundle, afterId, limit) {
   };
 }
 __name(readArtifactChunk, "readArtifactChunk");
-async function promoteChunk({ client, deviceId, storePath, casObjects, projection, label, metrics, objectConcurrency, maxObjectPackBytes, verbose }) {
+async function promoteChunk({ client, deviceId, storePath, casObjects, projection, label, metrics, objectConcurrency, uploadConcurrency, maxObjectPackBytes, verbose }) {
   const objectEntries = casObjects.map((c6) => c6.entry);
   const totalStart = Date.now();
   const planStart = Date.now();
@@ -54747,6 +54892,7 @@ async function promoteChunk({ client, deviceId, storePath, casObjects, projectio
     storePath,
     missingObjects,
     objectConcurrency,
+    uploadConcurrency,
     ...maxObjectPackBytes ? {
       maxObjectPackBytes
     } : {},
@@ -54858,7 +55004,7 @@ async function promotePhase(tasks, concurrency, worker) {
   return results;
 }
 __name(promotePhase, "promotePhase");
-async function promoteChunkedUpload({ client, deviceId, storePath, bundle, maxObjectsPerPlan, maxRowsPerCommit, maxObjectPackBytes, objectConcurrency, batchConcurrency, verbose, progress, totalBatches, checkpoint }) {
+async function promoteChunkedUpload({ client, deviceId, storePath, bundle, maxObjectsPerPlan, maxRowsPerCommit, maxObjectPackBytes, objectConcurrency, uploadConcurrency, batchConcurrency, verbose, progress, totalBatches, checkpoint }) {
   let batchCount = 0;
   let lastReceipt = null;
   let metrics = emptySyncMetrics(objectConcurrency);
@@ -54883,6 +55029,7 @@ async function promoteChunkedUpload({ client, deviceId, storePath, bundle, maxOb
         projection: toProjection(chunk.rows),
         label: `${label} batch ${phaseStart + cursor.sequence}`,
         objectConcurrency,
+        uploadConcurrency,
         ...maxObjectPackBytes ? {
           maxObjectPackBytes
         } : {},
@@ -54974,6 +55121,7 @@ async function promoteChunkedUpload({ client, deviceId, storePath, bundle, maxOb
         label: `chunk ${batchCount}`,
         metrics,
         objectConcurrency,
+        uploadConcurrency,
         ...maxObjectPackBytes ? {
           maxObjectPackBytes
         } : {},
@@ -55016,10 +55164,26 @@ function syncCommand() {
     if (!tenantHint) {
       throw new CliUserError("no active tenant. Run `prosa auth use <tenant>` first.");
     }
+    const uploadConcurrency = new AdaptiveUploadConcurrencyController(options.objectConcurrency, (change) => {
+      if (!options.verbose) return;
+      const direction = change.reason === "retry" ? "reduced" : "increased";
+      process.stderr.write(`adaptive object concurrency ${direction} ${change.previous}->${change.current} after ${change.reason}
+`);
+    });
     const client = new ProsaApiClient({
       baseUrl: server,
       token: entry.token,
-      tenantId: tenantHint
+      tenantId: tenantHint,
+      onRetry: /* @__PURE__ */ __name((event) => {
+        if (event.operation.startsWith("object ")) uploadConcurrency.recordRetry();
+        if (options.verbose) {
+          process.stderr.write(`retry ${event.operation} attempt=${event.attempt}/${event.maxAttempts} delayMs=${event.delayMs} reason=${event.reason}
+`);
+        }
+      }, "onRetry"),
+      onRequestSuccess: /* @__PURE__ */ __name((event) => {
+        if (event.operation.startsWith("object ")) uploadConcurrency.recordSuccess();
+      }, "onRequestSuccess")
     });
     const storePath = path12.resolve(options.store ?? defaultBundlePath13());
     const exists = await bundleManifestExists(storePath);
@@ -55111,6 +55275,7 @@ function syncCommand() {
             maxRowsPerCommit: handshake.limits.maxRowsPerCommit,
             maxObjectPackBytes: handshake.limits.maxObjectBytes,
             objectConcurrency: options.objectConcurrency,
+            uploadConcurrency,
             batchConcurrency: options.batchConcurrency,
             verbose: options.verbose,
             progress,