@madewithremy/admin 0.1.14 → 0.1.16

package/dist/prod.js

@@ -342,6 +342,62 @@ function statsForMethod(ctx, methodId, params = {}) {
   );
 }
 
+// src/sleep.ts
+function sleep(ms) {
+  return new Promise((r) => setTimeout(r, ms));
+}
+
+// src/poll.ts
+async function pollUntil(read, isSettled, options) {
+  const start2 = Date.now();
+  let movedAt = start2;
+  let lastKey;
+  let transientSince = null;
+  while (true) {
+    let value;
+    try {
+      value = await read();
+      transientSince = null;
+    } catch (err) {
+      if (!isTransientError(err)) {
+        throw err;
+      }
+      transientSince ??= Date.now();
+      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
+        throw err;
+      }
+      options.onProgress?.(
+        `retrying after a gateway error\u2026 (${elapsedSeconds(start2)}s)`
+      );
+      await sleep(options.pollMs);
+      continue;
+    }
+    if (isSettled(value)) {
+      return { settled: true, value };
+    }
+    if (options.progressKey) {
+      const key = options.progressKey(value);
+      if (key !== lastKey) {
+        lastKey = key;
+        movedAt = Date.now();
+      }
+    }
+    if (Date.now() - movedAt > options.timeoutMs) {
+      return { settled: false, value };
+    }
+    if (options.describe) {
+      options.onProgress?.(options.describe(value, start2));
+    }
+    await sleep(options.pollMs);
+  }
+}
+function elapsedSeconds(start2) {
+  return Math.round((Date.now() - start2) / 1e3);
+}
+function timedOut(timeoutMs, still) {
+  return `Timed out after ${Math.round(timeoutMs / 1e3)}s ${still}`;
+}
+
 // src/output.ts
 var PRETTY = process.stdout.isTTY === true;
 function out(data) {
@@ -352,6 +408,31 @@ function progress(message) {
   process.stderr.write(`${message}
 `);
 }
+function describeEta(eta, now = Date.now()) {
+  if (!eta) {
+    return null;
+  }
+  const minutes = Math.max(1, Math.round((Date.parse(eta) - now) / 6e4));
+  if (minutes < 90) {
+    return `about ${minutes} min left`;
+  }
+  const hours = Math.round(minutes / 60 * 10) / 10;
+  return `about ${hours} h left`;
+}
+function renderProgress(p, start2, extra) {
+  const count = p.total !== null ? `${p.done.toLocaleString()}/${p.total.toLocaleString()} ${p.unit}` : p.done > 0 ? `${p.done.toLocaleString()} ${p.unit}` : null;
+  const rate = p.ratePerMinute !== null && p.ratePerMinute > 0 ? `${Math.round(p.ratePerMinute).toLocaleString()}/min` : null;
+  const parts = [
+    `${p.phase}\u2026`,
+    count,
+    rate,
+    describeEta(p.etaAt),
+    extra ?? null,
+    p.stalled ? "no progress for 30 min" : null
+  ].filter((part) => Boolean(part));
+  const elapsed = start2 !== void 0 ? ` (${elapsedSeconds(start2)}s)` : "";
+  return `${parts.join(" \xB7 ")}${elapsed}`;
+}
 
 // src/commands/requests.ts
 var requestsSpecs = {
@@ -866,11 +947,6 @@ import { execFileSync } from "child_process";
 // src/config.ts
 var WORKSPACE_DIR = process.env["WORKSPACE_DIR"] || "/home/vercel-sandbox/workspace";
 
-// src/sleep.ts
-function sleep(ms) {
-  return new Promise((r) => setTimeout(r, ms));
-}
-
 // src/ops/releases.ts
 function list3(ctx, params = {}) {
   return call(
@@ -2874,50 +2950,6 @@ import path5 from "path";
 
 // src/ops/dataSources.ts
 import { createHash as createHash2 } from "crypto";
-
-// src/poll.ts
-async function pollUntil(read, isSettled, options) {
-  const start2 = Date.now();
-  let transientSince = null;
-  while (true) {
-    let value;
-    try {
-      value = await read();
-      transientSince = null;
-    } catch (err) {
-      if (!isTransientError(err)) {
-        throw err;
-      }
-      transientSince ??= Date.now();
-      if (Date.now() - transientSince > TRANSIENT_GRACE_MS) {
-        throw err;
-      }
-      options.onProgress?.(
-        `retrying after a gateway error\u2026 (${elapsedSeconds(start2)}s)`
-      );
-      await sleep(options.pollMs);
-      continue;
-    }
-    if (isSettled(value)) {
-      return { settled: true, value };
-    }
-    if (Date.now() - start2 > options.timeoutMs) {
-      return { settled: false, value };
-    }
-    if (options.describe) {
-      options.onProgress?.(options.describe(value, start2));
-    }
-    await sleep(options.pollMs);
-  }
-}
-function elapsedSeconds(start2) {
-  return Math.round((Date.now() - start2) / 1e3);
-}
-function timedOut(timeoutMs, still) {
-  return `Timed out after ${Math.round(timeoutMs / 1e3)}s ${still}`;
-}
-
-// src/ops/dataSources.ts
 var DEFAULT_WAIT_TIMEOUT_MS = 15 * 60 * 1e3;
 var POLL_MS2 = 3e3;
 function base(appId) {
@@ -2998,6 +3030,7 @@ async function waitForIngest(ctx, params) {
       timeoutMs,
       pollMs: POLL_MS2,
       describe: (tracked2, start2) => `ingesting\u2026 ${tracked2.length - pendingOf(tracked2).length}/${tracked2.length} done (${elapsedSeconds(start2)}s)`,
+      progressKey: (tracked2) => pendingOf(tracked2).length,
       onProgress
     }
   );
@@ -3034,6 +3067,7 @@ async function waitForCandidate(ctx, params) {
       timeoutMs,
       pollMs: POLL_MS2,
       describe: (progress2, start2) => progress2 ? `rebuilding\u2026 ${progress2.done}/${progress2.total} done (${elapsedSeconds(start2)}s)` : `rebuilding\u2026 (${elapsedSeconds(start2)}s)`,
+      progressKey: (progress2) => progress2?.processing ?? null,
       onProgress: params.onProgress
     }
   );
@@ -3186,7 +3220,10 @@ async function waitForMove(ctx, params) {
     {
       timeoutMs,
       pollMs: POLL_MS2,
-      describe: (source2, start2) => `moving\u2026 ${source2.migration.copied}/${source2.migration.total} documents (${elapsedSeconds(start2)}s)`,
+      describe: (source2, start2) => renderProgress(source2.migration.progress, start2),
+      // A corpus of any size moves as long as it keeps moving; the timeout
+      // bounds silence.
+      progressKey: (source2) => source2.migration?.copied ?? null,
       onProgress: params.onProgress
     }
   );
@@ -3354,6 +3391,13 @@ function resume(ctx, params) {
     controlBody(params)
   );
 }
+function retry(ctx, params) {
+  return call(
+    ctx,
+    "POST",
+    `${base2(ctx.appId)}/${encodeURIComponent(params.id)}/retry`
+  );
+}
 function cancel(ctx, params) {
   return call(
     ctx,
@@ -3380,6 +3424,9 @@ async function settle(ctx, params, isSettled) {
       timeoutMs,
       pollMs: POLL_MS3,
       describe: describeProgress,
+      // A load of any length is waited out while it moves; the timeout bounds
+      // silence. Planning moves by objects listed, a run by documents.
+      progressKey: (job2) => `${job2.state}:${job2.progress.done}`,
       onProgress: params.onProgress
     }
   );
@@ -3415,14 +3462,15 @@ async function settle(ctx, params, isSettled) {
   return { status: "done", job };
 }
 function describeProgress(job, start2) {
-  const elapsed = `(${elapsedSeconds(start2)}s)`;
-  if (job.state === "planning") {
-    return job.enumerationDone ? `planning\u2026 reading a sample of ${job.counts.objectsSeen.toLocaleString()} objects ${elapsed}` : `planning\u2026 ${job.counts.objectsSeen.toLocaleString()} objects counted ${elapsed}`;
+  if (job.state === "planning" && job.enumerationDone) {
+    return renderProgress(
+      { ...job.progress, phase: "planning" },
+      start2,
+      `reading a sample of ${job.counts.objectsSeen.toLocaleString()} objects`
+    );
   }
-  const total = job.plan?.projected.documents ?? job.counts.objectsSeen;
-  const processed = job.counts.documentsDone + job.counts.documentsSkipped + job.counts.documentsFailed;
-  const spend = `$${job.estimatedDollars.toFixed(2)}${job.budgetDollars !== null ? ` of $${job.budgetDollars.toFixed(2)}` : ""}`;
-  return `${job.state}\u2026 ${processed.toLocaleString()}/${total.toLocaleString()} documents \xB7 ${spend} ${elapsed}`;
+  const spend = job.state === "planning" ? null : `$${job.estimatedDollars.toFixed(2)}${job.budgetDollars !== null ? ` of $${job.budgetDollars.toFixed(2)}` : ""}`;
+  return renderProgress(job.progress, start2, spend);
 }
 
 // src/commands/_shared/dataSources.ts
@@ -3690,7 +3738,7 @@ function hydrate(ctx, params) {
     ctx,
     "POST",
     `${base4(ctx.appId)}/hydrate`,
-    { slug: params.slug, ...params.since ? { since: params.since } : {} }
+    { slug: params.slug }
   );
 }
 function reindex(ctx, params) {
@@ -3716,7 +3764,10 @@ async function waitForHydration(ctx, params) {
     {
       timeoutMs,
       pollMs: POLL_MS4,
-      describe: (source2, start2) => `loading\u2026 ${source2.hydration.copied}/${source2.hydration.total} documents (${elapsedSeconds(start2)}s)`,
+      describe: (source2, start2) => renderProgress(source2.hydration.progress, start2),
+      // A reload of any size is waited out while it moves; the timeout
+      // bounds silence.
+      progressKey: (source2) => source2.hydration?.copied ?? null,
       onProgress: params.onProgress
     }
   );
@@ -3904,10 +3955,9 @@ async function waitForSet(ctx, params) {
 var UUID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i;
 var dataSourceEvalsSpecs = {
   "datasources hydrate": {
-    usage: "Usage: remy-admin datasources hydrate [--source <slug>] [--since <iso>] [--wait] [--timeout <sec>]",
+    usage: "Usage: remy-admin datasources hydrate [--source <slug>] [--wait] [--timeout <sec>]",
     flags: {
       source: { type: "string" },
-      since: { type: "string" },
       wait: { type: "boolean" },
       timeout: { type: "string" }
     }
@@ -4078,14 +4128,7 @@ function selectorFromFlag(raw) {
 }
 async function hydrate2(ctx, a) {
   const slug = a.str("source") || DEFAULT_SOURCE;
-  const since = a.str("since");
-  if (since && Number.isNaN(Date.parse(since))) {
-    fatal("--since must be an ISO 8601 timestamp, e.g. 2026-09-08T15:17:00Z.");
-  }
-  const started = await hydrate(ctx, {
-    slug,
-    ...since ? { since: new Date(since).toISOString() } : {}
-  });
+  const started = await hydrate(ctx, { slug });
   if (started.resident || !a.bool("wait")) {
     out({
       dataSource: slug,
@@ -4380,7 +4423,7 @@ var dataSourceEvalsHandlers = {
 };
 var dataSourceEvalsHelp = `
 Warming and sampling:
-  remy-admin datasources hydrate [--source <slug>] [--since <iso>] [--wait]
+  remy-admin datasources hydrate [--source <slug>] [--wait]
   remy-admin datasources sample --source <slug> --size <n> [--as <slug>] [--filter <k=v,...|json>] [--stratify <key>] [--placement <id|shared>] [--wait]
 
   A large source whose index was evicted (the shared pool holds a working set,
@@ -4388,15 +4431,9 @@ Warming and sampling:
   background; small sources reload inside the search and never show it.
   \`hydrate\` starts that reload ahead of time \u2014 before a demo, say \u2014 and
   \`--wait\` blocks until the index is back. \`datasources list\` shows the
-  reload's progress under \`hydration\`.
-
-  \`hydrate --since <iso>\` is a refill rather than a warm-up: it copies every
-  document built at or after that time into the index whether or not the index
-  is resident. Dedicated capacity that restored from a snapshot is missing
-  everything written after the snapshot was taken; the platform starts this
-  refill itself when it sees the restore, and \`infra logs\` shows it. Use the
-  flag for a restore that predates that, with the snapshot's time from the
-  resource log.
+  reload's progress under \`hydration\`. The same reload is what dedicated
+  capacity runs on its own after a lost node or a resume; \`infra logs\` shows
+  it start, and the source's \`hydration\` shows it land.
 
   \`sample\` draws documents from a source into a new one that shares its
   exact pinned config, so what you measure on the sample says something about
@@ -4512,6 +4549,14 @@ var dataSourceJobsSpecs = {
       timeout: { type: "string" }
     }
   },
+  "datasources jobs retry": {
+    usage: "Usage: remy-admin datasources jobs retry <id> [--wait] [--timeout <sec>]",
+    positionals: [{ name: "id", required: true }],
+    flags: {
+      wait: { type: "boolean" },
+      timeout: { type: "string" }
+    }
+  },
   "datasources jobs cancel": {
     usage: "Usage: remy-admin datasources jobs cancel <id>",
     positionals: [{ name: "id", required: true }]
@@ -4620,6 +4665,15 @@ async function jobsResume(ctx, a) {
   }
   await waitAndReport(ctx, a, id, "job", { action: "resume" });
 }
+async function jobsRetry(ctx, a) {
+  const id = a.req("id");
+  const { job } = await retry(ctx, { id });
+  if (!a.bool("wait")) {
+    out({ job });
+    return;
+  }
+  await waitAndReport(ctx, a, id, "job", { action: "retry" });
+}
 async function jobsCancel(ctx, a) {
   out(await cancel(ctx, { id: a.req("id") }));
 }
@@ -4630,6 +4684,7 @@ var dataSourceJobsHandlers = {
   "datasources jobs approve": jobsApprove,
   "datasources jobs pause": jobsPause,
   "datasources jobs resume": jobsResume,
+  "datasources jobs retry": jobsRetry,
   "datasources jobs cancel": jobsCancel
 };
 var dataSourceJobsHelp = `
@@ -4640,6 +4695,7 @@ Bulk ingestion (jobs):
   remy-admin datasources jobs status <id> [--wait] [--timeout <sec>]
   remy-admin datasources jobs approve <id> [--wait] [--timeout <sec>]
   remy-admin datasources jobs pause|resume|cancel <id>
+  remy-admin datasources jobs retry <id> [--wait] [--timeout <sec>]
 
   A job loads a whole corpus at once. It reads either every object under a
   prefix of one of the app's file stores (\`files put\` fills one; an app can
@@ -4677,6 +4733,14 @@ Bulk ingestion (jobs):
   from the checkpoint and retries failed batches; \`jobs cancel\` stops it and
   keeps what was indexed. Search works on the partial corpus throughout.
 
+  A batch that fails five times stays failed and the job finishes without it,
+  with its documents left mid-build; \`jobs status\` shows the batch's error.
+  Once the cause is fixed (capacity that was restoring, a provider outage),
+  \`jobs retry <id>\` puts the job back to running over just those batches and
+  finishes it again. A source with documents mid-build refuses to move or
+  re-vectorize (data_source_busy) until they finish, so retry is what clears
+  that.
+
   One bulk operation per source: a job refuses to start during a move or while
   a candidate version exists, and moves, re-vectorizes and deletes refuse while
   a job is in flight (data_source_busy). Interactive \`add\` keeps working.
@@ -7643,7 +7707,14 @@ async function waitForPhase(ctx, params) {
     {
       timeoutMs,
       pollMs: POLL_MS5,
-      describe: (resource2, start2) => `${resource2.phase}${resource2.detail ? ` \xB7 ${resource2.detail}` : ""}${resource2.resizingTo ? ` \xB7 resizing to ${resource2.resizingTo.label}` : ""}\u2026 (${elapsedSeconds(start2)}s)`,
+      describe: (resource2, start2) => resource2.progress ? renderProgress(
+        resource2.progress,
+        start2,
+        resource2.resizingTo ? `resizing to ${resource2.resizingTo.label}` : null
+      ) : `${resource2.phase}${resource2.detail ? ` \xB7 ${resource2.detail}` : ""}${resource2.resizingTo ? ` \xB7 resizing to ${resource2.resizingTo.label}` : ""}\u2026 (${elapsedSeconds(start2)}s)`,
+      // A rebuild after a resume runs as long as it must; the timeout bounds
+      // silence, and a phase or step change counts as movement.
+      progressKey: (resource2) => `${resource2.phase}:${resource2.progress?.done ?? ""}:${resource2.detail ?? ""}`,
       onProgress: params.onProgress
     }
   );
@@ -7929,26 +8000,29 @@ Placing a data source on a resource:
 
   Searches on a source whose resource is not active fail with
   capacity_<phase>: capacity_hibernated once parked, capacity_hibernating /
-  capacity_resuming while it moves. \`active\` means searchable: a resumed
-  resource reports active only once its data is restored. capacity_restoring
-  is the rare case of an active instance that was replaced (node loss) and is
-  refilling from its snapshot. The messages are written for the app's end
-  user; the stable code is what to act on.
+  capacity_resuming while it moves. Once active, a source whose index is being
+  rebuilt from durable storage (after a resume, a resize or a lost node)
+  answers index_warming with the rebuild's progress and ETA until it lands;
+  \`datasources list\` shows it under hydration and the resource under
+  progress. The messages are written for the app's end user; the stable code
+  is what to act on.
 
 Resizing:
-  A resize keeps the data. The resource is parked (a fresh snapshot), its spec
-  is swapped, and it comes back up restored from that snapshot, so searches on
-  its sources pause for the few minutes it takes. A hibernated resource swaps
-  in place and stays parked. Growing needs a month's credits at the new rate;
-  shrinking is refused below what the resource holds (resize_too_small).
-  \`infra get\` shows resizingTo until the swap has happened.
+  A resize keeps the documents: the resource is parked, its spec is swapped,
+  and it comes back up with its sources' indexes rebuilt from durable storage
+  in the background (nothing is re-embedded). Searches on those sources answer
+  index_warming until the rebuild lands. A hibernated resource swaps in place
+  and stays parked. Growing needs a month's credits at the new rate; shrinking
+  is refused below what the resource holds (resize_too_small). \`infra get\`
+  shows resizingTo until the swap has happened.
 
 Phases and timing:
   requested \u2192 provisioning \u2192 active \u21C4 hibernated \u2192 destroyed, with hibernating /
   resuming / decommissioning in between and failed when a change cannot complete
-  (retried automatically; resume or hibernate to retry by hand). Each transition
-  takes about two minutes, more with more data: snapshots and restores scale
-  with the collection. --wait defaults to 600s; pass --timeout for a large one.
+  (retried automatically; resume or hibernate to retry by hand). A transition
+  takes a minute or two; the rebuild that follows a resume runs under its own
+  progress with a measured ETA. --wait gives up only after --timeout (default
+  600s) WITHOUT movement, so a long rebuild is waited out while it moves.
 
 Billing and the log:
   \`infra get\` reports resource.billing: what the resource has actually been
@@ -7957,7 +8031,8 @@ Billing and the log:
 
   The log is the platform's narration of the resource, not the instance's
   stdout: each step a transition went through ("Waiting for capacity",
-  "Restoring collections 2/3"), each failed attempt, and each failure with the
+  "Rebuilding archive v1 from its artifacts: 14,333,673 document(s)"), each
+  failed attempt, and each failure with the
   pod's recent Kubernetes events and the warnings-and-errors tail of the
   instance log at that moment (line.data). \`infra get\` includes the newest
   200 lines; \`infra logs\` pages the rest and, with --follow, prints new lines

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@madewithremy/admin",
-  "version": "0.1.14",
+  "version": "0.1.16",
   "description": "Admin SDK + CLI for managing production Remy apps",
   "type": "module",
   "main": "./dist/index.js",