@headband/cli 0.6.0 → 0.6.1

package/README.md

@@ -528,8 +528,11 @@ headband schema forget <index> [--yes] [--json]
 - `apply` runs the migration files the server does not have yet, in order. A
   new index is created and backfilled from every source row (rows
   `shouldIndex` rejects are left out). A schema change builds a new index
-  beside the live one, backfills it and switches the name over. Removing an
-  index needs `--allow-destructive`. `--json` prints
+  beside the live one, backfills it and switches the name over. Rows your app
+  writes during the backfill still go to the live index, so `apply` then
+  compares every document of the new index with its source row and writes the
+  ones that differ, once before the switch and once after it (CLI 0.6.1).
+  Removing an index needs `--allow-destructive`. `--json` prints
   `{appliedFiles, appliedOps, projectId}`.
 - `apply` recognises migrations the server already has from its schema state,
   so a fresh checkout or CI applies without `headband/migrations/.applied.json`.
@@ -600,7 +603,7 @@ changes to an outbox table, and the worker pushes them to Headband. The
 headband sync init [--table <name>] [--write] [--apply] [--json]
 headband sync worker [--loop] [--budget-ms <ms>] [--batch-size <n>] [--reconcile-every <n>] [--project <id>] [--json]
 headband sync worker --scaffold docker [--force] [--json]
-headband sync reconcile [--fix] [--json]
+headband sync reconcile [--content] [--fix] [--json]
 headband sync retry-dead-letters [--json]
 headband sync status [--json]
 ```
@@ -617,7 +620,9 @@ headband sync status [--json]
   `--json` sends progress to stderr and prints the report when the worker
   stops. A row that fails is printed with its model, ids and error; a batch
   that fails is retried row by row, so one bad row never fails the others.
-  Rows that keep failing move to the dead letters. Each invocation is recorded
+  Rows that keep failing move to the dead letters. A batch that fails because
+  Headband or the network is unavailable goes back to the queue without
+  counting an attempt, so an outage never fills the dead letters (CLI 0.6.1). Each invocation is recorded
   as one run under the connector `orm-sync` (`GET /v1/connectors/runs`); with
   the org-wide login key that needs `--project <id>` or the config's
   `projectId`. `--reconcile-every <n>` runs `reconcile` after every n-th batch.
@@ -626,8 +631,11 @@ headband sync status [--json]
   service for it. The container needs `DATABASE_URL` and `HEADBAND_API_KEY`.
 - `reconcile` compares the source rows that belong in each index (every row
   `shouldIndex` keeps) with the documents in it, and reports how many are
-  missing and how many are extra (counts, not ids). `--fix` pushes the missing
-  and deletes the extra. It exits
+  missing and how many are extra (counts, not ids). `--content` also compares
+  every document with its freshly resolved source row and counts the stale
+  ones; it reads the whole index, so it takes about as long as a backfill (CLI
+  0.6.1). `--fix` pushes the missing (and with `--content` the stale) and
+  deletes the extra. It exits
   `1` when drift is left. `--json` prints `{indexes, drift}`. The index's ids
   are read by cursor, so a Visor index past 1000 documents is compared id by
   id too.

package/dist/index.js

@@ -35,6 +35,7 @@ __export(index_exports, {
   main: () => main,
   parseArgv: () => parseArgv,
   parseExpires: () => parseExpires,
+  parseIndexList: () => parseIndexList,
   parsePermissions: () => parsePermissions,
   run: () => run,
   stdinIsInteractive: () => stdinIsInteractive,
@@ -1202,7 +1203,7 @@ function validateConfig(config, ormSchema) {
         continue;
       }
       const hasPath = model.relations.some(
-        (r) => r.fromModel === spec.model && r.toModel === index.source && r.kind === "many"
+        (r) => r.fromModel === index.source && r.toModel === spec.model && r.kind === "many"
       );
       if (!hasPath) {
         errors.push({
@@ -1488,6 +1489,67 @@ var ExportInterruptedError = class extends Error {
     this.name = "ExportInterruptedError";
   }
 };
+function refusedWalk(error) {
+  if (!(error instanceof ApiError) || error.code !== "feature_not_supported") return null;
+  if (!/primary key/i.test(error.message)) return null;
+  const pk = /add `([^`]+)`/.exec(error.message)?.[1] ?? null;
+  const named = ["filterableAttributes", "sortableAttributes", "displayedAttributes"].filter(
+    (name) => error.message.includes(name)
+  );
+  return { pk, lists: named.length ? named : ["filterableAttributes", "sortableAttributes"] };
+}
+var DECLARE_WAIT_MS = 15 * 6e4;
+var DECLARE_POLL_MS = 3e3;
+async function declareExportKey(client, index, io, opts = {}) {
+  const sleep = opts.sleep ?? ((ms) => new Promise((r) => setTimeout(r, ms)));
+  const now = opts.now ?? Date.now;
+  const probe = () => client.get(`/v1/documents?index=${encodeURIComponent(index)}&limit=1&cursor=`);
+  let refused;
+  try {
+    await probe();
+    return false;
+  } catch (error) {
+    refused = refusedWalk(error);
+    if (!refused) throw error;
+    if (!refused.pk) throw error;
+  }
+  const { pk, lists } = refused;
+  const path2 = `/v1/indexes/${encodeURIComponent(index)}/settings`;
+  const current = await client.get(path2);
+  const patch = {};
+  for (const name of lists) {
+    const list = Array.isArray(current?.[name]) ? current[name] : [];
+    if (list.includes("*") || list.includes(pk)) continue;
+    patch[name] = [...list, pk];
+  }
+  io.stderr(
+    `Declaring \`${pk}\` in ${Object.keys(patch).join(" and ") || lists.join(" and ")} on ${index}, so the export can read past 1000 documents. Visor rebuilds the index for this change, which can take minutes on a large one.`
+  );
+  if (Object.keys(patch).length > 0) {
+    try {
+      await client.patch(path2, patch);
+    } catch (error) {
+      if (!(error instanceof ApiError) || error.status < 500) throw error;
+    }
+  }
+  const deadline = now() + (opts.waitMs ?? DECLARE_WAIT_MS);
+  for (; ; ) {
+    try {
+      await probe();
+      io.stderr(`\`${pk}\` is declared. Exporting.`);
+      return true;
+    } catch (error) {
+      const waiting = refusedWalk(error) || error instanceof ApiError && error.status >= 500;
+      if (!waiting) throw error;
+    }
+    if (now() >= deadline) {
+      throw new CliError(
+        `\`${pk}\` is still not declared on ${index} after ${Math.round((opts.waitMs ?? DECLARE_WAIT_MS) / 6e4)} minutes. Visor may still be rebuilding it: run headband docs export again later.`
+      );
+    }
+    await sleep(DECLARE_POLL_MS);
+  }
+}
 function exportRefusalAdvice(error, index) {
   if (!(error instanceof ApiError) || error.code !== "feature_not_supported") return null;
   if (!/primary key/i.test(error.message)) return null;
@@ -1498,7 +1560,8 @@ function exportRefusalAdvice(error, index) {
   );
   const lists = missing.length ? missing.join(" and ") : "filterableAttributes and sortableAttributes";
   return [
-    `To export the whole index, declare ${key} on it once:`,
+    `To export the whole index, run it again with --declare-key: it declares ${key} on the index`,
+    `once (Visor rebuilds the index for it), waits for it, and exports. Or declare it yourself:`,
     `  1. headband indexes settings get ${index} > settings.json`,
     `  2. Add ${key} to ${lists} in settings.json.`,
     `  3. headband indexes settings set ${index} --file settings.json`,
@@ -1514,6 +1577,41 @@ function exportFailure(error, index) {
 ${advice}`);
 }
 
+// src/content-drift.ts
+async function readRemoteDocuments(client, index, primaryKey) {
+  const batchSize = await defaultExportBatchSize(client, index);
+  const docs = /* @__PURE__ */ new Map();
+  for await (const { results } of walkDocuments(client, { index, batchSize })) {
+    for (const doc of results) {
+      const id = doc?.[primaryKey];
+      if (id !== void 0 && id !== null) docs.set(String(id), doc);
+    }
+  }
+  return docs;
+}
+function wire(value) {
+  return value === void 0 ? null : JSON.parse(JSON.stringify(value));
+}
+function canonical(value) {
+  if (Array.isArray(value)) return `[${value.map(canonical).join(",")}]`;
+  if (value !== null && typeof value === "object") {
+    const entries = Object.keys(value).sort().map((k) => `${JSON.stringify(k)}:${canonical(value[k])}`);
+    return `{${entries.join(",")}}`;
+  }
+  return JSON.stringify(value ?? null);
+}
+function documentsMatch(expected, remote, primaryKey) {
+  if (!remote) return false;
+  for (const [key, raw] of Object.entries(expected)) {
+    if (key === primaryKey) {
+      if (String(raw) !== String(remote[key])) return false;
+      continue;
+    }
+    if (canonical(wire(raw)) !== canonical(wire(remote[key]))) return false;
+  }
+  return true;
+}
+
 // ../../src/lib/density/catalog-constants.ts
 function formatMemory(mb) {
   return mb >= 1024 ? `${mb / 1024} GB` : `${mb} MB`;
@@ -2357,6 +2455,14 @@ var HeadbandClient = class {
     if (opts?.projectId !== void 0) body.projectId = opts.projectId;
     await this.request("POST", "/v1/indexes", body);
   }
+  /** The index (GET /v1/indexes/{uid}): its engine among the rest. */
+  async getIndex(uid) {
+    return this.request("GET", `/v1/indexes/${encodeURIComponent(uid)}`);
+  }
+  /** Deletes an index (DELETE /v1/indexes/{uid}): `schema apply` removes a reindex it could not set up. */
+  async deleteIndex(uid) {
+    await this.request("DELETE", `/v1/indexes/${encodeURIComponent(uid)}`);
+  }
   async patchSettings(uid, settings) {
     await this.request("PATCH", `/v1/indexes/${uid}/settings`, settings);
   }
@@ -2512,7 +2618,36 @@ async function createIndexIdempotent(ctx, uid, opts) {
     ctx.io.stdout(`Index "${uid}" already exists. Continuing.`);
   }
 }
+var ES_EVERY_FIELD = [
+  "searchableAttributes",
+  "filterableAttributes",
+  "sortableAttributes"
+];
+async function indexEngine(ctx, uid) {
+  try {
+    return (await ctx.client.getIndex(uid)).engine ?? null;
+  } catch {
+    return null;
+  }
+}
+function settingsForEngine(ctx, uid, engine, settings) {
+  if (engine !== "elasticsearch") return settings;
+  const out = { ...settings };
+  const left = [];
+  for (const key of ES_EVERY_FIELD) {
+    if (out[key] === void 0) continue;
+    delete out[key];
+    left.push(key);
+  }
+  if (left.length > 0) {
+    ctx.io.stdout(
+      `"${uid}" is on Elasticsearch, which searches, filters and sorts every field without these lists, so they are not sent: ${left.join(", ")}.`
+    );
+  }
+  return out;
+}
 async function patchSettingsBeforeBackfill(ctx, uid, settings) {
+  if (Object.keys(settings).length === 0) return false;
   try {
     await ctx.client.patchSettings(uid, settings);
     return false;
@@ -2536,13 +2671,14 @@ async function patchSettingsAfterBackfill(ctx, uid, settings) {
 function settingsRefused(uid, err) {
   const message = err instanceof Error ? err.message : String(err);
   const { status, code } = err ?? {};
-  return new CliError(
+  const refused = new CliError(
     `Index "${uid}" exists, but its settings were refused: ${message.replace(/\.?$/, ".")} Change the index's settings in headband.config.ts, then re-run "headband schema apply": it picks up the existing index.`,
     {
       ...typeof status === "number" ? { exitCode: exitCodeForStatus(status) } : {},
       ...typeof code === "string" ? { code } : {}
     }
   );
+  return Object.assign(refused, { refusal: message });
 }
 function postState(client, state) {
   return client.postSchemaState(state);
@@ -2590,13 +2726,83 @@ async function backfill(ctx, def, targetUid, abortPoint, onPage) {
   );
   return { pushed, excluded };
 }
+async function catchUp(ctx, def, targetUid, when) {
+  if (!ctx.readRemoteDocuments) return;
+  const remote = await ctx.readRemoteDocuments(targetUid, def.primaryKey);
+  const keep = /* @__PURE__ */ new Set();
+  let pushed = 0;
+  let cursor = null;
+  for (; ; ) {
+    const page = await ctx.adapter.listIds(def.source, cursor, ctx.backfillBatchSize);
+    if (page.ids.length === 0) break;
+    const docs = await ctx.adapter.resolveDocs(def.name, page.ids);
+    const toPush = [];
+    for (const doc of docs) {
+      const sourceRow = doc.__source ?? doc;
+      delete doc.__source;
+      if (def.shouldIndex && !def.shouldIndex(sourceRow)) continue;
+      const key = String(doc[def.primaryKey]);
+      keep.add(key);
+      if (!documentsMatch(doc, remote.get(key), def.primaryKey)) toPush.push(doc);
+    }
+    if (toPush.length > 0) {
+      const result = await ctx.client.bulkPush(targetUid, toPush, def.primaryKey);
+      if (result?.partialFailure) {
+        throw new CliError(
+          `Catching up "${def.name}" ${when} stopped: ${partialFailureSummary(result.partialFailure)}. Re-run "headband schema apply" once resolved.`
+        );
+      }
+      pushed += toPush.length;
+    }
+    cursor = page.nextCursor;
+    if (cursor === null || page.ids.length < ctx.backfillBatchSize) break;
+  }
+  const extra = [...remote.keys()].filter((key) => !keep.has(key));
+  for (let offset = 0; offset < extra.length; offset += ctx.backfillBatchSize) {
+    const result = await ctx.client.deleteDocs(
+      targetUid,
+      extra.slice(offset, offset + ctx.backfillBatchSize)
+    );
+    if (result?.partialFailure) {
+      throw new CliError(
+        `Catching up "${def.name}" ${when} stopped: ${partialFailureSummary(result.partialFailure)}. Re-run "headband schema apply" once resolved.`
+      );
+    }
+  }
+  ctx.io.stdout(
+    `Caught up "${def.name}" ${when}: ${pushed} changed or new document(s) written, ${extra.length} removed.`
+  );
+}
+async function abandonReindex(ctx, index, physical, err) {
+  let removed;
+  try {
+    await ctx.client.deleteIndex(physical);
+    removed = `The new index "${physical}" was removed`;
+  } catch (deleteErr) {
+    const why = deleteErr instanceof Error ? deleteErr.message : String(deleteErr);
+    removed = `The new index "${physical}" could not be removed (${why}): remove it with "headband indexes delete ${physical}"`;
+  }
+  const reason = err?.refusal ?? (err instanceof Error ? err.message : String(err));
+  return new CliError(
+    `The reindex of "${index}" stopped: the new index's settings were refused: ${String(reason).replace(/\.?$/, ".")} ${removed}, and "${index}" is unchanged. Change the index's settings in headband.config.ts, then re-run "headband schema apply".`,
+    {
+      ...err instanceof CliError ? { exitCode: err.exitCode, code: err.code } : {}
+    }
+  );
+}
 async function applyReindex(op, ctx, existing, expectedParentHash) {
   const def = ctx.config.indexes.find((d) => d.name === op.index);
   if (!def) {
     throw new CliError(`Cannot reindex "${op.index}": no matching index in the loaded config.`);
   }
+  const settings = settingsForEngine(ctx, op.index, await indexEngine(ctx, op.index), op.settings);
   await createIndexIdempotent(ctx, op.toPhysicalUid, { primaryKey: def.primaryKey });
-  const settingsAfter = await patchSettingsBeforeBackfill(ctx, op.toPhysicalUid, op.settings);
+  let settingsAfter;
+  try {
+    settingsAfter = await patchSettingsBeforeBackfill(ctx, op.toPhysicalUid, settings);
+  } catch (err) {
+    throw await abandonReindex(ctx, op.index, op.toPhysicalUid, err);
+  }
   const version = (existing?.version ?? 0) + 1;
   const schemaHashValue = docShapeHash(def);
   const settingsHashValue = settingsHash(def);
@@ -2625,8 +2831,10 @@ async function applyReindex(op, ctx, existing, expectedParentHash) {
     "before the alias flip",
     (cursor) => writeState("migrating", cursor)
   );
-  if (settingsAfter) await patchSettingsAfterBackfill(ctx, op.toPhysicalUid, op.settings);
+  if (settingsAfter) await patchSettingsAfterBackfill(ctx, op.toPhysicalUid, settings);
+  await catchUp(ctx, def, op.toPhysicalUid, "before the alias flip");
   await ctx.client.patchAlias(op.index, op.toPhysicalUid);
+  await catchUp(ctx, def, op.index, "after the alias flip");
   await writeState("applied", null);
   return {
     indexUid: op.index,
@@ -2646,9 +2854,15 @@ async function applyOp(op, ctx, existing, expectedParentHash) {
       );
     }
     await createIndexIdempotent(ctx, op.index, { primaryKey: def.primaryKey });
-    const settingsAfter = await patchSettingsBeforeBackfill(ctx, op.index, op.settings);
+    const settings = settingsForEngine(
+      ctx,
+      op.index,
+      await indexEngine(ctx, op.index),
+      op.settings
+    );
+    const settingsAfter = await patchSettingsBeforeBackfill(ctx, op.index, settings);
     await backfill(ctx, def, op.index, "before recording the index as applied");
-    if (settingsAfter) await patchSettingsAfterBackfill(ctx, op.index, op.settings);
+    if (settingsAfter) await patchSettingsAfterBackfill(ctx, op.index, settings);
     const next = {
       indexUid: op.index,
       version: 1,
@@ -2664,7 +2878,13 @@ async function applyOp(op, ctx, existing, expectedParentHash) {
     if (!existing) {
       throw new CliError(`Cannot patch settings for "${op.index}": no applied schema state found.`);
     }
-    await ctx.client.patchSettings(op.index, op.settings);
+    const settings = settingsForEngine(
+      ctx,
+      op.index,
+      await indexEngine(ctx, op.index),
+      op.settings
+    );
+    if (Object.keys(settings).length > 0) await ctx.client.patchSettings(op.index, settings);
     const def = ctx.config.indexes.find((d) => d.name === op.index);
     const next = {
       ...existing,
@@ -2720,7 +2940,8 @@ async function runSchemaApply(deps) {
     client: deps.client,
     backfillBatchSize: deps.backfillBatchSize ?? DEFAULT_BACKFILL_BATCH_SIZE,
     io: deps.io,
-    projectId: deps.projectId
+    projectId: deps.projectId,
+    readRemoteDocuments: deps.readRemoteDocuments
   };
   let appliedOps = 0;
   let appliedFiles = 0;
@@ -2917,6 +3138,12 @@ function pgOutboxDriver(exec, opts) {
       );
       return rows?.[0]?.status === "dead_letter" ? "dead_letter" : "pending";
     },
+    async release(ids, error) {
+      await exec(
+        `UPDATE ${table} SET last_error = $2, claimed_at = NULL WHERE id = ANY($1) AND status = 'pending'`,
+        [ids, error]
+      );
+    },
     async deadLetters(limit) {
       const { rows } = await exec(
         `SELECT * FROM ${table} WHERE status = 'dead_letter' ORDER BY id LIMIT $1`,
@@ -3154,6 +3381,7 @@ function describeRow(row) {
 var emptyOutcome = () => ({
   done: [],
   retry: [],
+  release: [],
   deadLetter: [],
   pushed: 0,
   deleted: 0
@@ -3213,7 +3441,7 @@ async function runSyncWorker(opts) {
       primaryKeyByIndex,
       backoff
     );
-    return { ...rowOutcomes(batch, resolved, failures), pushed, deleted };
+    return { ...rowOutcomes(batch, resolved, failures), release: [], pushed, deleted };
   };
   const processIsolated = async (batch) => {
     const outcome = emptyOutcome();
@@ -3226,7 +3454,10 @@ async function runSyncWorker(opts) {
         outcome.pushed += one.pushed;
         outcome.deleted += one.deleted;
       } catch (err) {
-        outcome.retry.push([row, errorMessage(err)]);
+        (isTransientBatchError(err) ? outcome.release : outcome.retry).push([
+          row,
+          errorMessage(err)
+        ]);
       }
     }
     return outcome;
@@ -3245,12 +3476,14 @@ async function runSyncWorker(opts) {
     try {
       outcome = await processBatch(rows);
     } catch (err) {
-      if (rows.length > 1 && !isTransientBatchError(err)) {
+      const transient = isTransientBatchError(err);
+      if (rows.length > 1 && !transient) {
         outcome = await processIsolated(rows);
       } else {
         wholeBatchFailed = true;
         const message = errorMessage(err);
-        outcome = { ...emptyOutcome(), retry: rows.map((row) => [row, message]) };
+        const failed = rows.map((row) => [row, message]);
+        outcome = transient ? { ...emptyOutcome(), release: failed } : { ...emptyOutcome(), retry: failed };
         log(
           `[headband] a batch of ${rows.length} outbox row(s) failed and will be retried: ${message}`
         );
@@ -3272,6 +3505,17 @@ async function runSyncWorker(opts) {
         log(`[headband] ${describeRow(row)} failed: ${message}`);
       }
     }
+    if (outcome.release.length > 0) {
+      if (outbox.release) {
+        await outbox.release(
+          outcome.release.map(([row]) => row.id),
+          outcome.release[0][1]
+        );
+        report.retried += outcome.release.length;
+      } else {
+        outcome.retry.push(...outcome.release);
+      }
+    }
     for (const [row, message] of outcome.retry) {
       const status = await outbox.markFailed(row.id, message);
       if (status === "dead_letter") {
@@ -3288,7 +3532,7 @@ async function runSyncWorker(opts) {
         }
       }
     }
-    if (outcome.retry.length > 0) {
+    if (outcome.retry.length > 0 || outcome.release.length > 0) {
       consecutiveFailures += 1;
       await backoff(consecutiveFailures);
     } else {
@@ -3512,6 +3756,66 @@ async function runSyncWorkerCmd(deps) {
     }
   }
 }
+async function reconcileContent(deps, def, pageSize) {
+  const remote = await deps.readRemoteDocuments(def.name, def.primaryKey);
+  const keep = /* @__PURE__ */ new Set();
+  let missing = 0;
+  let stale = 0;
+  let cursor = null;
+  for (; ; ) {
+    const page = await deps.adapter.listIds(def.source, cursor, pageSize);
+    if (page.ids.length === 0) break;
+    const docs = await deps.adapter.resolveDocs(def.name, page.ids);
+    const toPush = [];
+    for (const doc of docs) {
+      const sourceRow = doc.__source ?? doc;
+      delete doc.__source;
+      if (def.shouldIndex && !def.shouldIndex(sourceRow)) continue;
+      const key = String(doc[def.primaryKey]);
+      keep.add(key);
+      const indexed = remote.get(key);
+      if (!indexed) missing += 1;
+      else if (!documentsMatch(doc, indexed, def.primaryKey)) stale += 1;
+      else continue;
+      toPush.push(doc);
+    }
+    if (deps.fix && toPush.length > 0) {
+      const result = await deps.client.bulkPush(def.name, toPush, def.primaryKey);
+      if (result?.partialFailure) {
+        throw new CliError(
+          `Repair push for "${def.name}" was incomplete: ${partialFailureSummary(result.partialFailure)}.`
+        );
+      }
+    }
+    if (page.nextCursor === null) break;
+    cursor = page.nextCursor;
+  }
+  const extraIds = [...remote.keys()].filter((key) => !keep.has(key));
+  if (deps.fix) {
+    for (let offset = 0; offset < extraIds.length; offset += pageSize) {
+      const result = await deps.client.deleteDocs(
+        def.name,
+        extraIds.slice(offset, offset + pageSize)
+      );
+      if (result?.partialFailure) {
+        throw new CliError(
+          `Removing extra documents from "${def.name}" was incomplete: ${partialFailureSummary(result.partialFailure)}.`
+        );
+      }
+    }
+  }
+  const drift = missing + extraIds.length + stale;
+  return {
+    index: def.name,
+    sourceCount: keep.size,
+    remoteCount: remote.size,
+    missing,
+    extra: extraIds.length,
+    stale,
+    drift,
+    fixed: Boolean(deps.fix) && drift !== 0
+  };
+}
 async function indexableSourceIds(adapter, def, pageSize) {
   const ids = /* @__PURE__ */ new Map();
   let cursor = null;
@@ -3556,6 +3860,17 @@ async function runSyncReconcile(deps) {
   const pageSize = deps.pageSize ?? RECONCILE_PAGE_SIZE;
   const rows = [];
   for (const def of deps.config.indexes) {
+    if (deps.content && deps.readRemoteDocuments) {
+      try {
+        rows.push(await reconcileContent(deps, def, pageSize));
+        continue;
+      } catch (err) {
+        if (err instanceof CliError) throw err;
+        deps.io.stderr(
+          `Could not read the documents of "${def.name}", comparing ids only: ${err instanceof Error ? err.message : String(err)}`
+        );
+      }
+    }
     const source = await indexableSourceIds(deps.adapter, def, pageSize);
     let remoteKeys = null;
     let remoteCount = -1;
@@ -3619,6 +3934,7 @@ async function runSyncReconcile(deps) {
     }
     rows.push(row);
   }
+  const contentCompared = rows.some((r) => r.stale !== void 0 && r.stale !== null);
   printTable(
     deps.io,
     rows.map((r) => ({
@@ -3626,9 +3942,10 @@ async function runSyncReconcile(deps) {
       remoteCount: r.remoteCount === -1 ? "unknown" : r.remoteCount,
       missing: r.missing ?? "",
       extra: r.extra ?? "",
+      stale: r.stale ?? "",
       fixed: r.fixed ? "yes" : ""
     })),
-    ["index", "sourceCount", "remoteCount", "missing", "extra", "drift", "fixed"]
+    contentCompared ? ["index", "sourceCount", "remoteCount", "missing", "extra", "stale", "drift", "fixed"] : ["index", "sourceCount", "remoteCount", "missing", "extra", "drift", "fixed"]
   );
   const unfixed = rows.filter((r) => r.drift !== 0 && !r.fixed);
   if (unfixed.length > 0 && !deps.fix) {
@@ -3798,13 +4115,14 @@ var syncWorker = [
   "package name, and prints a docker-compose service."
 ].join("\n");
 var syncReconcile = [
-  "Usage: headband sync reconcile [--fix] [--json]",
+  "Usage: headband sync reconcile [--content] [--fix] [--json]",
   "",
   "Per index, compares the source rows that belong in it (every row shouldIndex keeps) with",
   "the documents in it, and prints how many documents are missing and how many are extra",
   "(counts, not ids; --json gives the same counts). --fix pushes the missing ones and deletes",
-  "the extra ones. Exits 1 when drift is left. Documents whose content went",
-  "stale are not detected: a reindex (schema apply) rewrites every document.",
+  "the extra ones. Exits 1 when drift is left. --content also compares every document with its",
+  "freshly resolved source row and counts the stale ones; with --fix it rewrites them. It reads",
+  "the whole index, so it takes about as long as a backfill.",
   "Needs the adapter export and DATABASE_URL."
 ].join("\n");
 var syncRetry = [
@@ -3839,7 +4157,7 @@ var SUBCOMMAND_HELP = {
     "  headband sync init [--table <name>] [--write] [--apply] [--json]",
     "  headband sync worker [--loop] [--budget-ms <ms>] [--batch-size <n>] [--reconcile-every <n>] [--project <id>] [--json]",
     "  headband sync worker --scaffold docker [--force] [--json]",
-    "  headband sync reconcile [--fix] [--json]",
+    "  headband sync reconcile [--content] [--fix] [--json]",
     "  headband sync retry-dead-letters [--json]",
     "  headband sync status [--json]",
     "",
@@ -4984,6 +5302,7 @@ var VALUE_FLAGS = /* @__PURE__ */ new Set([
   "--client",
   "--instance",
   "--instances",
+  "--indexes",
   "--db-url",
   "--owner-url",
   "--tls-mode",
@@ -5020,6 +5339,7 @@ var BOOLEAN_FLAGS = /* @__PURE__ */ new Set([
   "--write",
   "--force",
   "--fix",
+  "--content",
   "--loop",
   "--accept-broad-role",
   "--rebuild",
@@ -5183,7 +5503,7 @@ Commands:
   branch checkout <name> [--project <id>] [--json]
   branch reset|rollback|restore|set-primary (coming with bucket storage)
   keys list [--json]
-  keys create <name> --type <admin|search> [--project <id>] [--branch <name>] [--instances <id,id>] [--permissions <a,b>] [--expires <30d|ISO date>] [--json]
+  keys create <name> --type <admin|search> [--project <id>] [--branch <name>] [--instances <id,id>] [--indexes <uid,uid>] [--permissions <a,b>] [--expires <30d|ISO date>] [--json]
   keys revoke <id> [--yes] [--json]
   indexes list [--json]                      (index works in place of indexes)
   indexes create <uid> [--primary-key <pk>] [--engine <engine>] [--instance <id>] [--json]
@@ -5201,7 +5521,7 @@ Commands:
   instances connect <id> [--client <node|curl>] [--json]
   warm <index>... | --project <id> [--branch <name>] [--level auto|stage|start] [--json]
   docs push <file> --index <uid> [--json]
-  docs export --index <uid> [--out <file>] [--format <jsonl|json>] [--batch <n>]
+  docs export --index <uid> [--out <file>] [--format <jsonl|json>] [--batch <n>] [--declare-key]
   docs delete <docId>... --index <uid> [--json]
   search <index> [<query>] [--limit N] [--filter <expr>] [--mode <instant|retrieval>] [--json]
   tasks list [--index <uid>] [--limit <n>] [--json]
@@ -5216,7 +5536,7 @@ Commands:
   sync init [--table <name>] [--write] [--apply] [--json]
   sync worker [--loop] [--budget-ms <ms>] [--batch-size <n>] [--reconcile-every <n>] [--project <id>] [--json]
   sync worker --scaffold docker [--force] [--json]
-  sync reconcile [--fix] [--json]
+  sync reconcile [--content] [--fix] [--json]
   sync retry-dead-letters [--json]
   sync status [--json]
 
@@ -5252,13 +5572,14 @@ function printCommandHelp(command, io) {
     keys: [
       "Usage:",
       "  headband keys list [--json]",
-      "  headband keys create <name> --type <admin|search> [--project <id>] [--branch <name>] [--instances <id,id>] [--permissions <a,b>] [--expires <30d|ISO date>] [--json]",
+      "  headband keys create <name> --type <admin|search> [--project <id>] [--branch <name>] [--instances <id,id>] [--indexes <uid,uid>] [--permissions <a,b>] [--expires <30d|ISO date>] [--json]",
       "  headband keys revoke <id> [--yes] [--json]",
       "",
       "keys create mints a key for one branch: --branch (with --project, else the checkout's project),",
       "else the checked-out branch, else the --project's primary branch. With none of these,",
       "--type admin mints an org key, and --type search is refused: a search key needs a branch.",
-      "--instances <id,id> limits the key to those dedicated engines (on a branch: its engines among them)."
+      "--instances <id,id> limits the key to those dedicated engines (on a branch: its engines among them).",
+      "--indexes <uid,uid> limits a search key to those indexes; any other index answers 404."
     ].join("\n"),
     indexes: [
       "Usage:",
@@ -5290,7 +5611,7 @@ function printCommandHelp(command, io) {
     docs: [
       "Usage:",
       "  headband docs push <file> --index <uid> [--json]",
-      "  headband docs export --index <uid> [--out <file>] [--format <jsonl|json>] [--batch <n>] [--json]",
+      "  headband docs export --index <uid> [--out <file>] [--format <jsonl|json>] [--batch <n>] [--declare-key] [--json]",
       "  headband docs delete <docId>... --index <uid> [--json]",
       "",
       "docs push sends the documents in a .json (an array) or .jsonl file, 1000 at a time.",
@@ -5300,6 +5621,9 @@ function printCommandHelp(command, io) {
       "  --out <file>     to a file. The format follows the extension (.json is one array,",
       "                   anything else JSONL) unless --format says otherwise.",
       "  --batch <n>      documents per request. Default 1000, or 250 on Typesense.",
+      "  --declare-key    a Visor index past 1000 documents whose primary key is not",
+      "                   filterable and sortable (made before 2026-10-21): declare the",
+      "                   key on it first (Visor rebuilds the index), wait, then export.",
       "  If the export stops on an error, the documents written so far are kept in",
       "  <file>.partial, and <file> is not written.",
       "",
@@ -5510,7 +5834,12 @@ function parsePermissions(raw) {
 }
 function keyRow(key) {
   const branch = key.branch;
-  return { ...key, branch: branch?.name ?? "", engines: enginesColumn(key.instanceIds) };
+  return {
+    ...key,
+    branch: branch?.name ?? "",
+    engines: enginesColumn(key.instanceIds),
+    indexes: enginesColumn(key.indexUids)
+  };
 }
 function enginesColumn(instanceIds) {
   if (instanceIds === null) return "all";
@@ -5521,6 +5850,21 @@ function enginesColumn(instanceIds) {
   }
   return "none";
 }
+function parseIndexList(raw) {
+  const uids = [
+    ...new Set(
+      raw.split(",").map((x) => x.trim()).filter(Boolean)
+    )
+  ];
+  if (uids.length === 0) throw new CliError("--indexes needs at least one index uid.");
+  const bad = uids.find((uid) => !/^[A-Za-z0-9_-]{1,64}$/.test(uid));
+  if (bad) {
+    throw new CliError(
+      `Not an index uid: ${bad}. Index uids are letters, numbers, hyphens and underscores.`
+    );
+  }
+  return uids;
+}
 function parseInstanceList(raw) {
   const ids = [
     ...new Set(
@@ -5553,6 +5897,7 @@ async function runKeys(args, flags, io, confirm, cwd) {
         "projectId",
         "branch",
         "engines",
+        "indexes",
         "environment",
         "lastUsedAt",
         "expiresAt"
@@ -5574,6 +5919,11 @@ async function runKeys(args, flags, io, confirm, cwd) {
     }
     const instances = stringFlag3(flags, "--instances");
     if (instances !== void 0) body.instanceIds = parseInstanceList(instances);
+    if (stringFlag3(flags, "--index") !== void 0) {
+      throw new CliError("Unknown option --index for keys create. Did you mean --indexes?");
+    }
+    const indexes = stringFlag3(flags, "--indexes");
+    if (indexes !== void 0) body.indexUids = parseIndexList(indexes);
     const permissions = stringFlag3(flags, "--permissions");
     if (permissions) body.permissions = parsePermissions(permissions);
     const expires = stringFlag3(flags, "--expires");
@@ -5593,6 +5943,12 @@ async function runKeys(args, flags, io, confirm, cwd) {
         typeof keyId === "string" && keyId ? `This API did not limit the new key to the chosen engines, so key ${keyId} reaches wider than asked. Revoke it now: headband keys revoke ${keyId}` : "This API did not limit the new key to the chosen engines, so it reaches wider than asked. Find it with `headband keys list` and revoke it with `headband keys revoke <id>`."
       );
     }
+    if (indexes !== void 0 && !Array.isArray(response.data.key.indexUids)) {
+      const keyId = response.data.key.id;
+      throw new CliError(
+        typeof keyId === "string" && keyId ? `This API did not limit the new key to the chosen indexes, so key ${keyId} reaches every index of its branch. Revoke it now: headband keys revoke ${keyId}` : "This API did not limit the new key to the chosen indexes, so it reaches every index of its branch. Find it with `headband keys list` and revoke it with `headband keys revoke <id>`."
+      );
+    }
     if (boolFlag(flags, "--json")) {
       printJson(io, response);
       return;
@@ -5602,7 +5958,17 @@ async function runKeys(args, flags, io, confirm, cwd) {
     printTable(
       io,
       [keyRow(response.data.key)],
-      ["id", "name", "type", "environment", "branch", "engines", "keyPrefix", "expiresAt"]
+      [
+        "id",
+        "name",
+        "type",
+        "environment",
+        "branch",
+        "engines",
+        "indexes",
+        "keyPrefix",
+        "expiresAt"
+      ]
     );
     return;
   }
@@ -5738,6 +6104,7 @@ async function runDocsExport(flags, context, cwd) {
   const format = exportFormat(flags, out);
   const client = clientFromFlags(flags, cwd);
   const batchSize = parseIntFlag(flags, "--batch") ?? await defaultExportBatchSize(client, index);
+  if (boolFlag(flags, "--declare-key")) await declareExportKey(client, index, io);
   if (!out) {
     const writeLine = context.writeLine ?? ((line) => io.stdout(line));
     try {
@@ -5940,7 +6307,8 @@ async function runSchema(args, flags, io, confirm, cwd) {
       dir,
       allowDestructive,
       io: chatter,
-      projectId
+      projectId,
+      readRemoteDocuments: (uid, primaryKey) => readRemoteDocuments(apiClient, uid, primaryKey)
     });
     if (json) printJson(io, { ...result, projectId: projectId ?? null });
     else {
@@ -6120,8 +6488,10 @@ async function runSync(args, flags, io, confirm, cwd) {
       client,
       io: chatter,
       fix: boolFlag(flags, "--fix"),
+      content: boolFlag(flags, "--content"),
       getStats: (uid) => apiClient.get(`/v1/indexes/${encode(uid)}/stats`),
-      listRemoteIds: (uid, primaryKey) => listRemoteIds(apiClient, uid, primaryKey)
+      listRemoteIds: (uid, primaryKey) => listRemoteIds(apiClient, uid, primaryKey),
+      readRemoteDocuments: (uid, primaryKey) => readRemoteDocuments(apiClient, uid, primaryKey)
     });
     if (json) printJson(io, { indexes: rows, drift: reconcileHasDrift(rows) });
     if (reconcileHasDrift(rows)) process.exitCode = 1;
@@ -6143,7 +6513,7 @@ async function runSync(args, flags, io, confirm, cwd) {
   printCommandHelp("sync", io);
   throw new UsageError("Unknown sync command");
 }
-var CLI_VERSION = true ? "0.6.0" : "dev";
+var CLI_VERSION = true ? "0.6.1" : "dev";
 async function run(argv, context = { io: defaultIO }) {
   if (argv.length === 1 && (argv[0] === "--version" || argv[0] === "-v" || argv[0] === "version")) {
     context.io.stdout(CLI_VERSION);
@@ -6277,6 +6647,7 @@ function exitAfterFlush(code) {
   main,
   parseArgv,
   parseExpires,
+  parseIndexList,
   parsePermissions,
   run,
   stdinIsInteractive,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@headband/cli",
-  "version": "0.6.0",
+  "version": "0.6.1",
   "description": "CLI tool for setting up Headband search in any project",
   "bin": {
     "headband": "dist/index.js"