@alexify/migronaut 2.1.0 → 2.3.0

package/src/cli/table.js

@@ -1,3 +1,5 @@
+const { TARGET_LABELS, isDestructive } = require('../core/converge-plan.js');
+const { searchBuildState } = require('../core/search-index-spec.js');
 const { createColors, stripAnsi } = require('../utils/colors.js');
 const { formatDateTime } = require('../utils/date.js');
 // Shared with the logger and spinner — cell values come from the changelog and
@@ -270,12 +272,43 @@ function convergeActionCell(colors, action) {
   }
 }
 
-/** The detail column: what differs, or why the row is what it is */
+/** Where the server is with a search index, when it is not simply serving it */
+function searchBuildDetail(build) {
+  if (build === undefined) return '';
+  switch (searchBuildState(build)) {
+    case 'serving':
+      return '';
+    case 'failed':
+      return `FAILED${build.message ? `: ${build.message}` : ''}`;
+    case 'updating':
+      return 'updating';
+    case 'stale':
+      return 'STALE — not replicating';
+    default:
+      return build.status;
+  }
+}
+
+/** Whether a search index row is one worth showing even when nothing changes: not serving yet */
+function searchNotServing(action) {
+  return action.target === 'searchIndex' && searchBuildDetail(action.build) !== '';
+}
+
+/** The detail column: what differs, or why the row is what it is — and a search index's build */
 function convergeDetail(action) {
   if (action.reason === 'name' && action.liveName !== undefined) {
     return `renamed from "${action.liveName}"`;
   }
-  return action.reason ?? '';
+  if (action.target !== 'searchIndex') return action.reason ?? '';
+  const parts = [];
+  const type = action.to?.type ?? action.from?.type;
+  if (type === 'vectorSearch' && (action.action === 'create' || action.action === 'modify')) {
+    parts.push('vectorSearch');
+  }
+  if (action.reason) parts.push(action.reason);
+  const build = searchBuildDetail(action.build);
+  if (build) parts.push(build);
+  return parts.join(' · ');
 }
 
 /**
@@ -286,22 +319,32 @@ function convergeDetail(action) {
 function renderConvergeTable(result, { all = false } = {}) {
   const colors = palette();
   const cells = [];
-  const counts = { change: 0, destructive: 0, conflict: 0, keep: 0, unchanged: 0, touched: 0 };
+  const counts = {
+    change: 0,
+    destructive: 0,
+    conflict: 0,
+    keep: 0,
+    searchKeep: 0,
+    skip: 0,
+    unchanged: 0,
+    touched: 0,
+  };
   for (const collection of result.collections) {
     let changes = 0;
     for (const action of collection.actions) {
       if (action.action === 'unchanged') counts.unchanged += 1;
+      else if (action.action === 'keep' && action.target === 'searchIndex') counts.searchKeep += 1;
       else if (action.action === 'keep') counts.keep += 1;
       else if (action.action === 'conflict') counts.conflict += 1;
+      else if (action.action === 'skip') counts.skip += 1;
       else changes += 1;
-      if (action.target === 'index' && (action.action === 'drop' || action.action === 'recreate')) {
-        counts.destructive += 1;
-      }
-      if (action.action === 'unchanged' && !all) continue;
+      if (isDestructive(action)) counts.destructive += 1;
+      if (action.action === 'unchanged' && !all && !searchNotServing(action)) continue;
+      const named = action.target === 'index' || action.target === 'searchIndex';
       cells.push([
         truncate(sanitize(collection.name), MAX_NAME_WIDTH),
-        action.target,
-        action.target === 'index' ? truncate(sanitize(action.name), MAX_NAME_WIDTH) : '',
+        TARGET_LABELS[action.target] ?? action.target,
+        named ? truncate(sanitize(action.name), MAX_NAME_WIDTH) : '',
         convergeActionCell(colors, action.action),
         truncate(sanitize(convergeDetail(action)), MAX_DETAIL_WIDTH),
       ]);
@@ -323,6 +366,22 @@ function renderConvergeTable(result, { all = false } = {}) {
   if (counts.destructive > 0) parts.push(`${counts.destructive} drop/rebuild`);
   if (counts.conflict > 0) parts.push(colors.red(`${counts.conflict} conflict(s)`));
   if (counts.keep > 0) parts.push(`${counts.keep} undeclared index(es) kept`);
+  if (counts.searchKeep > 0) parts.push(`${counts.searchKeep} undeclared search index(es) kept`);
+  if (counts.skip > 0) {
+    parts.push(colors.yellow(`${counts.skip} search index(es) skipped — Search unavailable`));
+  }
+  let building = 0;
+  let stale = 0;
+  let failed = 0;
+  for (const index of result.search?.notReady ?? []) {
+    const state = searchBuildState(index);
+    if (state === 'failed') failed += 1;
+    else if (state === 'stale') stale += 1;
+    else building += 1;
+  }
+  if (building > 0) parts.push(`${building} search index(es) building`);
+  if (stale > 0) parts.push(colors.yellow(`${stale} search index(es) stale`));
+  if (failed > 0) parts.push(colors.red(`${failed} search index(es) failed`));
   if (counts.unchanged > 0 && !all) parts.push(`${counts.unchanged} unchanged`);
   const line = parts.join(' · ');
   if (cells.length === 0) return line;

package/src/core/audit.js

@@ -1,5 +1,9 @@
+const { mapLimit } = require('../utils/concurrency.js');
 const { errorText } = require('../utils/error.js');
+const { SEARCH_UNAVAILABLE_HINT, listSearchIndexes, probeSearch } = require('./converge-search.js');
 const { toLockInfo } = require('./lock.js');
+const { normalizeLiveSearchIndex, searchBuildState } = require('./search-index-spec.js');
+const { READ_OPTIONS, readServer } = require('./server-info.js');
 
 /** Changelog indexes ensureIndexes() creates — audit warns when any is absent */
 const EXPECTED_INDEXES = [
@@ -12,14 +16,16 @@ const EXPECTED_INDEXES = [
 
 /**
  * Read-only health check of the setup: configuration, connectivity,
- * transaction support, indexes, lock state and checksum drift.
+ * transaction support, indexes, lock state, checksum drift — and Atlas Search,
+ * when declared collections hold search indexes.
  *
  * Fixes nothing — it reports, so an operator can see in one command why a
  * migration would fail before running one. Every check is independent: a
  * failure in one is recorded and the rest still run.
  *
  * Pure orchestration over capabilities the MigratorKit injects (`deps`), so it
- * needs none of the kit's private state.
+ * needs none of the kit's private state. `deps.definitions()` (optional) gives
+ * the declared collections, for the search check.
  */
 async function runAudit(deps) {
   const checks = [];
@@ -133,7 +139,20 @@ async function runAudit(deps) {
     record('checksums', 'warn', `Could not read status: ${errorText(error)}`);
   }
 
-  // 7. Runtime. TypeScript migrations need a runtime that can strip types.
+  // 7. Search — only where declared collections hold search indexes.
+  await auditSearch(deps, db, config, record);
+
+  // 8. Background migrations — only where any is registered.
+  if (typeof deps.background === 'function') {
+    try {
+      const finding = await deps.background();
+      if (finding !== null) record('background', finding.status, finding.detail);
+    } catch (error) {
+      record('background', 'warn', `Could not check background migrations: ${errorText(error)}`);
+    }
+  }
+
+  // 9. Runtime. TypeScript migrations need a runtime that can strip types.
   // Feature detection instead of version parsing: it also catches a run
   // under `--no-experimental-strip-types` on an otherwise capable Node.
   const nodeVersion = process.versions.node;
@@ -152,6 +171,82 @@ async function runAudit(deps) {
   return auditReport(checks);
 }
 
+/** Search index lists the audit reads at once */
+const AUDIT_READ_CONCURRENCY = 8;
+
+/**
+ * Whether the server has the Atlas Search that declared search indexes need,
+ * and whether any of them failed to build. Records nothing when none is
+ * declared — or when the definitions do not load: converge reports that
+ * itself, with every issue.
+ */
+async function auditSearch(deps, db, config, record) {
+  if (typeof deps.definitions !== 'function') return;
+  let definitions;
+  try {
+    definitions = await deps.definitions();
+  } catch {
+    return;
+  }
+  // One pass: the collections that declare search indexes, the names each
+  // declares, and how many there are in all.
+  const declaring = [];
+  let declared = 0;
+  for (const definition of definitions) {
+    if (definition.searchIndexes === undefined) continue;
+    const names = new Set();
+    for (const index of definition.searchIndexes) names.add(index.name);
+    declaring.push({ definition, names });
+    declared += definition.searchIndexes.length;
+  }
+  if (declaring.length === 0) return;
+  const counted = `${declared} search index(es) declared in ${declaring.length} collection(s)`;
+  try {
+    const probe = await probeSearch(db, await readServer(db), {
+      collection: declaring[0].definition.name,
+      readOptions: READ_OPTIONS,
+    });
+    if (!probe.available) {
+      const skip = config.onSearchUnavailable === 'skip';
+      record(
+        'search',
+        skip ? 'warn' : 'fail',
+        `Not available on this server (${probe.reason}) — ${counted}; ` +
+          (skip
+            ? "converge skips them (onSearchUnavailable: 'skip')"
+            : `converge refuses them: ${SEARCH_UNAVAILABLE_HINT}`),
+      );
+      return;
+    }
+    const failed = [];
+    const stale = [];
+    // One list per collection, a few at a time; reported in declaration order.
+    const lists = await mapLimit(declaring, AUDIT_READ_CONCURRENCY, ({ definition }) =>
+      listSearchIndexes(db, definition.name, READ_OPTIONS),
+    );
+    for (const [position, { definition, names }] of declaring.entries()) {
+      for (const raw of lists[position]) {
+        const index = normalizeLiveSearchIndex(raw);
+        if (!names.has(index.name)) continue;
+        const state = searchBuildState(index);
+        const label = `${definition.name}.${index.name}${index.message ? ` (${index.message})` : ''}`;
+        if (state === 'failed') failed.push(label);
+        else if (state === 'stale') stale.push(label);
+      }
+    }
+    if (failed.length > 0 || stale.length > 0) {
+      const parts = [];
+      if (failed.length > 0) parts.push(`failed to build: ${failed.join(', ')}`);
+      if (stale.length > 0) parts.push(`stale (not replicating): ${stale.join(', ')}`);
+      record('search', 'warn', `Available — ${parts.join('; ')}`);
+    } else {
+      record('search', 'pass', `Available — ${counted}`);
+    }
+  } catch (error) {
+    record('search', 'warn', `Could not check Atlas Search: ${errorText(error)}`);
+  }
+}
+
 /** Roll individual checks up into the report shape (one pass, both counters) */
 function auditReport(checks) {
   let failed = 0;

package/src/core/background-audit.js

@@ -0,0 +1,139 @@
+const { STATE_SUMMARY, probeHint } = require('./background.js');
+const { probeOldShape } = require('./background-drift.js');
+
+/**
+ * What `audit` says about background migrations — read-only, from what the
+ * kit injects (`deps`): the states, their partitions and leases, the files
+ * on disk, the changelog's background records and the live watchers.
+ */
+
+/** How long a background migration may sit in `pending`, or `running` without progress, before audit warns */
+const STALL_MS = 15 * 60_000;
+
+/**
+ * What `audit` says about background migrations: `{ status, detail }` with
+ * the worst finding — or `null` when none is registered. `deps.checksumOf`
+ * reads the file on disk; `deps.backgroundRecords` the changelog's
+ * background records.
+ */
+async function auditFindings(deps, { now = Date.now() } = {}) {
+  const states = await deps.store.list({}, { projection: STATE_SUMMARY });
+  const records = await deps.backgroundRecords();
+  if (states.length === 0 && records.length === 0) return null;
+  const failures = [];
+  const warnings = [];
+  const byName = new Map();
+  const running = [];
+  for (const state of states) {
+    byName.set(state._id, state);
+    if (state.status === 'running') running.push(state._id);
+  }
+  // Read once for all of them, not once per state.
+  const live = await deps.store.liveLeasesOf(running);
+  const hints = new Map();
+  const hintOf = async (spec) => {
+    const key = `${spec.collection}\u0000${spec.field}`;
+    if (!hints.has(key)) hints.set(key, await probeHint(deps, spec));
+    return hints.get(key);
+  };
+  const counts = {};
+  for (const state of states) {
+    const name = state._id;
+    counts[state.status] = (counts[state.status] ?? 0) + 1;
+    if (state.status === 'failed') {
+      failures.push(`${name} failed${state.lastError ? ` (${state.lastError})` : ''}`);
+      continue;
+    }
+    const registeredAt = new Date(state.registeredAt).getTime();
+    const progressAt = new Date(
+      state.lastProgressAt ?? state.startedAt ?? state.registeredAt,
+    ).getTime();
+    if (state.status === 'pending' && now - registeredAt > STALL_MS) {
+      warnings.push(
+        `${name} has been pending since ${new Date(registeredAt).toISOString()} — is a runner up?`,
+      );
+    }
+    if (state.status === 'paused') warnings.push(`${name} is paused`);
+    if (state.status === 'running') {
+      if ((live.get(name) ?? 0) === 0 && now - progressAt > STALL_MS) {
+        warnings.push(
+          `${name} is running but stalled — no lane for ${Math.round((now - progressAt) / 60_000)} min`,
+        );
+      }
+    }
+    if (state.status === 'blocked') {
+      for (const required of state.waitsFor ?? []) {
+        if (byName.get(required)?.status === 'failed') {
+          warnings.push(`${name} is blocked by ${required}, which failed`);
+        }
+      }
+    }
+    if (state.plan !== undefined) {
+      const current = await deps.store.partitionCounts(name, {
+        generation: state.generation,
+        plan: state.plan.token,
+      });
+      if (current.failed > 0 && state.status !== 'failed') {
+        warnings.push(`${name} has ${current.failed} failed partition(s)`);
+      }
+      // Of the current generation only: the done partitions of the pass before
+      // are kept on purpose (finalize drops the generation before last).
+      const orphaned = await deps.store.countForeignPlans(name, {
+        generation: state.generation,
+        plan: state.plan.token,
+      });
+      if (orphaned > 0) warnings.push(`${name} keeps ${orphaned} partition(s) of an old plan`);
+    }
+    if (state.status === 'completed') {
+      if ((state.badIds ?? []).length > 0) {
+        warnings.push(
+          `${name} completed with ${state.badIds.length} document(s) it could not migrate`,
+        );
+      }
+      if (state.spec?.mode === 'declarative' && state.direction !== 'revert') {
+        const hint = await hintOf(state.spec);
+        if (
+          hint !== undefined &&
+          (await probeOldShape(deps, state, hint).catch(() => null)) !== null
+        ) {
+          warnings.push(
+            `${state.spec.collection} holds old-shape documents again (${name} completed)`,
+          );
+        }
+      }
+    }
+    try {
+      const checksum = await deps.checksumOf(name);
+      if (state.checksum !== undefined && checksum !== state.checksum) {
+        warnings.push(`${name} changed on disk since it was registered (repin it)`);
+      }
+    } catch {
+      warnings.push(`${name} is registered but its file is missing`);
+    }
+  }
+  for (const record of records) {
+    if (!byName.has(record.name)) {
+      warnings.push(`${record.name} is applied but its background migration is not registered`);
+    }
+  }
+  // Live drift watchers, when drift is streamed: one left to the poll for
+  // long, or one nobody has led for long, is worth a look.
+  for (const row of (await deps.watchRows?.()) ?? []) {
+    const quietMs = now - new Date(row.updatedAt).getTime();
+    if (!(quietMs > STALL_MS)) continue;
+    const minutes = Math.round(quietMs / 60_000);
+    warnings.push(
+      row.state === 'fallback'
+        ? `the drift watcher of ${row._id} has fallen back to polling for ${minutes} min`
+        : `the drift watcher of ${row._id} has had no live leader for ${minutes} min`,
+    );
+  }
+  const summary = Object.entries(counts)
+    .map(([status, n]) => `${n} ${status}`)
+    .join(', ');
+  if (failures.length > 0) return { status: 'fail', detail: [...failures, ...warnings].join('; ') };
+  if (warnings.length > 0) return { status: 'warn', detail: warnings.join('; ') };
+  return { status: 'pass', detail: `${states.length} background migration(s): ${summary}` };
+}
+
+module.exports = { auditFindings };

package/src/core/background-drift.js

@@ -0,0 +1,126 @@
+const { errorText } = require('../utils/error.js');
+const { STATE_SUMMARY, control, probeHint } = require('./background.js');
+const { excludeBadIds, matchOf } = require('./background-engine.js');
+const { READ_OPTIONS } = require('./server-info.js');
+
+/**
+ * Drift: old-shape documents that appear after a background migration
+ * completed — an old pod, a forgotten worker, another service. The probes
+ * the requires guard, the poll (`verifyBackground`) and audit share: one
+ * hinted, time-boxed look per completed forward migration. Orchestration
+ * over what the kit injects (`deps`), as in background.js.
+ */
+
+/** How long one drift probe may run */
+const DRIFT_PROBE_MS = 5_000;
+
+/** Statuses still at work on a collection — its drift is not drift yet */
+const ACTIVE = new Set(['blocked', 'pending', 'running', 'paused']);
+
+/** One indexed look for a document of a completed forward migration's old shape */
+async function probeOldShape(deps, state, hint) {
+  const spec = state.spec;
+  return deps.db
+    .collection(spec.collection)
+    .findOne(excludeBadIds(matchOf(spec, 'forward'), state.badIds), {
+      projection: { _id: 1 },
+      hint,
+      maxTimeMS: DRIFT_PROBE_MS,
+      ...READ_OPTIONS,
+    });
+}
+
+/**
+ * Whether a completed forward migration's collection holds an old-shape
+ * document again — for the `requires` guard, which runs under the migration
+ * lock: one hinted probe, time-boxed. Where that is not possible (a step
+ * migration, no version index, a probe that ran out of time) the status is
+ * trusted — a scan of the whole collection on every `up` is not an option.
+ */
+async function stillDirty(deps, state) {
+  const spec = state.spec;
+  if (!spec || spec.mode !== 'declarative' || state.direction === 'revert') return false;
+  const hint = await probeHint(deps, spec);
+  if (hint === undefined) return false;
+  try {
+    return (await probeOldShape(deps, state, hint)) !== null;
+  } catch (error) {
+    deps.logger.warn(
+      `⚠ Could not check ${state._id} for old-shape documents: ${errorText(error)} — trusting its status`,
+      deps.fields({ background: state._id }),
+    );
+    return false;
+  }
+}
+
+/**
+ * The drift watch: old-shape documents that appeared after a background
+ * migration completed — an old pod, a forgotten worker, another service.
+ * One indexed probe per completed forward (declarative) migration; skipped
+ * where another one is still at work on the collection, where the validator
+ * already refuses the old shape (`to ≤ versioning.min`), and where the
+ * version index is missing. With `onDrift: 'reopen'` a finding reopens it —
+ * a new pass over what is left, not a reset — and with `'report'` it is only
+ * said. Chains (v1→v2→v3) converge on their own. No document id is reported.
+ * `streaming`: collections a live watcher leads right now — skipped too.
+ */
+async function verify(deps, { onDrift = 'reopen', collections, streaming } = {}) {
+  const states = await deps.store.list({}, { projection: STATE_SUMMARY });
+  const active = new Set();
+  for (const state of states) {
+    if (ACTIVE.has(state.status) && state.spec?.collection) active.add(state.spec.collection);
+  }
+  // In `backgroundDrift: 'stream'` mode a live watcher's collection is its, not the poll's.
+  for (const collection of streaming ?? []) active.add(collection);
+  const wanted = collections === undefined ? undefined : new Set(collections);
+  const result = { checked: 0, skipped: 0, drift: [] };
+  for (const state of states) {
+    const spec = state.spec;
+    if (state.status !== 'completed' || state.direction === 'revert') continue;
+    if (spec?.mode !== 'declarative') continue;
+    if (wanted !== undefined && !wanted.has(spec.collection)) continue;
+    const versioning = await deps.versioningOf?.(spec.collection);
+    if (active.has(spec.collection) || (versioning && spec.to <= versioning.min)) {
+      result.skipped += 1;
+      continue;
+    }
+    const hint = await probeHint(deps, spec);
+    if (hint === undefined) {
+      result.skipped += 1;
+      continue;
+    }
+    let found;
+    try {
+      found = await probeOldShape(deps, state, hint);
+    } catch (error) {
+      deps.logger.warn(
+        `⚠ Drift check of ${state._id} failed: ${errorText(error)}`,
+        deps.fields({ background: state._id }),
+      );
+      result.skipped += 1;
+      continue;
+    }
+    result.checked += 1;
+    if (found === null) continue;
+    const action = onDrift === 'reopen' ? 'reopened' : 'reported';
+    if (action === 'reopened') {
+      await control(deps, state._id, 'retry', { reason: 'old-shape documents reappeared' });
+    }
+    deps.telemetry?.backgroundDrift({ name: state._id });
+    deps.emit('background:drift', {
+      migration: state._id,
+      collection: spec.collection,
+      source: 'poll',
+      action,
+    });
+    deps.logger.warn(
+      `⚠ ${spec.collection}: old-shape documents appeared after ${state._id} completed — ` +
+        (action === 'reopened' ? 'reopened it' : 'an old release may still be writing'),
+      deps.fields({ background: state._id, collection: spec.collection, action }),
+    );
+    result.drift.push({ migration: state._id, collection: spec.collection, action });
+  }
+  return result;
+}
+
+module.exports = { probeOldShape, stillDirty, verify };