@alexify/migronaut 2.1.0 → 2.2.0

package/src/cli/commands/converge.js

@@ -1,11 +1,13 @@
 const { needsConfirmation } = require('../../core/converge-plan.js');
+const { searchBuildState } = require('../../core/search-index-spec.js');
 const { ConfigInvalidError, RunAbortedError } = require('../../errors/index.js');
 const { confirm, defineCommand, EXIT_CODES } = require('../shared.js');
 const { renderConvergeHistory, renderConvergeTable } = require('../table.js');
 
 /**
  * Every row the operator must confirm, across all collections: a dropped or
- * rebuilt index, or a validator change on a collection that holds data.
+ * rebuilt index, a dropped search index, or a validator change on a
+ * collection that holds data.
  */
 function actionsToConfirm(plan) {
   const found = [];
@@ -36,14 +38,18 @@ function assertNotStopped(stopRequested) {
 function registerConverge(program) {
   defineCommand(program, {
     name: 'converge',
-    description: 'Bring declared collections (indexes, validators) to their declared state',
+    description:
+      'Bring declared collections (indexes, search indexes, validators) to their declared state',
     options: [
       ['--dry-run', 'Show what would change without changing anything'],
       [
         '--check',
         `Exit with code ${EXIT_CODES.COLLECTIONS_DRIFT} if anything would change (CI gate; implies --dry-run)`,
       ],
-      ['--prune', 'Drop undeclared indexes (in collections whose definition does not decide)'],
+      [
+        '--prune',
+        'Drop undeclared indexes and search indexes (in collections whose definition does not decide)',
+      ],
       ['--ordered', 'Refuse while any migration is still pending'],
       ['--reason <text>', 'Why — recorded in the converge history (who: the OS user)'],
       ['--history', 'Show the converge history instead of converging (read-only)'],
@@ -52,6 +58,11 @@ function registerConverge(program) {
         '--rebuild-unique',
         'Allow rebuilding a unique index (drops the constraint until the new one is built)',
       ],
+      [
+        '--wait-search',
+        'Wait until every declared search index is queryable (overrides waitForSearchIndexes)',
+      ],
+      ['--no-wait-search', 'Do not wait for search indexes, whatever waitForSearchIndexes says'],
       [
         '-y, --yes',
         'Drop and rebuild indexes, and change validators, without asking (required with --json)',
@@ -64,6 +75,13 @@ function registerConverge(program) {
     // ask only when the plan drops or rebuilds an index, then apply — the way
     // `unlock` reads the lock before asking.
     run: async (migrator, opts, _positionals, { logger, json, spinner, stopRequested }) => {
+      const waitSearch = typeof opts.waitSearch === 'boolean' ? opts.waitSearch : undefined;
+      if (waitSearch !== undefined && (opts.history || opts.dryRun || opts.check)) {
+        throw new ConfigInvalidError(
+          `--${waitSearch ? '' : 'no-'}wait-search applies to a real converge — not to ` +
+            `${opts.history ? '--history' : opts.check ? '--check' : '--dry-run'}`,
+        );
+      }
       if (opts.history) {
         return migrator.convergeHistory(
           opts.limit !== undefined ? { limit: Number(opts.limit) } : {},
@@ -98,16 +116,13 @@ function registerConverge(program) {
           // applies without one.
           if (json) {
             throw new ConfigInvalidError(
-              `converge would drop or rebuild an index, or change a validator ` +
+              `converge would drop or rebuild an index, drop a search index, or change a validator ` +
                 `(${destructive.length} change(s)) — pass --yes to confirm in --json mode`,
               { destructive },
             );
           }
           logger.info(renderConvergeTable(plan));
-          const uniqueRebuilds = destructive.filter(
-            (action) => action.action === 'recreate' && opts.rebuildUnique,
-          );
-          if (uniqueRebuilds.length > 0) {
+          if (opts.rebuildUnique && destructive.some((action) => action.action === 'recreate')) {
             logger.warn(
               '⚠ --rebuild-unique: a rebuilt unique index enforces nothing until it is built ' +
                 'again — a duplicate written in between makes it unbuildable',
@@ -127,6 +142,7 @@ function registerConverge(program) {
           noLock: opts.noLock,
           ...prune,
           ...ordered,
+          ...(waitSearch !== undefined ? { waitForSearchIndexes: waitSearch } : {}),
           ...(opts.reason !== undefined ? { reason: opts.reason } : {}),
         });
       } finally {
@@ -147,9 +163,21 @@ function registerConverge(program) {
       logger.info(renderConvergeTable(result, { all: Boolean(opts.verbose) }));
     },
     after: (result, { logger, opts }) => {
-      if (!opts.check || result === undefined || result.inSync) return;
+      if (!opts.check || result === undefined) return;
+      // A search index that failed to build serves nothing, whatever its
+      // definition says — the gate fails on it too, though converge cannot fix it.
       // .error writes to stderr, so JSON stdout stays a single clean document.
-      logger.error('✖ The database differs from the declared collections');
+      let failed = 0;
+      for (const index of result.search?.notReady ?? []) {
+        if (searchBuildState(index) !== 'failed') continue;
+        failed += 1;
+        logger.error(
+          `✖ Search index ${index.collection} "${index.name}" failed to build` +
+            `${index.message ? `: ${index.message}` : ''}`,
+        );
+      }
+      if (result.inSync && failed === 0) return;
+      if (!result.inSync) logger.error('✖ The database differs from the declared collections');
       // A dedicated code: a CI gate must tell "out of step" (act: converge)
       // from "the check itself crashed" (act: page).
       process.exitCode = EXIT_CODES.COLLECTIONS_DRIFT;

package/src/cli/table.js

@@ -1,3 +1,5 @@
+const { TARGET_LABELS, isDestructive } = require('../core/converge-plan.js');
+const { searchBuildState } = require('../core/search-index-spec.js');
 const { createColors, stripAnsi } = require('../utils/colors.js');
 const { formatDateTime } = require('../utils/date.js');
 // Shared with the logger and spinner — cell values come from the changelog and
@@ -270,12 +272,43 @@ function convergeActionCell(colors, action) {
   }
 }
 
-/** The detail column: what differs, or why the row is what it is */
+/** Where the server is with a search index, when it is not simply serving it */
+function searchBuildDetail(build) {
+  if (build === undefined) return '';
+  switch (searchBuildState(build)) {
+    case 'serving':
+      return '';
+    case 'failed':
+      return `FAILED${build.message ? `: ${build.message}` : ''}`;
+    case 'updating':
+      return 'updating';
+    case 'stale':
+      return 'STALE — not replicating';
+    default:
+      return build.status;
+  }
+}
+
+/** Whether a search index row is one worth showing even when nothing changes: not serving yet */
+function searchNotServing(action) {
+  return action.target === 'searchIndex' && searchBuildDetail(action.build) !== '';
+}
+
+/** The detail column: what differs, or why the row is what it is — and a search index's build */
 function convergeDetail(action) {
   if (action.reason === 'name' && action.liveName !== undefined) {
     return `renamed from "${action.liveName}"`;
   }
-  return action.reason ?? '';
+  if (action.target !== 'searchIndex') return action.reason ?? '';
+  const parts = [];
+  const type = action.to?.type ?? action.from?.type;
+  if (type === 'vectorSearch' && (action.action === 'create' || action.action === 'modify')) {
+    parts.push('vectorSearch');
+  }
+  if (action.reason) parts.push(action.reason);
+  const build = searchBuildDetail(action.build);
+  if (build) parts.push(build);
+  return parts.join(' · ');
 }
 
 /**
@@ -286,22 +319,32 @@ function convergeDetail(action) {
 function renderConvergeTable(result, { all = false } = {}) {
   const colors = palette();
   const cells = [];
-  const counts = { change: 0, destructive: 0, conflict: 0, keep: 0, unchanged: 0, touched: 0 };
+  const counts = {
+    change: 0,
+    destructive: 0,
+    conflict: 0,
+    keep: 0,
+    searchKeep: 0,
+    skip: 0,
+    unchanged: 0,
+    touched: 0,
+  };
   for (const collection of result.collections) {
     let changes = 0;
     for (const action of collection.actions) {
       if (action.action === 'unchanged') counts.unchanged += 1;
+      else if (action.action === 'keep' && action.target === 'searchIndex') counts.searchKeep += 1;
       else if (action.action === 'keep') counts.keep += 1;
       else if (action.action === 'conflict') counts.conflict += 1;
+      else if (action.action === 'skip') counts.skip += 1;
       else changes += 1;
-      if (action.target === 'index' && (action.action === 'drop' || action.action === 'recreate')) {
-        counts.destructive += 1;
-      }
-      if (action.action === 'unchanged' && !all) continue;
+      if (isDestructive(action)) counts.destructive += 1;
+      if (action.action === 'unchanged' && !all && !searchNotServing(action)) continue;
+      const named = action.target === 'index' || action.target === 'searchIndex';
       cells.push([
         truncate(sanitize(collection.name), MAX_NAME_WIDTH),
-        action.target,
-        action.target === 'index' ? truncate(sanitize(action.name), MAX_NAME_WIDTH) : '',
+        TARGET_LABELS[action.target] ?? action.target,
+        named ? truncate(sanitize(action.name), MAX_NAME_WIDTH) : '',
         convergeActionCell(colors, action.action),
         truncate(sanitize(convergeDetail(action)), MAX_DETAIL_WIDTH),
       ]);
@@ -323,6 +366,22 @@ function renderConvergeTable(result, { all = false } = {}) {
   if (counts.destructive > 0) parts.push(`${counts.destructive} drop/rebuild`);
   if (counts.conflict > 0) parts.push(colors.red(`${counts.conflict} conflict(s)`));
   if (counts.keep > 0) parts.push(`${counts.keep} undeclared index(es) kept`);
+  if (counts.searchKeep > 0) parts.push(`${counts.searchKeep} undeclared search index(es) kept`);
+  if (counts.skip > 0) {
+    parts.push(colors.yellow(`${counts.skip} search index(es) skipped — Search unavailable`));
+  }
+  let building = 0;
+  let stale = 0;
+  let failed = 0;
+  for (const index of result.search?.notReady ?? []) {
+    const state = searchBuildState(index);
+    if (state === 'failed') failed += 1;
+    else if (state === 'stale') stale += 1;
+    else building += 1;
+  }
+  if (building > 0) parts.push(`${building} search index(es) building`);
+  if (stale > 0) parts.push(colors.yellow(`${stale} search index(es) stale`));
+  if (failed > 0) parts.push(colors.red(`${failed} search index(es) failed`));
   if (counts.unchanged > 0 && !all) parts.push(`${counts.unchanged} unchanged`);
   const line = parts.join(' · ');
   if (cells.length === 0) return line;

package/src/core/audit.js

@@ -1,5 +1,9 @@
+const { mapLimit } = require('../utils/concurrency.js');
 const { errorText } = require('../utils/error.js');
+const { SEARCH_UNAVAILABLE_HINT, listSearchIndexes, probeSearch } = require('./converge-search.js');
 const { toLockInfo } = require('./lock.js');
+const { normalizeLiveSearchIndex, searchBuildState } = require('./search-index-spec.js');
+const { READ_OPTIONS, readServer } = require('./server-info.js');
 
 /** Changelog indexes ensureIndexes() creates — audit warns when any is absent */
 const EXPECTED_INDEXES = [
@@ -12,14 +16,16 @@ const EXPECTED_INDEXES = [
 
 /**
  * Read-only health check of the setup: configuration, connectivity,
- * transaction support, indexes, lock state and checksum drift.
+ * transaction support, indexes, lock state, checksum drift — and Atlas Search,
+ * when declared collections hold search indexes.
  *
  * Fixes nothing — it reports, so an operator can see in one command why a
  * migration would fail before running one. Every check is independent: a
  * failure in one is recorded and the rest still run.
  *
  * Pure orchestration over capabilities the MigratorKit injects (`deps`), so it
- * needs none of the kit's private state.
+ * needs none of the kit's private state. `deps.definitions()` (optional) gives
+ * the declared collections, for the search check.
  */
 async function runAudit(deps) {
   const checks = [];
@@ -133,7 +139,10 @@ async function runAudit(deps) {
     record('checksums', 'warn', `Could not read status: ${errorText(error)}`);
   }
 
-  // 7. Runtime. TypeScript migrations need a runtime that can strip types.
+  // 7. Search — only where declared collections hold search indexes.
+  await auditSearch(deps, db, config, record);
+
+  // 8. Runtime. TypeScript migrations need a runtime that can strip types.
   // Feature detection instead of version parsing: it also catches a run
   // under `--no-experimental-strip-types` on an otherwise capable Node.
   const nodeVersion = process.versions.node;
@@ -152,6 +161,82 @@ async function runAudit(deps) {
   return auditReport(checks);
 }
 
+/** Search index lists the audit reads at once */
+const AUDIT_READ_CONCURRENCY = 8;
+
+/**
+ * Whether the server has the Atlas Search that declared search indexes need,
+ * and whether any of them failed to build. Records nothing when none is
+ * declared — or when the definitions do not load: converge reports that
+ * itself, with every issue.
+ */
+async function auditSearch(deps, db, config, record) {
+  if (typeof deps.definitions !== 'function') return;
+  let definitions;
+  try {
+    definitions = await deps.definitions();
+  } catch {
+    return;
+  }
+  // One pass: the collections that declare search indexes, the names each
+  // declares, and how many there are in all.
+  const declaring = [];
+  let declared = 0;
+  for (const definition of definitions) {
+    if (definition.searchIndexes === undefined) continue;
+    const names = new Set();
+    for (const index of definition.searchIndexes) names.add(index.name);
+    declaring.push({ definition, names });
+    declared += definition.searchIndexes.length;
+  }
+  if (declaring.length === 0) return;
+  const counted = `${declared} search index(es) declared in ${declaring.length} collection(s)`;
+  try {
+    const probe = await probeSearch(db, await readServer(db), {
+      collection: declaring[0].definition.name,
+      readOptions: READ_OPTIONS,
+    });
+    if (!probe.available) {
+      const skip = config.onSearchUnavailable === 'skip';
+      record(
+        'search',
+        skip ? 'warn' : 'fail',
+        `Not available on this server (${probe.reason}) — ${counted}; ` +
+          (skip
+            ? "converge skips them (onSearchUnavailable: 'skip')"
+            : `converge refuses them: ${SEARCH_UNAVAILABLE_HINT}`),
+      );
+      return;
+    }
+    const failed = [];
+    const stale = [];
+    // One list per collection, a few at a time; reported in declaration order.
+    const lists = await mapLimit(declaring, AUDIT_READ_CONCURRENCY, ({ definition }) =>
+      listSearchIndexes(db, definition.name, READ_OPTIONS),
+    );
+    for (const [position, { definition, names }] of declaring.entries()) {
+      for (const raw of lists[position]) {
+        const index = normalizeLiveSearchIndex(raw);
+        if (!names.has(index.name)) continue;
+        const state = searchBuildState(index);
+        const label = `${definition.name}.${index.name}${index.message ? ` (${index.message})` : ''}`;
+        if (state === 'failed') failed.push(label);
+        else if (state === 'stale') stale.push(label);
+      }
+    }
+    if (failed.length > 0 || stale.length > 0) {
+      const parts = [];
+      if (failed.length > 0) parts.push(`failed to build: ${failed.join(', ')}`);
+      if (stale.length > 0) parts.push(`stale (not replicating): ${stale.join(', ')}`);
+      record('search', 'warn', `Available — ${parts.join('; ')}`);
+    } else {
+      record('search', 'pass', `Available — ${counted}`);
+    }
+  } catch (error) {
+    record('search', 'warn', `Could not check Atlas Search: ${errorText(error)}`);
+  }
+}
+
 /** Roll individual checks up into the report shape (one pass, both counters) */
 function auditReport(checks) {
   let failed = 0;

package/src/core/collections.js

@@ -1,12 +1,13 @@
 const fs = require('node:fs/promises');
 const path = require('node:path');
 const { ConfigInvalidError } = require('../errors/index.js');
-const { isPlainObject, regExpIssue, toWire } = require('../utils/canonical.js');
+const { isPlainObject, toWire, unsendable } = require('../utils/canonical.js');
 const { isCollectionName } = require('../utils/collection-name.js');
 const { mapLimit } = require('../utils/concurrency.js');
 const { errorText } = require('../utils/error.js');
 const { importUserFile, tsLoadMessageOrNull } = require('../utils/loader.js');
 const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('./index-spec.js');
+const { normalizeDeclaredSearchIndex, searchIndexIssues } = require('./search-index-spec.js');
 
 /**
  * Declared collections: validating a definition, normalizing it for the
@@ -23,6 +24,7 @@ const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('
 const DEFINITION_KEYS = [
   'name',
   'indexes',
+  'searchIndexes',
   'validator',
   'validationLevel',
   'validationAction',
@@ -40,27 +42,6 @@ function join(base, key) {
   return base.endsWith(':') ? `${base} ${key}` : `${base}.${key}`;
 }
 
-/** What makes a validator unsendable — a function, a symbol, a cycle — or null */
-function unsendable(value, seen = new Set()) {
-  if (seen.size === 0) {
-    const issue = regExpIssue(value);
-    if (issue) return issue;
-  }
-  const type = typeof value;
-  if (type === 'function') return 'must not contain functions';
-  if (type === 'symbol') return 'must not contain symbols';
-  if (value === null || type !== 'object') return null;
-  if (seen.has(value)) return 'must not contain circular references';
-  seen.add(value);
-  const items = Array.isArray(value) ? value : isPlainObject(value) ? Object.values(value) : [];
-  for (const item of items) {
-    const reason = unsendable(item, seen);
-    if (reason) return reason;
-  }
-  seen.delete(value);
-  return null;
-}
-
 const isEmptyObject = (value) => isPlainObject(value) && Object.keys(value).length === 0;
 
 function indexListIssues(indexes, base, issues) {
@@ -99,6 +80,34 @@ function indexListIssues(indexes, base, issues) {
   }
 }
 
+function searchIndexListIssues(searchIndexes, base, issues) {
+  const listPath = join(base, 'searchIndexes');
+  if (!Array.isArray(searchIndexes)) {
+    issues.push({ path: listPath, message: 'must be an array of search index definitions' });
+    return;
+  }
+  const names = new Map();
+  for (const [position, index] of searchIndexes.entries()) {
+    const indexPath = `${listPath}[${position}]`;
+    const found = searchIndexIssues(index, indexPath);
+    issues.push(...found);
+    if (found.length > 0) continue;
+    const { name } = normalizeDeclaredSearchIndex(index);
+    if (names.has(name)) {
+      issues.push({
+        path: indexPath,
+        message:
+          `has the same name as searchIndexes[${names.get(name)}] ("${name}")` +
+          (index.name === undefined
+            ? ' — a search index declared without a name is "default"'
+            : ''),
+      });
+    } else {
+      names.set(name, position);
+    }
+  }
+}
+
 /**
  * Validate one collection definition, returning `{ path, message }` issues
  * (empty when valid). `path` prefixes every issue (`collections[2]`, or
@@ -137,13 +146,20 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
     report('name', `"${name}" is one of migronaut's own collections`);
   }
 
-  if (definition.indexes === undefined && definition.validator === undefined) {
+  if (
+    definition.indexes === undefined &&
+    definition.searchIndexes === undefined &&
+    definition.validator === undefined
+  ) {
     issues.push({
       path: self,
-      message: 'declares neither indexes nor a validator — nothing to manage',
+      message: 'declares no indexes, searchIndexes or validator — nothing to manage',
     });
   }
   if (definition.indexes !== undefined) indexListIssues(definition.indexes, base, issues);
+  if (definition.searchIndexes !== undefined) {
+    searchIndexListIssues(definition.searchIndexes, base, issues);
+  }
 
   const { validator } = definition;
   if (validator !== undefined && validator !== null) {
@@ -203,8 +219,9 @@ function collectionsIssues(list, { reserved = [] } = {}) {
 
 /**
  * A valid definition in the planner's shape: indexes normalized (effective
- * names, server-form keys, the spec to send), the validator cleaned for the
- * wire. `indexes`/`validator` stay `undefined` when not managed.
+ * names, server-form keys, the spec to send), search indexes with their
+ * default name and type, the validator cleaned for the wire.
+ * `indexes`/`searchIndexes`/`validator` stay `undefined` when not managed.
  */
 function normalizeDefinition(definition, { name, source } = {}) {
   const { validator } = definition;
@@ -212,6 +229,13 @@ function normalizeDefinition(definition, { name, source } = {}) {
     name: definition.name ?? name,
     source,
     indexes: definition.indexes?.map((index) => normalizeDeclaredIndex(index)),
+    ...(definition.searchIndexes !== undefined
+      ? {
+          searchIndexes: definition.searchIndexes.map((index) =>
+            normalizeDeclaredSearchIndex(index),
+          ),
+        }
+      : {}),
     validator: validator === undefined || validator === null ? validator : toWire(validator),
     ...(definition.validationLevel !== undefined
       ? { validationLevel: definition.validationLevel }

package/src/core/config.js

@@ -4,6 +4,9 @@ const { pathToFileURL } = require('node:url');
 const { ConfigInvalidError } = require('../errors/index.js');
 const { isCollectionName } = require('../utils/collection-name.js');
 const { TELEMETRY_KEYS, telemetryIssues } = require('../utils/telemetry.js');
+
+/** The keys a `telemetry` object may hold — a Set, read once per key */
+const TELEMETRY_KEY_SET = new Set(TELEMETRY_KEYS);
 const { applyEnvFile } = require('../utils/env.js');
 const { errorText } = require('../utils/error.js');
 const { resolveLogger } = require('../utils/logger.js');
@@ -32,6 +35,9 @@ const DEFAULT_CONFIG = {
   onOutOfOrder: 'warn',
   reloadMigrations: false,
   convergeAfterUp: false,
+  onSearchUnavailable: 'fail',
+  waitForSearchIndexes: false,
+  searchIndexWaitTimeoutMs: 600_000,
 };
 
 /** Candidate config file names, checked in priority order within the cwd */
@@ -150,6 +156,19 @@ const CONFIG_KEYS = [
     optional: true,
   },
   { path: 'convergeAfterUp', check: isBoolean, message: 'must be a boolean', optional: true },
+  {
+    path: 'onSearchUnavailable',
+    check: (value) => value === 'fail' || value === 'skip',
+    message: "must be 'fail' or 'skip'",
+    optional: true,
+  },
+  { path: 'waitForSearchIndexes', check: isBoolean, message: 'must be a boolean', optional: true },
+  {
+    path: 'searchIndexWaitTimeoutMs',
+    check: isPositiveInteger,
+    message: 'must be a positive integer',
+    optional: true,
+  },
 ];
 
 /**
@@ -331,6 +350,17 @@ const ENV_KEYS = [
   { env: 'MIGRONAUT_RELOAD_MIGRATIONS', path: 'reloadMigrations', parse: parseBoolean },
   { env: 'MIGRONAUT_COLLECTIONS_DIR', path: 'collectionsDir', parse: parseString },
   { env: 'MIGRONAUT_CONVERGE_AFTER_UP', path: 'convergeAfterUp', parse: parseBoolean },
+  {
+    env: 'MIGRONAUT_ON_SEARCH_UNAVAILABLE',
+    path: 'onSearchUnavailable',
+    parse: parseEnum(['fail', 'skip']),
+  },
+  { env: 'MIGRONAUT_WAIT_FOR_SEARCH_INDEXES', path: 'waitForSearchIndexes', parse: parseBoolean },
+  {
+    env: 'MIGRONAUT_SEARCH_INDEX_WAIT_TIMEOUT_MS',
+    path: 'searchIndexWaitTimeoutMs',
+    parse: parsePositiveInteger,
+  },
 ];
 
 /** Build a partial config from the MIGRONAUT_* environment variables */
@@ -523,7 +553,7 @@ async function loadConfig(options = {}) {
   // (`trace`, `metrics`) instead of a tracer and a meter turns it off silently.
   if (config.telemetry) {
     for (const key in config.telemetry) {
-      if (!TELEMETRY_KEYS.includes(key)) (unknown ??= []).push(`telemetry.${key}`);
+      if (!TELEMETRY_KEY_SET.has(key)) (unknown ??= []).push(`telemetry.${key}`);
     }
   }
   if (unknown) {