@alexify/migronaut 2.1.0 → 2.3.0

package/src/core/bson-peer.js

@@ -0,0 +1,23 @@
+/**
+ * The driver's BSON, from the `mongodb` peer — loaded on first use, never at
+ * startup, and from this one module only (pinned by a test), so nothing else
+ * under src/ grows its own dependency on the driver's internals.
+ *
+ * What it is for: measuring a document the way the server will (a step's
+ * checkpoint must stay small) and writing one as relaxed EJSON for a dry
+ * run's report — both things only BSON itself can do exactly.
+ */
+let bson;
+
+function loadBson() {
+  bson ??= require('mongodb').BSON;
+  return bson;
+}
+
+/** The size of `value` as a BSON document, in bytes */
+const bsonSize = (value) => loadBson().calculateObjectSize(value);
+
+/** `value` as relaxed EJSON — plain JSON a person can read, types tagged where JSON has none */
+const toRelaxedEjson = (value) => loadBson().EJSON.serialize(value, { relaxed: true });
+
+module.exports = { bsonSize, loadBson, toRelaxedEjson };

package/src/core/changelog.js

@@ -109,6 +109,26 @@ class Changelog {
     return failed;
   }
 
+  /**
+   * Which of `names` are applied from a history that predates migronaut — a
+   * `baseline` or an import from migrate-mongo: never run here, so a
+   * background migration among them has no state, and counts as done.
+   */
+  async getAdoptedNames(db, names) {
+    if (names.length === 0) return [];
+    const docs = await this.#coll(db)
+      .find({
+        status: 'applied',
+        origin: { $in: ['baseline', 'migrate-mongo'] },
+        name: { $in: names },
+      })
+      .project({ name: 1, _id: 0 })
+      .toArray();
+    const adopted = [];
+    for (const doc of docs) adopted.push(doc.name);
+    return adopted;
+  }
+
   /** Return a single record by migration name, or null */
   async getByName(db, name) {
     return this.#coll(db).findOne({ name });
@@ -292,6 +312,18 @@ class Changelog {
     await this.#coll(db).bulkWrite(ops, { ordered: false });
   }
 
+  /**
+   * Re-pin an applied record to the file now on disk — `background repin`,
+   * so the strict drift check agrees with what is actually running.
+   */
+  async setChecksum(db, name, checksum) {
+    const result = await this.#coll(db).updateOne(
+      { name, status: 'applied' },
+      { $set: { checksum } },
+    );
+    return result.matchedCount === 1;
+  }
+
   /**
    * Mark a migration as reverted. Sets `status='reverted'` and `revertedAt=now`.
    * Never deletes the record — preserves the full audit history.

package/src/core/collections.js

@@ -1,12 +1,20 @@
 const fs = require('node:fs/promises');
 const path = require('node:path');
 const { ConfigInvalidError } = require('../errors/index.js');
-const { isPlainObject, regExpIssue, toWire } = require('../utils/canonical.js');
+const { isPlainObject, toWire, unsendable } = require('../utils/canonical.js');
 const { isCollectionName } = require('../utils/collection-name.js');
 const { mapLimit } = require('../utils/concurrency.js');
 const { errorText } = require('../utils/error.js');
 const { importUserFile, tsLoadMessageOrNull } = require('../utils/loader.js');
 const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('./index-spec.js');
+const { normalizeDeclaredSearchIndex, searchIndexIssues } = require('./search-index-spec.js');
+const { resolveVersioning, versioningIssues } = require('../versioning/config.js');
+const {
+  isVersioningIndexKey,
+  mergeVersioningValidator,
+  validatorVersioningIssues,
+  versioningIndex,
+} = require('./versioning-spec.js');
 
 /**
  * Declared collections: validating a definition, normalizing it for the
@@ -23,10 +31,12 @@ const { indexIssues, normalizeDeclaredIndex, sameDeclaredSignature } = require('
 const DEFINITION_KEYS = [
   'name',
   'indexes',
+  'searchIndexes',
   'validator',
   'validationLevel',
   'validationAction',
   'prune',
+  'versioning',
 ];
 const DEFINITION_KEY_SET = new Set(DEFINITION_KEYS);
 const VALIDATION_LEVELS = ['off', 'strict', 'moderate'];
@@ -40,27 +50,6 @@ function join(base, key) {
   return base.endsWith(':') ? `${base} ${key}` : `${base}.${key}`;
 }
 
-/** What makes a validator unsendable — a function, a symbol, a cycle — or null */
-function unsendable(value, seen = new Set()) {
-  if (seen.size === 0) {
-    const issue = regExpIssue(value);
-    if (issue) return issue;
-  }
-  const type = typeof value;
-  if (type === 'function') return 'must not contain functions';
-  if (type === 'symbol') return 'must not contain symbols';
-  if (value === null || type !== 'object') return null;
-  if (seen.has(value)) return 'must not contain circular references';
-  seen.add(value);
-  const items = Array.isArray(value) ? value : isPlainObject(value) ? Object.values(value) : [];
-  for (const item of items) {
-    const reason = unsendable(item, seen);
-    if (reason) return reason;
-  }
-  seen.delete(value);
-  return null;
-}
-
 const isEmptyObject = (value) => isPlainObject(value) && Object.keys(value).length === 0;
 
 function indexListIssues(indexes, base, issues) {
@@ -99,6 +88,68 @@ function indexListIssues(indexes, base, issues) {
   }
 }
 
+function searchIndexListIssues(searchIndexes, base, issues) {
+  const listPath = join(base, 'searchIndexes');
+  if (!Array.isArray(searchIndexes)) {
+    issues.push({ path: listPath, message: 'must be an array of search index definitions' });
+    return;
+  }
+  const names = new Map();
+  for (const [position, index] of searchIndexes.entries()) {
+    const indexPath = `${listPath}[${position}]`;
+    const found = searchIndexIssues(index, indexPath);
+    issues.push(...found);
+    if (found.length > 0) continue;
+    const { name } = normalizeDeclaredSearchIndex(index);
+    if (names.has(name)) {
+      issues.push({
+        path: indexPath,
+        message:
+          `has the same name as searchIndexes[${names.get(name)}] ("${name}")` +
+          (index.name === undefined
+            ? ' — a search index declared without a name is "default"'
+            : ''),
+      });
+    } else {
+      names.set(name, position);
+    }
+  }
+}
+
+/**
+ * Validate the `versioning` key and how it fits the rest of the definition:
+ * a declared validator must leave the managed fields to it, and a declared
+ * index must not duplicate the version index. Returns whether the definition
+ * is versioned (valid or not), so the validator checks below know a
+ * validator is coming.
+ */
+function versioningDefinitionIssues(definition, base, issues) {
+  if (definition.versioning === undefined) return false;
+  const path = join(base, 'versioning');
+  const found = versioningIssues(definition.versioning, path);
+  issues.push(...found);
+  if (found.length > 0) return true;
+  const versioning = resolveVersioning(definition.versioning);
+  if (isPlainObject(definition.validator) || definition.validator === null) {
+    for (const message of validatorVersioningIssues(definition.validator, versioning)) {
+      issues.push({ path: join(base, 'validator'), message });
+    }
+  }
+  if (versioning.index && Array.isArray(definition.indexes)) {
+    for (const [position, index] of definition.indexes.entries()) {
+      if (isPlainObject(index) && isVersioningIndexKey(index.key, versioning)) {
+        issues.push({
+          path: `${join(base, 'indexes')}[${position}]`,
+          message:
+            'is the version index, which versioning declares itself — remove it, or set ' +
+            'versioning.index: false',
+        });
+      }
+    }
+  }
+  return true;
+}
+
 /**
  * Validate one collection definition, returning `{ path, message }` issues
  * (empty when valid). `path` prefixes every issue (`collections[2]`, or
@@ -137,13 +188,22 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
     report('name', `"${name}" is one of migronaut's own collections`);
   }
 
-  if (definition.indexes === undefined && definition.validator === undefined) {
+  if (
+    definition.indexes === undefined &&
+    definition.searchIndexes === undefined &&
+    definition.validator === undefined &&
+    definition.versioning === undefined
+  ) {
     issues.push({
       path: self,
-      message: 'declares neither indexes nor a validator — nothing to manage',
+      message: 'declares no indexes, searchIndexes, validator or versioning — nothing to manage',
     });
   }
   if (definition.indexes !== undefined) indexListIssues(definition.indexes, base, issues);
+  if (definition.searchIndexes !== undefined) {
+    searchIndexListIssues(definition.searchIndexes, base, issues);
+  }
+  const versioning = versioningDefinitionIssues(definition, base, issues);
 
   const { validator } = definition;
   if (validator !== undefined && validator !== null) {
@@ -154,7 +214,8 @@ function definitionIssues(definition, { path: base, reserved = [], fallbackName
       if (reason) report('validator', reason);
     }
   }
-  const hasValidator = isPlainObject(validator) && !isEmptyObject(validator);
+  // The versioning rules are a validator of their own.
+  const hasValidator = (isPlainObject(validator) && !isEmptyObject(validator)) || versioning;
   for (const [key, allowed] of [
     ['validationLevel', VALIDATION_LEVELS],
     ['validationAction', VALIDATION_ACTIONS],
@@ -203,23 +264,56 @@ function collectionsIssues(list, { reserved = [] } = {}) {
 
 /**
  * A valid definition in the planner's shape: indexes normalized (effective
- * names, server-form keys, the spec to send), the validator cleaned for the
- * wire. `indexes`/`validator` stay `undefined` when not managed.
+ * names, server-form keys, the spec to send), search indexes with their
+ * default name and type, the validator cleaned for the wire.
+ * `indexes`/`searchIndexes`/`validator` stay `undefined` when not managed.
+ * `versioning` is resolved and folded in: its rules merged into the
+ * validator, its index appended to `indexes` — with `indexesPartial` when it
+ * is the only index declared, so the others stay unmanaged.
  */
 function normalizeDefinition(definition, { name, source } = {}) {
   const { validator } = definition;
+  let wireValidator = validator === undefined || validator === null ? validator : toWire(validator);
+  let indexes = definition.indexes?.map((index) => normalizeDeclaredIndex(index));
+  let validationLevel = definition.validationLevel;
+  let versioning;
+  let indexesPartial = false;
+  if (definition.versioning !== undefined) {
+    // Folded into an ordinary validator and an ordinary index, so the planner
+    // needs to know nothing about versioning.
+    versioning = resolveVersioning(definition.versioning);
+    // A validator synthesized for versioning alone applies `moderate`: an
+    // update to a legacy document that predates the rules stays possible.
+    if (wireValidator === undefined || isEmptyObject(wireValidator)) validationLevel ??= 'moderate';
+    wireValidator = mergeVersioningValidator(wireValidator, versioning);
+    if (versioning.index) {
+      // Marked: on a sharded collection the planner swaps in the shard-key-prefixed form.
+      const index = { ...normalizeDeclaredIndex(versioningIndex(versioning)), versionIndex: true };
+      // Declaring the version index alone must not make every other index of
+      // the collection "undeclared" — and so a candidate for prune.
+      indexesPartial = indexes === undefined;
+      indexes = [...(indexes ?? []), index];
+    }
+  }
   return {
     name: definition.name ?? name,
     source,
-    indexes: definition.indexes?.map((index) => normalizeDeclaredIndex(index)),
-    validator: validator === undefined || validator === null ? validator : toWire(validator),
-    ...(definition.validationLevel !== undefined
-      ? { validationLevel: definition.validationLevel }
+    indexes,
+    ...(indexesPartial ? { indexesPartial } : {}),
+    ...(definition.searchIndexes !== undefined
+      ? {
+          searchIndexes: definition.searchIndexes.map((index) =>
+            normalizeDeclaredSearchIndex(index),
+          ),
+        }
       : {}),
+    validator: wireValidator,
+    ...(validationLevel !== undefined ? { validationLevel } : {}),
     ...(definition.validationAction !== undefined
       ? { validationAction: definition.validationAction }
       : {}),
     ...(definition.prune !== undefined ? { prune: definition.prune } : {}),
+    ...(versioning !== undefined ? { versioning } : {}),
   };
 }
 

package/src/core/config.js

@@ -4,6 +4,9 @@ const { pathToFileURL } = require('node:url');
 const { ConfigInvalidError } = require('../errors/index.js');
 const { isCollectionName } = require('../utils/collection-name.js');
 const { TELEMETRY_KEYS, telemetryIssues } = require('../utils/telemetry.js');
+
+/** The keys a `telemetry` object may hold — a Set, read once per key */
+const TELEMETRY_KEY_SET = new Set(TELEMETRY_KEYS);
 const { applyEnvFile } = require('../utils/env.js');
 const { errorText } = require('../utils/error.js');
 const { resolveLogger } = require('../utils/logger.js');
@@ -32,6 +35,14 @@ const DEFAULT_CONFIG = {
   onOutOfOrder: 'warn',
   reloadMigrations: false,
   convergeAfterUp: false,
+  onSearchUnavailable: 'fail',
+  waitForSearchIndexes: false,
+  searchIndexWaitTimeoutMs: 600_000,
+  backgroundCollection: '_migronaut_background',
+  backgroundInline: false,
+  backgroundOnDrift: 'reopen',
+  backgroundDrift: 'poll',
+  backgroundShardAware: 'auto',
 };
 
 /** Candidate config file names, checked in priority order within the cwd */
@@ -41,6 +52,26 @@ const isNonEmptyString = (value) => typeof value === 'string' && value.length >
 const isBoolean = (value) => typeof value === 'boolean';
 const isPositiveInteger = (value) => Number.isInteger(value) && value > 0;
 const isExtension = (value) => value === 'ts' || value === 'js';
+const oneOf = (values) => (value) => values.includes(value);
+const quoted = (values) => values.map((value) => `'${value}'`).join(', ');
+
+const ON_DRIFT = ['reopen', 'report'];
+const DRIFT_MODES = ['poll', 'stream', 'both'];
+const SHARD_AWARE = ['auto', 'off'];
+
+/**
+ * The collections a background migration keeps its state in: the state
+ * documents themselves, their partitions and the drift watcher's resume
+ * tokens — the last two named after the first, so one setting moves all
+ * three (and a test of "must differ" covers them together).
+ */
+function backgroundCollectionNames(backgroundCollection) {
+  return {
+    state: backgroundCollection,
+    partitions: `${backgroundCollection}_partitions`,
+    watch: `${backgroundCollection}_watch`,
+  };
+}
 
 function isStringList(value) {
   if (!Array.isArray(value) || value.length === 0) return false;
@@ -150,6 +181,44 @@ const CONFIG_KEYS = [
     optional: true,
   },
   { path: 'convergeAfterUp', check: isBoolean, message: 'must be a boolean', optional: true },
+  {
+    path: 'onSearchUnavailable',
+    check: (value) => value === 'fail' || value === 'skip',
+    message: "must be 'fail' or 'skip'",
+    optional: true,
+  },
+  { path: 'waitForSearchIndexes', check: isBoolean, message: 'must be a boolean', optional: true },
+  {
+    path: 'searchIndexWaitTimeoutMs',
+    check: isPositiveInteger,
+    message: 'must be a positive integer',
+    optional: true,
+  },
+  {
+    path: 'backgroundCollection',
+    check: isCollectionName,
+    message: "must be a valid collection name (no '$'/NUL, not system.*)",
+    optional: true,
+  },
+  { path: 'backgroundInline', check: isBoolean, message: 'must be a boolean', optional: true },
+  {
+    path: 'backgroundOnDrift',
+    check: oneOf(ON_DRIFT),
+    message: `must be ${quoted(ON_DRIFT)}`,
+    optional: true,
+  },
+  {
+    path: 'backgroundDrift',
+    check: oneOf(DRIFT_MODES),
+    message: `must be ${quoted(DRIFT_MODES)}`,
+    optional: true,
+  },
+  {
+    path: 'backgroundShardAware',
+    check: oneOf(SHARD_AWARE),
+    message: `must be ${quoted(SHARD_AWARE)}`,
+    optional: true,
+  },
 ];
 
 /**
@@ -201,26 +270,51 @@ function validateConfig(config, options = {}) {
   // pure data, so this costs nothing. Definition *files* are loaded only when
   // a converge runs: importing them here would make one broken file block
   // every command, an emergency `down` included.
-  // Three bookkeeping collections, three jobs: sharing one would mix records.
-  const bookkeeping = ['migrationsCollection', 'lockCollection', 'convergeLogCollection'];
-  for (const [position, key] of bookkeeping.entries()) {
-    for (const other of bookkeeping.slice(0, position)) {
-      if (config[key] !== undefined && config[key] === config[other]) {
-        issues.push({ path: key, message: `must differ from ${other}` });
-      }
+  // Every bookkeeping collection has one job: sharing one would mix records.
+  const bookkeeping = bookkeepingCollections(config);
+  for (let position = 1; position < bookkeeping.length; position++) {
+    const [key, name, what] = bookkeeping[position];
+    if (name === undefined) continue;
+    for (let earlier = 0; earlier < position; earlier++) {
+      const [otherKey, otherName, otherWhat] = bookkeeping[earlier];
+      if (name !== otherName) continue;
+      issues.push({
+        path: key,
+        message: `${what ? `${what} (${name}) ` : ''}must differ from ${otherWhat ? `${otherKey}'s ${otherWhat}` : otherKey}`,
+      });
+      break;
     }
   }
   if (Array.isArray(config.collections)) {
-    const reserved = [
-      config.migrationsCollection,
-      config.lockCollection,
-      config.convergeLogCollection,
-    ];
+    const reserved = [];
+    for (const [, name] of bookkeeping) if (name !== undefined) reserved.push(name);
     for (const issue of collectionsIssues(config.collections, { reserved })) issues.push(issue);
   }
   return issues;
 }
 
+/**
+ * Every collection migronaut keeps records in, as `[configKey, name, what?]`
+ * — `what` names a collection derived from the key (the background
+ * partitions and watch collections).
+ */
+function bookkeepingCollections(config) {
+  const entries = [
+    ['migrationsCollection', config.migrationsCollection],
+    ['lockCollection', config.lockCollection],
+    ['convergeLogCollection', config.convergeLogCollection],
+  ];
+  if (typeof config.backgroundCollection === 'string') {
+    const names = backgroundCollectionNames(config.backgroundCollection);
+    entries.push(
+      ['backgroundCollection', names.state],
+      ['backgroundCollection', names.partitions, 'partitions collection'],
+      ['backgroundCollection', names.watch, 'watch collection'],
+    );
+  }
+  return entries;
+}
+
 const TRUE_BOOLEAN_STRINGS = new Set(['true', '1', 'yes']);
 const FALSE_BOOLEAN_STRINGS = new Set(['false', '0', 'no']);
 
@@ -331,6 +425,30 @@ const ENV_KEYS = [
   { env: 'MIGRONAUT_RELOAD_MIGRATIONS', path: 'reloadMigrations', parse: parseBoolean },
   { env: 'MIGRONAUT_COLLECTIONS_DIR', path: 'collectionsDir', parse: parseString },
   { env: 'MIGRONAUT_CONVERGE_AFTER_UP', path: 'convergeAfterUp', parse: parseBoolean },
+  {
+    env: 'MIGRONAUT_ON_SEARCH_UNAVAILABLE',
+    path: 'onSearchUnavailable',
+    parse: parseEnum(['fail', 'skip']),
+  },
+  { env: 'MIGRONAUT_WAIT_FOR_SEARCH_INDEXES', path: 'waitForSearchIndexes', parse: parseBoolean },
+  {
+    env: 'MIGRONAUT_SEARCH_INDEX_WAIT_TIMEOUT_MS',
+    path: 'searchIndexWaitTimeoutMs',
+    parse: parsePositiveInteger,
+  },
+  { env: 'MIGRONAUT_BACKGROUND_COLLECTION', path: 'backgroundCollection', parse: parseString },
+  { env: 'MIGRONAUT_BACKGROUND_INLINE', path: 'backgroundInline', parse: parseBoolean },
+  {
+    env: 'MIGRONAUT_BACKGROUND_ON_DRIFT',
+    path: 'backgroundOnDrift',
+    parse: parseEnum(ON_DRIFT),
+  },
+  { env: 'MIGRONAUT_BACKGROUND_DRIFT', path: 'backgroundDrift', parse: parseEnum(DRIFT_MODES) },
+  {
+    env: 'MIGRONAUT_BACKGROUND_SHARD_AWARE',
+    path: 'backgroundShardAware',
+    parse: parseEnum(SHARD_AWARE),
+  },
 ];
 
 /** Build a partial config from the MIGRONAUT_* environment variables */
@@ -523,7 +641,7 @@ async function loadConfig(options = {}) {
   // (`trace`, `metrics`) instead of a tracer and a meter turns it off silently.
   if (config.telemetry) {
     for (const key in config.telemetry) {
-      if (!TELEMETRY_KEYS.includes(key)) (unknown ??= []).push(`telemetry.${key}`);
+      if (!TELEMETRY_KEY_SET.has(key)) (unknown ??= []).push(`telemetry.${key}`);
     }
   }
   if (unknown) {
@@ -569,6 +687,8 @@ module.exports = {
   CONFIG_KEYS,
   DEFAULT_CONFIG,
   ENV_KEYS,
+  backgroundCollectionNames,
+  bookkeepingCollections,
   // Re-exported from utils/collection-name.js, where it moved so that
   // core/collections.js can use it without a require cycle through here.
   isCollectionName,