@alexify/migronaut 2.1.0 → 2.3.0

package/src/utils/migration-name.js

@@ -29,4 +29,36 @@ function assertMigrationName(name, context = {}) {
   }
 }
 
-module.exports = { assertMigrationName, isBareFilename };
+/**
+ * Why a file's `requires` export is not valid: an array of bare migration
+ * file names, no duplicates, each sorting strictly before the file itself
+ * (`name`). Files run in name order, so an edge that only ever points
+ * backwards can never close a cycle — the whole "is it a DAG?" question,
+ * answered by the name. Returns `{ path, message }` issues.
+ */
+function requiresIssues(requires, name) {
+  if (requires === undefined) return [];
+  if (!Array.isArray(requires)) {
+    return [{ path: 'requires', message: 'must be an array of migration file names' }];
+  }
+  const issues = [];
+  const seen = new Set();
+  for (const [position, required] of requires.entries()) {
+    const path = `requires[${position}]`;
+    if (!isBareFilename(required)) {
+      issues.push({ path, message: 'must be a bare migration file name' });
+    } else if (seen.has(required)) {
+      issues.push({ path, message: `names "${required}" twice` });
+    } else if (name !== undefined && required >= name) {
+      issues.push({
+        path,
+        message: `must name an earlier migration ("${required}" does not sort before "${name}")`,
+      });
+    } else {
+      seen.add(required);
+    }
+  }
+  return issues;
+}
+
+module.exports = { assertMigrationName, isBareFilename, requiresIssues };

package/src/utils/telemetry.js

@@ -13,6 +13,8 @@ const SPAN_STATUS_ERROR = 2;
 const SPANS = {
   RUN: 'migronaut.run',
   MIGRATION: 'migronaut.migration',
+  BACKGROUND_SLICE: 'migronaut.background.slice',
+  BACKGROUND_COORDINATE: 'migronaut.background.coordinate',
 };
 
 /** Every attribute key migronaut sets, on spans and on metric points */
@@ -28,12 +30,18 @@ const ATTRIBUTES = {
   LOCK_SKIPPED: 'migronaut.lock.skipped',
   LOCK_LOST_REASON: 'migronaut.lock.lost_reason',
   LOCK_WAIT_OUTCOME: 'migronaut.lock.wait.outcome',
+  SEARCH_WAIT_OUTCOME: 'migronaut.converge.search.wait.outcome',
   MIGRATION_NAME: 'migronaut.migration.name',
   MIGRATION_DIRECTION: 'migronaut.migration.direction',
   MIGRATION_BATCH: 'migronaut.migration.batch',
   MIGRATION_INDEX: 'migronaut.migration.index',
   MIGRATION_TOTAL: 'migronaut.migration.total',
   MIGRATION_TRANSACTION: 'migronaut.migration.transaction',
+  BACKGROUND_NAME: 'migronaut.background.name',
+  BACKGROUND_OUTCOME: 'migronaut.background.outcome',
+  BACKGROUND_RESULT: 'migronaut.background.result',
+  BACKGROUND_REASON: 'migronaut.background.reason',
+  BACKGROUND_SHARD: 'migronaut.background.shard',
   ERROR_TYPE: 'error.type',
   /** The database a run is against — OpenTelemetry's database semantic convention */
   DB_NAMESPACE: 'db.namespace',
@@ -46,6 +54,15 @@ const METRICS = {
   LOCK_WAIT_DURATION: 'migronaut.lock.wait.duration',
   LOCK_REFUSED: 'migronaut.lock.refused',
   LOCK_LOST: 'migronaut.lock.lost',
+  SEARCH_WAIT_DURATION: 'migronaut.converge.search.wait.duration',
+  BACKGROUND_DOCUMENTS: 'migronaut.background.documents',
+  BACKGROUND_SLICE_DURATION: 'migronaut.background.slice.duration',
+  BACKGROUND_BATCH_WRITE_DURATION: 'migronaut.background.batch.write.duration',
+  BACKGROUND_THROTTLE: 'migronaut.background.throttled',
+  BACKGROUND_DRIFT: 'migronaut.background.drift.detected',
+  BACKGROUND_TRANSACTION_RETRIES: 'migronaut.background.transaction.retried',
+  BACKGROUND_LEASES_RECLAIMED: 'migronaut.background.leases.reclaimed',
+  BACKGROUND_WATCH_DELAY: 'migronaut.background.watch.delay',
 };
 
 /**
@@ -130,6 +147,9 @@ function failureText(error) {
 
 /** The parts of `telemetry` — anything else in it is a typo, mentioned at debug level */
 const TELEMETRY_KEYS = Object.freeze(['tracer', 'meter', 'attributes']);
+
+/** The types a static attribute's value may have */
+const ATTRIBUTE_VALUE_TYPES = new Set(['string', 'number', 'boolean']);
 /** Static attributes are dimensions: a handful at most */
 const MAX_STATIC_ATTRIBUTES = 20;
 
@@ -184,7 +204,7 @@ function telemetryIssues(telemetry) {
       }
       for (const key of keys) {
         const value = attributes[key];
-        if (!['string', 'number', 'boolean'].includes(typeof value)) {
+        if (!ATTRIBUTE_VALUE_TYPES.has(typeof value)) {
           issues.push({
             path: `telemetry.attributes.${key}`,
             message: 'must be a string, a number or a boolean',
@@ -329,6 +349,51 @@ function createTelemetry(telemetry, { dbName } = {}) {
     '{refusal}',
   );
   const lockLost = counter(METRICS.LOCK_LOST, 'Migration locks lost mid-run', '{loss}');
+  const searchWaitDuration = histogram(
+    METRICS.SEARCH_WAIT_DURATION,
+    'Time a converge waited for its search index builds, by how the wait ended',
+  );
+
+  const backgroundDocuments = counter(
+    METRICS.BACKGROUND_DOCUMENTS,
+    'Documents a background migration handled, by result (migrated, skipped, conflict, failed)',
+    '{document}',
+  );
+  const backgroundSliceDuration = histogram(
+    METRICS.BACKGROUND_SLICE_DURATION,
+    'Duration of one background migration slice, by outcome',
+  );
+  const backgroundBatchWrite = histogram(
+    METRICS.BACKGROUND_BATCH_WRITE_DURATION,
+    'Time to write one background migration batch',
+  );
+  const backgroundThrottle = counter(
+    METRICS.BACKGROUND_THROTTLE,
+    'Adaptive throttle changes of background migrations, by reason',
+    '{change}',
+  );
+  const backgroundDrift = counter(
+    METRICS.BACKGROUND_DRIFT,
+    'Old-shape documents found after a background migration completed',
+    '{finding}',
+  );
+  const backgroundTransactionRetries = counter(
+    METRICS.BACKGROUND_TRANSACTION_RETRIES,
+    'Transactional background batches retried, by reason',
+    '{retry}',
+  );
+  const backgroundLeasesReclaimed = counter(
+    METRICS.BACKGROUND_LEASES_RECLAIMED,
+    'Partition leases reclaimed from a lane that stopped renewing',
+    '{lease}',
+  );
+  const backgroundWatchDelay = histogram(
+    METRICS.BACKGROUND_WATCH_DELAY,
+    'Time from an old-shape write to its upgrade by the live drift watcher',
+  );
+  const add = (instrument, value, attributes) => {
+    if (instrument && value > 0) safe(() => instrument.add(value, withBase(attributes)));
+  };
 
   // Durations are measured in milliseconds everywhere in migronaut and
   // reported in seconds, the unit OpenTelemetry's conventions settle on.
@@ -374,6 +439,65 @@ function createTelemetry(telemetry, { dbName } = {}) {
     lockLost() {
       increment(lockLost);
     },
+    /**
+     * A converge's wait for search index builds ended — `outcome` is
+     * `'ready'`, `'failed'`, `'timeout'`, `'unreadable'` or `'aborted'`. One
+     * point per wait, however many polls.
+     */
+    searchWaited({ waitedMs, outcome }) {
+      record(searchWaitDuration, waitedMs, { [ATTRIBUTES.SEARCH_WAIT_OUTCOME]: outcome });
+    },
+    /**
+     * A background migration slice ended: its duration by outcome, and the
+     * documents it handled by result. The partition is never an attribute —
+     * dimensions stay low-cardinality.
+     */
+    backgroundSliceEnded({ name, durationMs, outcome, counters = {}, error }) {
+      const at = { [ATTRIBUTES.BACKGROUND_NAME]: name };
+      record(backgroundSliceDuration, durationMs, {
+        ...at,
+        [ATTRIBUTES.BACKGROUND_OUTCOME]: outcome,
+        ...failure(error),
+      });
+      for (const [key, result] of [
+        ['migrated', 'migrated'],
+        ['skipped', 'skipped'],
+        ['conflicts', 'conflict'],
+        ['failed', 'failed'],
+      ]) {
+        add(backgroundDocuments, counters[key] ?? 0, {
+          ...at,
+          [ATTRIBUTES.BACKGROUND_RESULT]: result,
+        });
+      }
+    },
+    backgroundBatchWritten({ name, durationMs, shard }) {
+      record(backgroundBatchWrite, durationMs, {
+        [ATTRIBUTES.BACKGROUND_NAME]: name,
+        [ATTRIBUTES.BACKGROUND_SHARD]: shard,
+      });
+    },
+    backgroundThrottled({ name, reason }) {
+      add(backgroundThrottle, 1, {
+        [ATTRIBUTES.BACKGROUND_NAME]: name,
+        [ATTRIBUTES.BACKGROUND_REASON]: reason,
+      });
+    },
+    backgroundDrift({ name, count = 1 }) {
+      add(backgroundDrift, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
+    },
+    backgroundTransactionRetried({ name, reason, count = 1 }) {
+      add(backgroundTransactionRetries, count, {
+        [ATTRIBUTES.BACKGROUND_NAME]: name,
+        [ATTRIBUTES.BACKGROUND_REASON]: reason,
+      });
+    },
+    backgroundLeasesReclaimed({ name, count }) {
+      add(backgroundLeasesReclaimed, count, { [ATTRIBUTES.BACKGROUND_NAME]: name });
+    },
+    backgroundWatchDelay({ name, delayMs }) {
+      record(backgroundWatchDelay, delayMs, { [ATTRIBUTES.BACKGROUND_NAME]: name });
+    },
   };
 }
 

package/src/utils/template.js

@@ -145,6 +145,51 @@ module.exports = { description, up, down };
 `;
 }
 
+/**
+ * The built-in background migration template (`create --background`): the
+ * declarative form, with the knobs most worth knowing about spelled out.
+ */
+function defaultBackgroundTemplate(js, esm = false) {
+  const typed = js
+    ? "/** @type {import('@alexify/migronaut').DeclarativeBackgroundMigration} */\n"
+    : '';
+  const typeImport = js
+    ? ''
+    : "import type { DeclarativeBackgroundMigration } from '@alexify/migronaut';\n\n";
+  const annotation = js ? '' : ': DeclarativeBackgroundMigration';
+  const body = `{
+  // The collection to rewrite (\`up\` refuses this placeholder).
+  collection: 'TODO',
+  // Documents at version \`from\` (0: no version field yet) become version \`to\`.
+  from: 1,
+  to: 2,
+  // The new document for one old one — return it reshaped; migronaut sets the
+  // version, bumps the revision and writes only the fields that changed.
+  migrate: (doc) => {
+    // TODO: reshape doc, and return it
+    throw new Error('migrate is not written yet');
+  },
+  // The way back, for \`down\`. Without one, \`down\` refuses once documents
+  // were rewritten. Never \`(doc) => doc\`: that would stamp the old version
+  // on documents still in the new shape.
+  // revert: ({ shipping, ...doc }) => ({ ...doc, address: shipping.address }),
+  // Partitions worked at once, across every process (default 1).
+  // maxParallel: 4,
+}`;
+  if (esm) {
+    return `${typeImport}export const description = '';
+
+${typed}export const background${annotation} = ${body};
+`;
+  }
+  return `${typeImport}const description = '';
+
+${typed}const background${annotation} = ${body};
+
+module.exports = { description, background };
+`;
+}
+
 /** Extensions a custom `--template` file may have — anything else is refused */
 const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
 
@@ -152,7 +197,8 @@ const TEMPLATE_EXTENSIONS = ['.ts', '.js', '.cjs', '.mjs'];
 const MAX_TEMPLATE_BYTES = 1024 * 1024;
 
 /** Resolve template file contents — a custom template if provided, else the built-in */
-async function resolveTemplateContent(templatePath, js, esm = false) {
+async function resolveTemplateContent(templatePath, js, esm = false, background = false) {
+  if (background) return defaultBackgroundTemplate(js, esm);
   if (templatePath) {
     const ext = path.extname(templatePath);
     if (!TEMPLATE_EXTENSIONS.includes(ext)) {
@@ -198,6 +244,7 @@ async function createMigrationFile(options) {
     // Match the project's module system, so a generated migration never makes
     // Node reparse it and warn.
     await isEsmProject(options.dir),
+    options.background === true,
   );
   try {
     // 'wx' fails if the path exists — creating a migration must never silently
@@ -358,6 +405,27 @@ function configBody(values, createExtension) {
   // collectionsDir: './collections',
   // Converge at the end of every bulk \`migronaut up\`.
   // convergeAfterUp: false,
+  // Search indexes declared for a server without Atlas Search: refuse the
+  // converge ('fail'), or converge everything else and skip them ('skip').
+  // onSearchUnavailable: 'fail',
+  // Hold converge until every declared search index is queryable — for at
+  // most searchIndexWaitTimeoutMs (10 minutes by default).
+  // waitForSearchIndexes: false,
+  // searchIndexWaitTimeoutMs: 600000,
+
+  // ── Background migrations (experimental) ────────────────────
+  // A migration file with \`export const background = {…}\` is registered by
+  // \`up\` and runs in partitions (BullMQ, \`migronaut background run\`, or an
+  // in-process runner) without holding the migration lock.
+  // backgroundCollection: '_migronaut_background',
+  // Run it to the end inside the \`up\` that registers it instead.
+  // backgroundInline: false,
+  // Old-shape documents after it completed: reopen it ('reopen') or report.
+  // backgroundOnDrift: 'reopen',
+  // How drift is watched: 'poll' (every 10 minutes), 'stream', or 'both'.
+  // backgroundDrift: 'poll',
+  // Partition a sharded collection by its shard key ('auto') or not ('off').
+  // backgroundShardAware: 'auto',
 
   // ── Lifecycle hooks (code only — not available in JSON config) ──
   // hooks: {

package/src/versioning/config.js

@@ -0,0 +1,155 @@
+const { ConfigInvalidError } = require('../errors/index.js');
+const { isPlainObject } = require('./internal.js');
+
+/**
+ * The `versioning` block of a collection definition — one source of truth for
+ * the shape version a collection is at, read by converge (validator, index,
+ * the `min` guard), by the background-migration engine and by the
+ * application's repository layer through `defineShapes`.
+ *
+ * Validated strictly, like every other definition key: a typo (`currnet`)
+ * must not read as "not versioned".
+ */
+
+const VERSIONING_KEYS = ['current', 'min', 'field', 'revision', 'revisionField', 'index'];
+const VERSIONING_KEY_SET = new Set(VERSIONING_KEYS);
+
+const VERSIONING_DEFAULTS = Object.freeze({
+  min: 1,
+  field: '__v',
+  revision: true,
+  revisionField: '__rev',
+  index: true,
+});
+
+/** Longest field name accepted for the version or revision field */
+const MAX_FIELD_NAME = 64;
+
+/**
+ * Why `name` cannot be a system field, or `null`. Top-level only: the engine
+ * writes it with `$set`/`$inc`, so a dot would address a nested path and a
+ * leading `$` an operator; `_id` is immutable.
+ */
+function fieldNameIssue(name) {
+  if (typeof name !== 'string' || name.length === 0) return 'must be a non-empty string';
+  if (name.length > MAX_FIELD_NAME) return `must be at most ${MAX_FIELD_NAME} characters`;
+  if (name.startsWith('$')) return "must not start with '$'";
+  if (name.includes('.')) return "must be a top-level field (no '.')";
+  if (name.includes('\0')) return 'must not contain NUL';
+  if (name === '_id') return 'must not be _id';
+  // As a JavaScript key it names the prototype: every write of it would be lost.
+  if (name === '__proto__') return 'must not be __proto__';
+  return null;
+}
+
+const isCount = (value, floor) => Number.isSafeInteger(value) && value >= floor;
+
+/**
+ * Validate a `versioning` value, returning `{ path, message }` issues (empty
+ * when valid). `path` is where it sits — `collections[2].versioning`,
+ * `orders.ts: versioning`.
+ */
+function versioningIssues(value, path = 'versioning') {
+  if (!isPlainObject(value)) {
+    return [
+      {
+        path,
+        message: 'must be an object: { current, min?, field?, revision?, revisionField?, index? }',
+      },
+    ];
+  }
+  const issues = [];
+  const report = (key, message) => issues.push({ path: `${path}.${key}`, message });
+  for (const key of Object.keys(value)) {
+    if (!VERSIONING_KEY_SET.has(key)) {
+      report(key, `is not a versioning key (expected one of: ${VERSIONING_KEYS.join(', ')})`);
+    }
+  }
+  const { current, min, field, revision, revisionField, index } = value;
+  const currentValid = isCount(current, 1);
+  if (current === undefined) report('current', 'is required');
+  else if (!currentValid) report('current', 'must be an integer ≥ 1');
+  // The default min (1) never exceeds a valid current (≥ 1).
+  if (min !== undefined) {
+    if (!isCount(min, 0)) report('min', 'must be an integer ≥ 0');
+    else if (currentValid && min > current) {
+      report('min', `must not exceed current (${current})`);
+    }
+  }
+  if (field !== undefined) {
+    const issue = fieldNameIssue(field);
+    if (issue) report('field', issue);
+  }
+  if (revision !== undefined && typeof revision !== 'boolean') {
+    report('revision', 'must be a boolean');
+  }
+  if (revisionField !== undefined) {
+    const issue = fieldNameIssue(revisionField);
+    if (issue) report('revisionField', issue);
+    else if (revision === false) report('revisionField', 'has no effect with revision: false');
+  }
+  if (revision !== false) {
+    const versionName = field ?? VERSIONING_DEFAULTS.field;
+    const revisionName = revisionField ?? VERSIONING_DEFAULTS.revisionField;
+    if (versionName === revisionName) {
+      report('revisionField', `must differ from the version field ("${versionName}")`);
+    }
+  }
+  if (index !== undefined && typeof index !== 'boolean') report('index', 'must be a boolean');
+  return issues;
+}
+
+/**
+ * A validated `versioning` value with every default filled in, frozen.
+ * `revisionField` is `null` when revisions are off.
+ *
+ * @throws {ConfigInvalidError} with every issue in `context.issues`
+ */
+function resolveVersioning(value, { path = 'versioning' } = {}) {
+  const issues = versioningIssues(value, path);
+  if (issues.length > 0) {
+    throw new ConfigInvalidError(`Invalid ${path}: ${issues[0].path} ${issues[0].message}`, {
+      issues,
+    });
+  }
+  const revision = value.revision ?? VERSIONING_DEFAULTS.revision;
+  return Object.freeze({
+    current: value.current,
+    min: value.min ?? VERSIONING_DEFAULTS.min,
+    field: value.field ?? VERSIONING_DEFAULTS.field,
+    revision,
+    revisionField: revision ? (value.revisionField ?? VERSIONING_DEFAULTS.revisionField) : null,
+    index: value.index ?? VERSIONING_DEFAULTS.index,
+  });
+}
+
+/**
+ * The resolved versioning of `collection` among `definitions` — an array of
+ * definitions (each with `name`) or a `{ name: definition }` map — or `null`
+ * when it is not declared or not versioned.
+ */
+function versioningOf(definitions, collection) {
+  let definition;
+  if (Array.isArray(definitions)) {
+    for (const candidate of definitions) {
+      if (isPlainObject(candidate) && candidate.name === collection) {
+        definition = candidate;
+        break;
+      }
+    }
+  } else if (isPlainObject(definitions) && Object.hasOwn(definitions, collection)) {
+    definition = definitions[collection];
+  }
+  if (!isPlainObject(definition) || definition.versioning === undefined) return null;
+  return resolveVersioning(definition.versioning, { path: `${collection}.versioning` });
+}
+
+module.exports = {
+  MAX_FIELD_NAME,
+  VERSIONING_DEFAULTS,
+  VERSIONING_KEYS,
+  fieldNameIssue,
+  resolveVersioning,
+  versioningIssues,
+  versioningOf,
+};