@alexify/migronaut 1.0.0 → 2.0.0

package/src/cli/index.js

@@ -1,5 +1,6 @@
 const { Command } = require('./args.js');
 const { registerAudit } = require('./commands/audit.js');
+const { registerBaseline } = require('./commands/baseline.js');
 const { registerCreate } = require('./commands/create.js');
 const { registerDown } = require('./commands/down.js');
 const { registerDryRun } = require('./commands/dry-run.js');
@@ -37,6 +38,7 @@ function buildProgram() {
 
   registerInit(program);
   registerImport(program);
+  registerBaseline(program);
   registerUp(program);
   registerDown(program);
   registerRedo(program);

package/src/cli/shared.js

@@ -1,6 +1,6 @@
 const { createInterface } = require('node:readline/promises');
 const { createSpinner } = require('./spinner.js');
-const { ConfigInvalidError, MigronautError } = require('../errors/index.js');
+const { ConfigInvalidError, MigronautError, RunAbortedError } = require('../errors/index.js');
 const { errorText } = require('../utils/error.js');
 const { createLogger } = require('../utils/logger.js');
 const { redactDeep, redactUris } = require('../utils/redact.js');
@@ -50,7 +50,11 @@ const SIGNAL_EXIT_CODES = { SIGINT: 130, SIGTERM: 143 };
  * exits immediately for an operator who cannot wait.
  *
  * Returns a function that removes the handlers again, so a long-lived process
- * calling the CLI repeatedly does not accumulate them.
+ * calling the CLI repeatedly does not accumulate them. The function carries a
+ * `stopRequested()` accessor: a signal that lands while nothing is running yet
+ * (the cosmetic pre-connect) makes `migrator.stop()` a no-op, so the caller
+ * must consult this flag itself before starting the run — otherwise the
+ * handler's "then stopping" promise above would be a lie.
  */
 function attachSignalHandlers(migrator, spinner, logger) {
   let stopping = false;
@@ -76,9 +80,11 @@ function attachSignalHandlers(migrator, spinner, logger) {
     process.on(signal, handler);
     handlers.push([signal, handler]);
   }
-  return () => {
+  const detach = () => {
     for (const [signal, handler] of handlers) process.off(signal, handler);
   };
+  detach.stopRequested = () => stopping;
+  return detach;
 }
 
 /**
@@ -225,6 +231,12 @@ async function withMigrator(opts, fn, options = {}) {
         throw error;
       }
     }
+    // A signal during the pre-connect lands before any run window exists, so
+    // migrator.stop() was a no-op — honor it here, before starting the run the
+    // handler already told the operator would be stopped.
+    if (detachSignals.stopRequested?.()) {
+      throw new RunAbortedError('Stopped by signal before the run started', { results: [] });
+    }
     await fn(migrator, { logger, json, opts });
   } catch (error) {
     // Safety net: clear any spinner still spinning before printing the error.

package/src/core/audit.js

@@ -99,15 +99,20 @@ async function runAudit(deps) {
     record('lock', 'warn', `Could not read the lock: ${errorText(error)}`);
   }
 
-  // 6. Checksum drift and missing files, from the same rows `status` renders.
+  // 6. Checksum drift, missing files, pending count and ordering, from the
+  //    same rows `status` renders.
   try {
     const rows = await deps.status();
-    // One pass over the rows collects both signals.
+    // One pass over the rows collects every signal.
     const drifted = [];
+    const outOfOrder = [];
     let pending = 0;
     for (const row of rows) {
       if (row.checksumOk === false) drifted.push(row.file);
-      if (row.status === 'pending') pending += 1;
+      // A recorded failed attempt still counts as pending work — the file
+      // will be retried by the next `up`.
+      if (row.status === 'pending' || row.status === 'failed') pending += 1;
+      if (row.outOfOrder) outOfOrder.push(row.file);
     }
     if (drifted.length > 0) {
       record('checksums', 'fail', `Edited after being applied: ${drifted.join(', ')}`);
@@ -115,6 +120,15 @@ async function runAudit(deps) {
       record('checksums', 'pass', 'No drift among applied migrations');
     }
     record('pending', pending === 0 ? 'pass' : 'warn', `${pending} pending migration(s)`);
+    if (outOfOrder.length > 0) {
+      record(
+        'ordering',
+        'warn',
+        `Pending but older than the newest applied migration: ${outOfOrder.join(', ')}`,
+      );
+    } else {
+      record('ordering', 'pass', 'No out-of-order pending migrations');
+    }
   } catch (error) {
     record('checksums', 'warn', `Could not read status: ${errorText(error)}`);
   }

package/src/core/baseline.js

@@ -0,0 +1,80 @@
+const { computeChecksum } = require('../utils/checksum.js');
+const { mapLimit } = require('../utils/concurrency.js');
+
+/** Simultaneous file hashes — same EMFILE bound as every other multi-file path */
+const FS_CONCURRENCY = 16;
+
+/**
+ * Adopt an existing database with no prior migration tool: mark migration
+ * files on disk as applied — checksums taken from disk, one shared batch,
+ * `origin: 'baseline'` — without executing anything. The database is assumed
+ * to already be in the state those files describe (they were applied by hand,
+ * by a home-grown script, or reconstructed after the fact).
+ *
+ * Forward-only: baselined records were never executed by migronaut, so
+ * `down`/`redo` refuse them (the same `origin` preflight import uses).
+ * Idempotent: already-applied names are skipped, so a partial baseline can
+ * simply be re-run.
+ *
+ * Pure orchestration over capabilities the MigratorKit injects (`deps`):
+ * `{db, changelog, logger, fields, filepath, listMigrationFiles, nextBatch,
+ * truncateAtTarget, environment, executedBy, runId, assertNotAborted}`.
+ */
+async function runBaseline(deps, options, signal) {
+  const { db, changelog, logger } = deps;
+
+  const files = await deps.listMigrationFiles();
+  const applied = new Set(await changelog.getAppliedNames(db));
+  let targets = [];
+  for (const file of files) {
+    if (!applied.has(file)) targets.push(file);
+  }
+  if (options.to !== undefined) {
+    targets = deps.truncateAtTarget(targets, files, options.to);
+  }
+  const skipped = files.length - targets.length;
+
+  if (targets.length === 0) {
+    logger.info('Nothing to baseline', deps.fields({ skipped }));
+    return { baselined: [], skipped, batch: null };
+  }
+
+  // Checksums come from the files as they are NOW — that is the contract: the
+  // baseline asserts "the database already matches these exact files", and
+  // later drift checks police edits against this snapshot.
+  const checksums = await mapLimit(targets, FS_CONCURRENCY, (name) =>
+    computeChecksum(deps.filepath(name)),
+  );
+
+  const batch = await deps.nextBatch();
+  const records = new Array(targets.length);
+  for (let i = 0; i < targets.length; i++) {
+    records[i] = {
+      name: targets[i],
+      batch,
+      status: 'applied',
+      // No appliedAt: the changelog stamps it in server time.
+      duration: 0,
+      checksum: checksums[i],
+      environment: deps.environment(),
+      executedBy: deps.executedBy(),
+      origin: 'baseline',
+      ...(deps.runId() ? { runId: deps.runId() } : {}),
+    };
+  }
+
+  // One write, checked against the abort signal first: a baseline is all
+  // bookkeeping, so there is no safe partial point worth resuming from — and
+  // markAppliedBulk's upsert-by-name makes a re-run after any failure
+  // idempotent anyway.
+  deps.assertNotAborted(signal);
+  await changelog.markAppliedBulk(db, records);
+
+  logger.info(
+    `✔ Baselined ${records.length} migration(s) as applied (batch ${batch})`,
+    deps.fields({ baselined: records.length, skipped, batch }),
+  );
+  return { baselined: targets, skipped, batch };
+}
+
+module.exports = { runBaseline };

package/src/core/changelog.js

@@ -95,9 +95,18 @@ class Changelog {
     return this.#coll(db).findOne({ name });
   }
 
-  /** Return every currently-applied record, sorted by name ascending */
+  /**
+   * Every currently-applied record's `{name, checksum}`, sorted by name
+   * ascending — exactly what the strict bulk drift check consumes (the name
+   * doubles as the applied-set key). Projected like the module's other reads;
+   * widen the projection if a new caller needs more.
+   */
   async getApplied(db) {
-    return this.#coll(db).find({ status: 'applied' }).sort({ name: 1 }).toArray();
+    return this.#coll(db)
+      .find({ status: 'applied' })
+      .sort({ name: 1 })
+      .project({ _id: 0, name: 1, checksum: 1 })
+      .toArray();
   }
 
   /**
@@ -157,29 +166,54 @@ class Changelog {
     return this.#coll(db).find({ batch }).sort({ name: 1 }).toArray();
   }
 
+  /**
+   * The update document shared by markApplied and markAppliedBulk.
+   *
+   * `appliedAt` is stamped in **server time** (`$currentDate`) when the record
+   * does not carry one — the same clock discipline the lock's `$$NOW` uses:
+   * `redo` and `down --steps` sort by `appliedAt`, and a client-stamped value
+   * lets a skewed host mis-order the revert selection. An explicit `appliedAt`
+   * (import adopting a legacy changelog's historical timestamps) is written
+   * verbatim. `firstAppliedAt` is audit-only metadata, never sorted on, so its
+   * client-clock `$setOnInsert` fallback is acceptable ($setOnInsert cannot
+   * express server time).
+   */
+  static #appliedUpdate(record) {
+    // `name` comes from the filter on insert, so it must not also appear in an
+    // update operator (MongoDB rejects the conflicting path).
+    const { name, appliedAt, ...fields } = record;
+    const update = {
+      $set: fields,
+      // A re-apply clears the stale revert marker — and the failure trace a
+      // markFailed() from an earlier crashed attempt may have left.
+      $unset: { revertedAt: '', failedAt: '', error: '' },
+    };
+    if (appliedAt !== undefined) {
+      update.$set.appliedAt = appliedAt;
+      update.$setOnInsert = { firstAppliedAt: appliedAt };
+    } else {
+      update.$currentDate = { appliedAt: true };
+      update.$setOnInsert = { firstAppliedAt: new Date() };
+    }
+    return update;
+  }
+
   /**
    * Record a migration as applied. Upserts on `name` so re-applying a
    * previously-reverted migration (e.g. via `redo`) cannot violate the unique
    * index. Uses `$set` rather than a whole-document replace so audit fields
    * survive a re-apply: `firstAppliedAt` is stamped once, and the stale
-   * `revertedAt` from an earlier rollback is cleared.
+   * `revertedAt` from an earlier rollback is cleared. `appliedAt` is stamped
+   * server-side unless the record carries one — see {@link #appliedUpdate}.
    *
    * Pass `session` to make this write part of the migration's transaction, so
    * the migration and its changelog record commit together.
    */
   async markApplied(db, record, session) {
-    // `name` comes from the filter on insert, so it must not also appear in an
-    // update operator (MongoDB rejects the conflicting path).
-    const { name, ...fields } = record;
-    await this.#coll(db).updateOne(
-      { name },
-      {
-        $set: fields,
-        $setOnInsert: { firstAppliedAt: record.appliedAt },
-        $unset: { revertedAt: '' },
-      },
-      { upsert: true, ...(session ? { session } : {}) },
-    );
+    await this.#coll(db).updateOne({ name: record.name }, Changelog.#appliedUpdate(record), {
+      upsert: true,
+      ...(session ? { session } : {}),
+    });
   }
 
   /**
@@ -193,15 +227,10 @@ class Changelog {
     if (records.length === 0) return;
     const ops = new Array(records.length);
     for (let i = 0; i < records.length; i++) {
-      const { name, ...fields } = records[i];
       ops[i] = {
         updateOne: {
-          filter: { name },
-          update: {
-            $set: fields,
-            $setOnInsert: { firstAppliedAt: records[i].appliedAt },
-            $unset: { revertedAt: '' },
-          },
+          filter: { name: records[i].name },
+          update: Changelog.#appliedUpdate(records[i]),
           upsert: true,
         },
       };
@@ -222,10 +251,32 @@ class Changelog {
   async markReverted(db, name, session) {
     return this.#coll(db).updateOne(
       { name, status: 'applied' },
-      { $set: { status: 'reverted', revertedAt: new Date() } },
+      // Server time, like markApplied's appliedAt — one clock for the whole trail.
+      { $set: { status: 'reverted' }, $currentDate: { revertedAt: true } },
       session ? { session } : {},
     );
   }
+
+  /**
+   * Best-effort trace of a failed `up` attempt, so a crash-and-restart leaves
+   * DB-side evidence of what was in flight ("did the crashed run start X?")
+   * instead of depending on process logs that may not have been captured.
+   *
+   * The filter excludes `'applied'` records: a forced re-run's failure must
+   * never demote a migration the changelog says is applied — the upsert then
+   * collides on the unique `name` index, and the caller swallows that. Every
+   * read path filters on `status: 'applied'`, so a `'failed'` record never
+   * changes what runs; the next successful apply overwrites it (and clears
+   * `failedAt`/`error`).
+   */
+  async markFailed(db, record) {
+    const { name, ...fields } = record;
+    await this.#coll(db).updateOne(
+      { name, status: { $ne: 'applied' } },
+      { $set: { ...fields, status: 'failed' }, $currentDate: { failedAt: true } },
+      { upsert: true },
+    );
+  }
 }
 
 module.exports = { Changelog };

package/src/core/config.js

@@ -7,7 +7,12 @@ const { errorText } = require('../utils/error.js');
 const { resolveLogger } = require('../utils/logger.js');
 const { redactDeep } = require('../utils/redact.js');
 
-/** Default values applied when no flag, env var, or config-file value is present */
+/**
+ * Default values applied when no flag, env var, or config-file value is
+ * present. The single canonical home for every effective default — a use-site
+ * `??` fallback would hide these from the schema/template sync tests and let
+ * the same fact drift across hand-written copies.
+ */
 const DEFAULT_CONFIG = {
   migrationsDir: './migrations',
   migrationsCollection: '_migronaut_migrations',
@@ -18,6 +23,10 @@ const DEFAULT_CONFIG = {
   fileExtensions: ['.ts', '.js'],
   createExtension: 'js',
   sequential: false,
+  ensureIndexes: true,
+  onLockLost: 'abort',
+  onOutOfOrder: 'warn',
+  reloadMigrations: false,
 };
 
 /** Candidate config file names, checked in priority order within the cwd */
@@ -98,6 +107,12 @@ const CONFIG_KEYS = [
     message: "must be 'abort' or 'warn'",
     optional: true,
   },
+  {
+    path: 'onOutOfOrder',
+    check: (value) => value === 'warn' || value === 'error' || value === 'allow',
+    message: "must be 'warn', 'error' or 'allow'",
+    optional: true,
+  },
   {
     path: 'envFile',
     check: (value) => value === false || isNonEmptyString(value),
@@ -248,6 +263,11 @@ const ENV_KEYS = [
   { env: 'MIGRONAUT_TEMPLATE_PATH', path: 'templatePath', parse: parseString },
   { env: 'MIGRONAUT_TIMEOUT_MS', path: 'timeoutMs', parse: parsePositiveInteger },
   { env: 'MIGRONAUT_ON_LOCK_LOST', path: 'onLockLost', parse: parseEnum(['abort', 'warn']) },
+  {
+    env: 'MIGRONAUT_ON_OUT_OF_ORDER',
+    path: 'onOutOfOrder',
+    parse: parseEnum(['warn', 'error', 'allow']),
+  },
   { env: 'MIGRONAUT_ENSURE_INDEXES', path: 'ensureIndexes', parse: parseBoolean },
   { env: 'MIGRONAUT_RELOAD_MIGRATIONS', path: 'reloadMigrations', parse: parseBoolean },
 ];
@@ -394,7 +414,10 @@ async function loadConfig(options = {}) {
     : await discoverConfigFile(cwd);
 
   if (configFilePath) {
-    if (!(await pathExists(configFilePath))) {
+    // Only an explicit --config path needs the probe (a typo deserves a clear
+    // "not found") — discovery already proved existence, and re-checking it
+    // would pay a redundant fs.access on every invocation.
+    if (options.configPath && !(await pathExists(configFilePath))) {
       throw new ConfigInvalidError('Config file not found', { path: configFilePath });
     }
     const fileConfig = await loadConfigFile(configFilePath, options.lenient ?? false);

package/src/core/import-runner.js

@@ -1,4 +1,4 @@
-const { ImportTargetNotEmptyError } = require('../errors/index.js');
+const { ImportTargetNotEmptyError, MigronautError } = require('../errors/index.js');
 const { computeChecksum } = require('../utils/checksum.js');
 const { Changelog } = require('./changelog.js');
 const { isMigrateMongoDoc, mapMigrateMongoDocs } = require('./import.js');
@@ -52,7 +52,14 @@ async function runImport(deps, options, signal) {
       valid.push(doc);
     } else {
       skipped += 1;
-      logger.warn('⚠ Skipping source doc without a usable fileName');
+      // The _id is the only handle the operator has for locating the offending
+      // source document — an anonymous count is undebuggable after the fact.
+      // String() keeps an ObjectId safe for any sink; the default logger
+      // already sanitizes terminal escapes in DB-derived text.
+      logger.warn(
+        `⚠ Skipping source doc without a usable fileName (_id: ${String(doc?._id)})`,
+        deps.fields({ source, docId: String(doc?._id) }),
+      );
     }
   }
 
@@ -91,7 +98,10 @@ async function runImport(deps, options, signal) {
       );
       rowSources.set(fileName, resolved.source);
       if (resolved.source === 'missing') {
-        logger.warn(`⚠ File not found on disk: ${fileName} — checksum unverifiable`);
+        logger.warn(
+          `⚠ File not found on disk: ${fileName} — checksum unverifiable`,
+          deps.fields({ file: fileName, checksumSource: 'missing' }),
+        );
       }
       return resolved;
     },
@@ -120,9 +130,27 @@ async function runImport(deps, options, signal) {
   // adopting a 5,000-record changelog is 5 round trips, not 5,000, all while
   // holding the migration lock. The abort check between chunks lets stop() /
   // a lost lock halt a long import instead of running it to completion.
-  for (let start = 0; start < records.length; start += IMPORT_CHUNK_SIZE) {
-    deps.assertNotAborted(signal);
-    await targetChangelog.markAppliedBulk(db, records.slice(start, start + IMPORT_CHUNK_SIZE));
+  let written = 0;
+  try {
+    for (let start = 0; start < records.length; start += IMPORT_CHUNK_SIZE) {
+      deps.assertNotAborted(signal);
+      await targetChangelog.markAppliedBulk(db, records.slice(start, start + IMPORT_CHUNK_SIZE));
+      written += Math.min(IMPORT_CHUNK_SIZE, records.length - start);
+    }
+  } catch (error) {
+    // Earlier chunks are already committed; without a count the operator only
+    // discovers the half-populated target when the next plain `import` throws
+    // ImportTargetNotEmptyError. Recovery is safe — the upserts are keyed on
+    // `name`, so a --force re-run is idempotent and simply resumes.
+    logger.warn(
+      `⚠ Import interrupted after ${written}/${records.length} record(s) — ` +
+        'a --force re-run is idempotent and will resume',
+      deps.fields({ source, target, imported: written, total: records.length }),
+    );
+    if (error instanceof MigronautError && error.context) {
+      error.context = { ...error.context, imported: written, total: records.length };
+    }
+    throw error;
   }
 
   logger.info(

package/src/core/import.js

@@ -1,3 +1,12 @@
+const { mapLimit } = require('../utils/concurrency.js');
+
+/**
+ * Simultaneous checksum resolutions. Each one is a file read — an unbounded
+ * fan-out over a thousands-record legacy changelog would exhaust the
+ * descriptor limit (EMFILE), the exact hazard mapLimit exists for.
+ */
+const CHECKSUM_CONCURRENCY = 16;
+
 /** Returns true when a value looks like a usable migrate-mongo changelog doc */
 function isMigrateMongoDoc(value) {
   return (
@@ -30,13 +39,11 @@ async function mapMigrateMongoDocs(docs, options) {
     return delta !== 0 ? delta : a.fileName.localeCompare(b.fileName);
   });
 
-  // Independent per-doc disk reads — resolve them concurrently rather than
-  // one at a time.
-  const checksumPromises = [];
-  for (const doc of sorted) {
-    checksumPromises.push(options.resolveChecksum(doc.fileName, doc.fileHash));
-  }
-  const resolutions = await Promise.all(checksumPromises);
+  // Independent per-doc disk reads — resolved concurrently, but bounded:
+  // mapLimit preserves input order exactly like Promise.all would.
+  const resolutions = await mapLimit(sorted, CHECKSUM_CONCURRENCY, (doc) =>
+    options.resolveChecksum(doc.fileName, doc.fileHash),
+  );
 
   const records = [];
   for (let index = 0; index < sorted.length; index++) {

package/src/core/lock.js

@@ -81,13 +81,14 @@ class MigrationLock {
       owner: { $literal: owner },
     };
 
+    let result;
     try {
       // Upsert on plain `_id` — upserts reject `$expr` filters (server error
       // 224), so the staleness decision lives in the pipeline instead, still
       // in server time: take the lock when no `lockedAt` exists (fresh insert)
       // or the holder is stale; otherwise keep the current document untouched.
       // The read-back below tells those outcomes apart.
-      await collection.updateOne(
+      result = await collection.updateOne(
         { _id: LOCK_ID },
         [
           {
@@ -118,6 +119,15 @@ class MigrationLock {
       throw error;
     }
 
+    // A fresh upsert-insert on the unique `_id` already proves ownership — no
+    // other outcome can insert — and it is the common uncontended path, since
+    // release() deletes the document after every clean run. Skipping the
+    // read-back halves that path's round trips.
+    if (result.upsertedCount === 1) {
+      this.#owner = owner;
+      return;
+    }
+
     // Confirm we are the holder. A fresh lock left the document untouched, and
     // if two processes raced to reclaim the same stale lock only the last
     // writer's `owner` wins; either way the loser reads a different token here
@@ -166,12 +176,16 @@ class MigrationLock {
 
   /**
    * Release the lock by deleting the lock document. Scoped to our `owner` token
-   * (when held) so we never delete a lock that has since been reclaimed by
-   * another process.
+   * so we never delete a lock that has since been reclaimed by another process.
+   * With no token held this is a no-op — an unscoped delete here would be
+   * `forceRelease()` without its deliberate opt-in, and a future caller
+   * releasing twice (or before acquiring) must not silently steal a peer's
+   * live lock.
    * @throws {LockReleaseFailedError} when the delete operation fails
    */
   async release() {
-    const filter = this.#owner ? { _id: LOCK_ID, owner: this.#owner } : { _id: LOCK_ID };
+    if (!this.#owner) return;
+    const filter = { _id: LOCK_ID, owner: this.#owner };
     try {
       await this.#db.collection(this.#collectionName).deleteOne(filter);
       this.#owner = undefined;
@@ -332,8 +346,16 @@ async function runWithLock(lock, options, fn) {
   } catch (releaseError) {
     // Never let a release failure replace the reason the run failed: that would
     // report "Failed to release migration lock" instead of the actual migration
-    // error. When the run succeeded, the release failure is the only news.
-    if (!failed) throw releaseError;
+    // error. When the run succeeded, the release failure is the only news —
+    // but the migrations DID apply, and that list must survive onto the error,
+    // or "everything applied, only the lock cleanup failed" (the lock document
+    // self-heals via its TTL) is indistinguishable from a failed run.
+    if (!failed) {
+      if (releaseError instanceof LockReleaseFailedError && Array.isArray(result)) {
+        releaseError.context = { ...releaseError.context, results: [...result] };
+      }
+      throw releaseError;
+    }
     const message = errorText(releaseError);
     options.logger.warn(`⚠ Failed to release the migration lock: ${message}`, {
       event: 'lock:release-failed',