@alexify/migronaut 2.2.0 → 2.4.0

package/src/core/lock.js

@@ -8,9 +8,15 @@ const { errorText } = require('../utils/error.js');
 const { randomId } = require('../utils/id.js');
 const { safeUsername } = require('../utils/user.js');
 
-/** Fixed `_id` of the singleton lock document */
+/** Fixed `_id` of the singleton migration lock document */
 const LOCK_ID = 'migronaut_lock';
 
+/** What the migration lock is called in messages — another lock passes its own `label` */
+const LOCK_LABEL = 'migration lock';
+
+/** `migration lock` → `Migration lock`, for the start of a sentence */
+const sentence = (label) => label.charAt(0).toUpperCase() + label.slice(1);
+
 /** Returns true when an error is a MongoDB duplicate-key error (code 11000) */
 function isDuplicateKeyError(error) {
   return typeof error === 'object' && error !== null && 'code' in error && error.code === 11000;
@@ -40,11 +46,19 @@ function toLockInfo(doc) {
  * MongoDB-native distributed lock backed by a single document, using an atomic
  * upsert as a test-and-set. A lock older than `ttlSeconds` is considered stale
  * and may be reclaimed.
+ *
+ * `options.id` is the lock document's `_id` (default: the migration lock's)
+ * and `options.label` what messages call it — so other locks (a background
+ * migration's coordinator, `background:<name>`) live next to the migration
+ * lock in the same collection without ever touching it: `unlock`, the audit
+ * and `forceRelease` of the migration lock only see its own document.
  */
 class MigrationLock {
   #db;
   #collectionName;
   #ttlSeconds;
+  #id;
+  #label;
   /** Token naming this instance as the current holder; set on acquire, cleared on release */
   #owner;
   /**
@@ -55,10 +69,22 @@ class MigrationLock {
    */
   #nonce;
 
-  constructor(db, collectionName, ttlSeconds) {
+  constructor(db, collectionName, ttlSeconds, { id = LOCK_ID, label = LOCK_LABEL } = {}) {
     this.#db = db;
     this.#collectionName = collectionName;
     this.#ttlSeconds = ttlSeconds;
+    this.#id = id;
+    this.#label = label;
+  }
+
+  /** The lock document's `_id` */
+  get id() {
+    return this.#id;
+  }
+
+  /** What messages call this lock */
+  get label() {
+    return this.#label;
   }
 
   /** The token identifying this holder, or undefined when the lock is not held */
@@ -91,7 +117,7 @@ class MigrationLock {
     // strings: a pipeline expression would otherwise interpret a leading `$`
     // in a value as a field path.
     const lockDoc = {
-      _id: LOCK_ID,
+      _id: this.#id,
       lockedAt: '$$NOW',
       pid: { $literal: process.pid },
       host: { $literal: os.hostname() },
@@ -109,7 +135,7 @@ class MigrationLock {
       // or the holder is stale; otherwise keep the current document untouched.
       // The read-back below tells those outcomes apart.
       result = await collection.updateOne(
-        { _id: LOCK_ID },
+        { _id: this.#id },
         [
           {
             $replaceWith: {
@@ -131,8 +157,8 @@ class MigrationLock {
     } catch (error) {
       if (isDuplicateKeyError(error)) {
         // Two processes raced the very first insert; the loser lands here.
-        const holder = await collection.findOne({ _id: LOCK_ID });
-        throw new LockAlreadyHeldError('Migration lock is already held', {
+        const holder = await collection.findOne({ _id: this.#id });
+        throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
           holder: toLockInfo(holder) ?? undefined,
           ttlMs: this.ttlMs,
         });
@@ -155,9 +181,9 @@ class MigrationLock {
     // writer's document wins; either way the loser reads a different one here
     // and backs off instead of running concurrently. The nonce is what makes
     // that hold when both carry the same owner token.
-    const current = await collection.findOne({ _id: LOCK_ID });
+    const current = await collection.findOne({ _id: this.#id });
     if (!current || current.owner !== owner || current.nonce !== nonce) {
-      throw new LockAlreadyHeldError('Migration lock is already held', {
+      throw new LockAlreadyHeldError(`${sentence(this.#label)} is already held`, {
         holder: toLockInfo(current) ?? undefined,
         ttlMs: this.ttlMs,
       });
@@ -178,7 +204,7 @@ class MigrationLock {
     }
     const result = await this.#db
       .collection(this.#collectionName)
-      .updateOne({ _id: LOCK_ID, owner: this.#owner, nonce: this.#nonce }, [
+      .updateOne({ _id: this.#id, owner: this.#owner, nonce: this.#nonce }, [
         { $set: { lockedAt: '$$NOW' } },
       ]);
     return result.matchedCount === 1;
@@ -186,7 +212,7 @@ class MigrationLock {
 
   /** Read the current lock document, or null when no lock is held */
   async inspect() {
-    return this.#db.collection(this.#collectionName).findOne({ _id: LOCK_ID });
+    return this.#db.collection(this.#collectionName).findOne({ _id: this.#id });
   }
 
   /**
@@ -196,8 +222,8 @@ class MigrationLock {
    */
   async forceRelease() {
     const collection = this.#db.collection(this.#collectionName);
-    const existing = await collection.findOne({ _id: LOCK_ID });
-    await collection.deleteOne({ _id: LOCK_ID });
+    const existing = await collection.findOne({ _id: this.#id });
+    await collection.deleteOne({ _id: this.#id });
     return existing;
   }
 
@@ -213,14 +239,14 @@ class MigrationLock {
    */
   async release() {
     if (!this.#owner) return;
-    const filter = { _id: LOCK_ID, owner: this.#owner, nonce: this.#nonce };
+    const filter = { _id: this.#id, owner: this.#owner, nonce: this.#nonce };
     try {
       await this.#db.collection(this.#collectionName).deleteOne(filter);
       this.#owner = undefined;
       this.#nonce = undefined;
     } catch (error) {
       throw new LockReleaseFailedError(
-        'Failed to release migration lock',
+        `Failed to release ${this.#label}`,
         { error: errorText(error) },
         { cause: error },
       );
@@ -253,6 +279,9 @@ class MigrationLock {
  */
 async function runWithLock(lock, options, fn) {
   const controller = new AbortController();
+  // Any object with acquire/renew/release/ttlMs is a lock here — a partition
+  // lease included; `label` names it in the messages.
+  const label = typeof lock?.label === 'string' ? lock.label : LOCK_LABEL;
 
   if (options.noLock) {
     options.logger.warn('⚠ Running without a lock (--no-lock) — concurrent runs are unsafe', {
@@ -278,15 +307,13 @@ async function runWithLock(lock, options, fn) {
   const loseLock = (reason) => {
     // The most alert-worthy line this module emits — structured fields so a
     // JSON sink can trigger on it without parsing the human string.
-    options.logger.warn(`⚠ Lost the migration lock mid-run (${reason})`, {
+    options.logger.warn(`⚠ Lost the ${label} mid-run (${reason})`, {
       event: 'lock:lost',
       reason,
     });
     options.onLockLostEvent?.(reason);
     if (abortOnLoss && !controller.signal.aborted) {
-      controller.abort(
-        new LockLostError('Lost the migration lock mid-run', { reason, aborted: true }),
-      );
+      controller.abort(new LockLostError(`Lost the ${label} mid-run`, { reason, aborted: true }));
     }
   };
 
@@ -381,7 +408,7 @@ async function runWithLock(lock, options, fn) {
         } catch (releaseError) {
           const message = errorText(releaseError);
           options.logger.warn(
-            `⚠ Failed to release the migration lock early: ${message} — it frees itself once ` +
+            `⚠ Failed to release the ${label} early: ${message} — it frees itself once ` +
               'its TTL runs out',
             { event: 'lock:release-failed', error: message, early: true },
           );
@@ -424,7 +451,7 @@ async function runWithLock(lock, options, fn) {
       throw releaseError;
     }
     const message = errorText(releaseError);
-    options.logger.warn(`⚠ Failed to release the migration lock: ${message}`, {
+    options.logger.warn(`⚠ Failed to release the ${label}: ${message}`, {
       event: 'lock:release-failed',
       error: message,
     });
@@ -434,4 +461,4 @@ async function runWithLock(lock, options, fn) {
   return result;
 }
 
-module.exports = { LOCK_ID, MigrationLock, runWithLock, toLockInfo };
+module.exports = { LOCK_ID, LOCK_LABEL, MigrationLock, runWithLock, toLockInfo };

package/src/core/migration-logger.js

@@ -0,0 +1,279 @@
+const { isPlainObject } = require('../utils/canonical.js');
+const { errorText } = require('../utils/error.js');
+const { jobFields } = require('../utils/job-ref.js');
+const { BOUNDS, redactBounded, redactOutbound } = require('../utils/redact.js');
+
+/**
+ * The logger a migration gets as `ctx.logger`, and the `migration:log` event.
+ *
+ * Every call is a line on the kit's own logger with the run's correlation
+ * bound into its fields (run id, migration, direction, batch, attempt, job) —
+ * a JSON sink can join it to the kit's lines, the changelog and the queue job
+ * without parsing anything. A call whose fields say `userland: true` is also
+ * emitted as `migration:log`, for the application to keep: migronaut stores
+ * none of it. The marker is per call, so operational noise stays in the logs
+ * and only what the author meant for their users reaches the event.
+ *
+ * Mechanism only: no kit, no decisions about where lines go — `sink` is the
+ * kit's resolved logger, `emitter` its event channel.
+ */
+
+const MIGRATION_LOG_EVENT = 'migration:log';
+const USERLAND = 'userland';
+
+/** The longest message an event carries; the log line keeps the whole one */
+const MAX_MESSAGE_LENGTH = 2048;
+
+/** A counter, one per run (or lane slice): orders a run's events within one millisecond */
+function sequence() {
+  let next = 0;
+  return () => {
+    next += 1;
+    return next;
+  };
+}
+
+/**
+ * The run-level correlation of an ordinary run, frozen — `ctx.run` in
+ * `beforeAll`/`afterAll`, and what every migration's own adds to.
+ * `base` holds the run id and, when the caller gave them, the job and the
+ * actor: `{ id, jobId?, groupId?, requestedBy?, reason? }`.
+ */
+function runInfo(base, direction) {
+  return Object.freeze({ id: base.id, direction, ...withoutId(base) });
+}
+
+/** `ctx.run` of one attempt of one migration — a new frozen object per attempt */
+function migrationRunInfo(run, { migration, batch, attempt }) {
+  return Object.freeze({
+    ...run,
+    migration,
+    ...(batch !== undefined ? { batch } : {}),
+    attempt,
+  });
+}
+
+function withoutId(base) {
+  const rest = {};
+  for (const key of Object.keys(base)) {
+    if (key !== 'id' && base[key] !== undefined) rest[key] = base[key];
+  }
+  return rest;
+}
+
+/** Keys of `info` a log line leaves out: who asked and why go to the event only */
+const EVENT_ONLY = new Set(['requestedBy', 'reason']);
+
+/**
+ * The one mapping from a context's correlation (`ctx.run`, or a background
+ * migration's `ctx.background`) to what a log line binds and what the event
+ * carries — so the three can never disagree. `id` becomes `runId`, a
+ * background `name` becomes `migration`; undefined values are left out.
+ */
+function correlationOf(kind, info, direction) {
+  const event = { kind };
+  const line = {};
+  const put = (key, value) => {
+    if (value === undefined) return;
+    event[key] = value;
+    if (!EVENT_ONLY.has(key)) line[key] = value;
+  };
+  if (kind === 'background') {
+    put('runId', info.runId);
+    put('migration', info.name);
+    put('direction', direction);
+    for (const key of Object.keys(info)) {
+      if (key !== 'runId' && key !== 'name') put(key, info[key]);
+    }
+  } else {
+    put('runId', info.id);
+    for (const key of Object.keys(info)) {
+      if (key !== 'id') put(key, info[key]);
+    }
+  }
+  return { event, line };
+}
+
+/** A log message as text: an Error by its (redacted) message, anything else stringified */
+function messageText(msg) {
+  if (typeof msg === 'string') return msg;
+  if (msg instanceof Error) return errorText(msg);
+  return msg === undefined ? '' : String(msg);
+}
+
+/**
+ * What a run's loggers have already said about themselves — one debug line
+ * each for a call dropped and a call cut to the event's bounds, per run (or
+ * lane slice), not per call: `{ dropped, truncated }`.
+ */
+function notices() {
+  return { dropped: false, truncated: false };
+}
+
+/**
+ * A logger bound to one context's correlation.
+ *
+ * - `sink` — the kit's resolved logger (already guarded, `null` config → silent);
+ * - `kind` — `'migration'` or `'background'`, and `info` — the frozen
+ *   `ctx.run` / `ctx.background` (`direction` given separately for background);
+ * - `emitter` — `{ wanted(), emit(payload) }`, absent where nothing may be
+ *   emitted (a dry run); `nextSeq` — the run's counter;
+ * - `noticed` — the run's {@link notices}: a dropped or a cut call leaves one
+ *   debug line, since an invisible one is undebuggable;
+ * - `dryRun` — marks every line, and emits nothing.
+ *
+ * Pino's own argument order — `(fields, msg)` — is accepted too.
+ */
+function createMigrationLogger({
+  sink,
+  kind,
+  info,
+  direction,
+  emitter,
+  nextSeq,
+  noticed = notices(),
+  dryRun = false,
+}) {
+  const { event, line } = correlationOf(kind, info, direction);
+  if (dryRun) line.dryRun = true;
+  /** The one debug line for `what` (a key of `noticed`) — the call that caused it goes on */
+  const notice = (what, text) => {
+    if (noticed[what]) return;
+    noticed[what] = true;
+    sink.debug(text, { ...line });
+  };
+  const write = (level) => (first, second) => {
+    // Logging must never break a migration: a getter that throws, a message
+    // that cannot be stringified — the call is dropped, the run goes on.
+    try {
+      let msg = first;
+      let fields = second;
+      if (isPlainObject(first) && (second === undefined || typeof second === 'string')) {
+        msg = second;
+        fields = first;
+      }
+      const text = messageText(msg);
+      const plain = isPlainObject(fields);
+      if (plain) sink[level](text, { ...fields, ...line });
+      else if (fields === undefined) sink[level](text, { ...line });
+      else sink[level](text, { ...line, value: fields });
+      if (!plain || fields[USERLAND] !== true || !emitter || dryRun || !emitter.wanted()) return;
+      const data = redactBounded(fields, { omit: USERLAND });
+      // The event leaves the process: the values a server error quotes (an
+      // E11000's duplicate key) are masked too, as everywhere text leaves it.
+      // The log line keeps them — it is what a developer debugs with.
+      const message = redactOutbound(text);
+      const clipped = message.length > MAX_MESSAGE_LENGTH;
+      const truncated = data.truncated || clipped;
+      emitter.emit({
+        ...event,
+        level,
+        msg: clipped ? `${message.slice(0, MAX_MESSAGE_LENGTH)}…` : message,
+        data: data.value,
+        at: new Date(),
+        seq: nextSeq(),
+        ...(truncated ? { truncated: true } : {}),
+      });
+      if (truncated) {
+        notice(
+          'truncated',
+          'A ctx.logger call was cut to the bounds of its migration:log event ' +
+            `(message ${MAX_MESSAGE_LENGTH} characters; data ${BOUNDS.depth} levels, ` +
+            `${BOUNDS.entries} entries, ${BOUNDS.string}-character strings)`,
+        );
+      }
+    } catch (error) {
+      // See above: dropped — but said once, at debug level.
+      try {
+        notice('dropped', `A ctx.logger call was dropped: ${errorText(error)}`);
+      } catch {
+        // Not even that: the thrown value cannot be described.
+      }
+    }
+  };
+  return { debug: write('debug'), info: write('info'), warn: write('warn'), error: write('error') };
+}
+
+/**
+ * The log side of one ordinary run, made when it takes its id — what its
+ * migrations' contexts and the kit's own lines bind:
+ *
+ * - `job` — the queue job it works for (`{ id, groupId? }`), a copy, as a run
+ *   it drives inline hands it on; `fields` — the same as log fields
+ *   (`{ jobId?, groupId? }`), on every line of the kit's for the run;
+ * - `info(direction)` — `ctx.run` of the run itself (`beforeAll`/`afterAll`),
+ *   made once per direction (`redo` has two);
+ * - `logger(info)` — `ctx.logger` bound to `info`;
+ * - `attempt(direction, { migration, batch, attempt })` — what one attempt
+ *   of one migration adds to its context: `{ run, logger }`.
+ *
+ * Every logger of the run shares its counter and its notices, so `seq` runs
+ * across the whole run and a dropped call is said once. A logger keeps them
+ * after the run ended (a timed-out body's late call is still numbered).
+ */
+function createRunLog({ id, job, actor = {}, sink, emitter }) {
+  const ref =
+    job === undefined
+      ? undefined
+      : { id: job.id, ...(job.groupId !== undefined ? { groupId: job.groupId } : {}) };
+  const fields = jobFields(ref);
+  const base = { id, ...fields, ...actor };
+  const nextSeq = sequence();
+  const noticed = notices();
+  const infos = new Map();
+  const info = (direction) => {
+    let run = infos.get(direction);
+    if (run === undefined) {
+      run = runInfo(base, direction);
+      infos.set(direction, run);
+    }
+    return run;
+  };
+  const logger = (bound) =>
+    createMigrationLogger({ sink, kind: 'migration', info: bound, emitter, nextSeq, noticed });
+  return {
+    job: ref,
+    fields,
+    info,
+    logger,
+    attempt(direction, { migration, batch, attempt }) {
+      const run = migrationRunInfo(info(direction), { migration, batch, attempt });
+      return { run, logger: logger(run) };
+    },
+  };
+}
+
+/**
+ * `logs(info, direction)` for one background lane slice, live watcher or dry
+ * run: a `ctx.logger` per `ctx.background`, all sharing one counter and one
+ * set of notices. A dry run's mark every line and emit nothing.
+ */
+function backgroundLogs({ sink, emitter, dryRun = false }) {
+  const nextSeq = sequence();
+  const noticed = notices();
+  return (info, direction) =>
+    createMigrationLogger({
+      sink,
+      kind: 'background',
+      info,
+      direction,
+      ...(dryRun ? {} : { emitter }),
+      nextSeq,
+      noticed,
+      dryRun,
+    });
+}
+
+module.exports = {
+  MAX_MESSAGE_LENGTH,
+  MIGRATION_LOG_EVENT,
+  USERLAND,
+  backgroundLogs,
+  correlationOf,
+  createMigrationLogger,
+  createRunLog,
+  migrationRunInfo,
+  notices,
+  runInfo,
+  sequence,
+};