@alexify/migronaut 2.3.0 → 2.4.0

package/src/core/background-kit.js

@@ -24,9 +24,10 @@ const { sleep } = require('./background-throttle.js');
  * and watchers. A flow like background.js: it gets what it needs from the
  * kit (`host`), built only by migrator.js:
  *
- * `{ store, deps(owner?), newId(), logger, emit(event, payload),
- *    registered(name), status(name) }` — `deps` is background.js's deps,
- * `registered` the state or NotAppliedError, `status` the public view.
+ * `{ store, deps(owner?, { job }?), newId(), logger, emit(event, payload),
+ *    registered(name), status(name) }` — `deps` is background.js's deps (`job`:
+ * the queue job a run driving it inline works for), `registered` the state or
+ * NotAppliedError, `status` the public view.
  */
 
 /**
@@ -69,10 +70,16 @@ const irreversibleBackground = (name) =>
  * it is done (`untilDone`) or for one round. `inline` (a run waiting for it
  * under the migration lock) cannot wait out a file that changed on disk
  * since it was registered — that wait is for a deploy in progress, and this
- * run is the deploy — so it fails instead.
+ * run is the deploy — so it fails instead. `job`: the queue job that run works
+ * for, named on every lane's lines and `migration:log` events.
  */
-async function drive(host, name, { signal, sliceMs, untilDone = true, concurrency = 1, inline }) {
-  const deps = host.deps(host.newId());
+async function drive(
+  host,
+  name,
+  { signal, sliceMs, untilDone = true, concurrency = 1, inline, job },
+) {
+  const jobOption = job ? { job } : {};
+  const deps = host.deps(host.newId(), jobOption);
   const stopped = () =>
     new RunAbortedError(`Stopped driving background migration ${name} — it goes on from here`, {
       migration: name,
@@ -87,7 +94,7 @@ async function drive(host, name, { signal, sliceMs, untilDone = true, concurrenc
     if (answer.next === 'process') {
       const state = await host.registered(name);
       const count = Math.max(1, Math.min(concurrency, state.spec?.maxParallel ?? 1));
-      await lanes(host, name, count, { signal, sliceMs, untilDone });
+      await lanes(host, name, count, { signal, sliceMs, untilDone, jobOption });
     } else {
       if (inline && answer.reason === 'checksum') {
         throw new ChecksumMismatchError(
@@ -112,13 +119,13 @@ async function drive(host, name, { signal, sliceMs, untilDone = true, concurrenc
 }
 
 /** `count` lanes at once: the first that fails for good stops the others, and its error is thrown */
-async function lanes(host, name, count, { signal, sliceMs, untilDone }) {
+async function lanes(host, name, count, { signal, sliceMs, untilDone, jobOption }) {
   const stop = new AbortController();
   const laneSignal = signal ? AbortSignal.any([signal, stop.signal]) : stop.signal;
   const running = [];
   for (let i = 0; i < count; i++) {
     running.push(
-      lane(host, name, { signal: laneSignal, sliceMs, untilDone }).catch((error) => {
+      lane(host, name, { signal: laneSignal, sliceMs, untilDone, jobOption }).catch((error) => {
         if (!stop.signal.aborted) stop.abort(error);
         throw error;
       }),
@@ -135,14 +142,14 @@ async function lanes(host, name, count, { signal, sliceMs, untilDone }) {
  * retry can fix — the file changed or is gone, the deployment cannot run it
  * — ends the lane, and so do `MAX_LANE_FAILURES` in a row.
  */
-async function lane(host, name, { signal, sliceMs, untilDone }) {
+async function lane(host, name, { signal, sliceMs, untilDone, jobOption = {} }) {
   let failures = 0;
   for (;;) {
     if (signal?.aborted) return;
     const owner = host.newId();
     let slice;
     try {
-      slice = await runSlice(host.deps(owner), name, { signal, sliceMs, owner });
+      slice = await runSlice(host.deps(owner, jobOption), name, { signal, sliceMs, owner });
       failures = 0;
     } catch (error) {
       if (signal?.aborted) return;

package/src/core/background-watch.js

@@ -518,17 +518,22 @@ function startWatch(deps, options = {}) {
   async function rewrite(job, doc) {
     if (!job.spec.transaction) return applyBatch(job, [doc], { db: deps.db });
     let current = doc;
+    // Each transaction — the driver's retries included — runs the
+    // transformation again; what it logs says which one it was.
+    let transactions = 0;
     for (let attempt = 0; ; attempt++) {
       const session = deps.client.startSession();
       try {
         let result;
         await session.withTransaction(async () => {
+          transactions += 1;
           result = await applyBatch(job, [current], {
             db: deps.db,
             session,
             ctxExtra: { session, db: deps.db, client: deps.client },
             abortOnConflict: true,
             strict: true,
+            attempt: transactions,
           });
         }, transactionOptions(job.spec));
         return result;

package/src/core/background.js

@@ -182,6 +182,12 @@ async function jobFor(deps, name, state, { direction } = {}) {
     match,
     ...(await partitionerFor(deps, spec, dir)),
     logger: deps.logger,
+    // For ctx.background and ctx.logger: the lane working it and its queue job
+    // (with its group when a run drives it inline).
+    ...(deps.logs ? { logs: deps.logs } : {}),
+    ...(deps.runId !== undefined ? { runId: deps.runId } : {}),
+    ...(deps.job?.id !== undefined ? { jobId: deps.job.id } : {}),
+    ...(deps.job?.groupId !== undefined ? { groupId: deps.job.groupId } : {}),
   };
 }
 

package/src/core/migration-logger.js

@@ -0,0 +1,279 @@
+const { isPlainObject } = require('../utils/canonical.js');
+const { errorText } = require('../utils/error.js');
+const { jobFields } = require('../utils/job-ref.js');
+const { BOUNDS, redactBounded, redactOutbound } = require('../utils/redact.js');
+
+/**
+ * The logger a migration gets as `ctx.logger`, and the `migration:log` event.
+ *
+ * Every call is a line on the kit's own logger with the run's correlation
+ * bound into its fields (run id, migration, direction, batch, attempt, job) —
+ * a JSON sink can join it to the kit's lines, the changelog and the queue job
+ * without parsing anything. A call whose fields say `userland: true` is also
+ * emitted as `migration:log`, for the application to keep: migronaut stores
+ * none of it. The marker is per call, so operational noise stays in the logs
+ * and only what the author meant for their users reaches the event.
+ *
+ * Mechanism only: no kit, no decisions about where lines go — `sink` is the
+ * kit's resolved logger, `emitter` its event channel.
+ */
+
+const MIGRATION_LOG_EVENT = 'migration:log';
+const USERLAND = 'userland';
+
+/** The longest message an event carries; the log line keeps the whole one */
+const MAX_MESSAGE_LENGTH = 2048;
+
+/** A counter, one per run (or lane slice): orders a run's events within one millisecond */
+function sequence() {
+  let next = 0;
+  return () => {
+    next += 1;
+    return next;
+  };
+}
+
+/**
+ * The run-level correlation of an ordinary run, frozen — `ctx.run` in
+ * `beforeAll`/`afterAll`, and what every migration's own adds to.
+ * `base` holds the run id and, when the caller gave them, the job and the
+ * actor: `{ id, jobId?, groupId?, requestedBy?, reason? }`.
+ */
+function runInfo(base, direction) {
+  return Object.freeze({ id: base.id, direction, ...withoutId(base) });
+}
+
+/** `ctx.run` of one attempt of one migration — a new frozen object per attempt */
+function migrationRunInfo(run, { migration, batch, attempt }) {
+  return Object.freeze({
+    ...run,
+    migration,
+    ...(batch !== undefined ? { batch } : {}),
+    attempt,
+  });
+}
+
+function withoutId(base) {
+  const rest = {};
+  for (const key of Object.keys(base)) {
+    if (key !== 'id' && base[key] !== undefined) rest[key] = base[key];
+  }
+  return rest;
+}
+
+/** Keys of `info` a log line leaves out: who asked and why go to the event only */
+const EVENT_ONLY = new Set(['requestedBy', 'reason']);
+
+/**
+ * The one mapping from a context's correlation (`ctx.run`, or a background
+ * migration's `ctx.background`) to what a log line binds and what the event
+ * carries — so the three can never disagree. `id` becomes `runId`, a
+ * background `name` becomes `migration`; undefined values are left out.
+ */
+function correlationOf(kind, info, direction) {
+  const event = { kind };
+  const line = {};
+  const put = (key, value) => {
+    if (value === undefined) return;
+    event[key] = value;
+    if (!EVENT_ONLY.has(key)) line[key] = value;
+  };
+  if (kind === 'background') {
+    put('runId', info.runId);
+    put('migration', info.name);
+    put('direction', direction);
+    for (const key of Object.keys(info)) {
+      if (key !== 'runId' && key !== 'name') put(key, info[key]);
+    }
+  } else {
+    put('runId', info.id);
+    for (const key of Object.keys(info)) {
+      if (key !== 'id') put(key, info[key]);
+    }
+  }
+  return { event, line };
+}
+
+/** A log message as text: an Error by its (redacted) message, anything else stringified */
+function messageText(msg) {
+  if (typeof msg === 'string') return msg;
+  if (msg instanceof Error) return errorText(msg);
+  return msg === undefined ? '' : String(msg);
+}
+
+/**
+ * What a run's loggers have already said about themselves — one debug line
+ * each for a call dropped and a call cut to the event's bounds, per run (or
+ * lane slice), not per call: `{ dropped, truncated }`.
+ */
+function notices() {
+  return { dropped: false, truncated: false };
+}
+
+/**
+ * A logger bound to one context's correlation.
+ *
+ * - `sink` — the kit's resolved logger (already guarded, `null` config → silent);
+ * - `kind` — `'migration'` or `'background'`, and `info` — the frozen
+ *   `ctx.run` / `ctx.background` (`direction` given separately for background);
+ * - `emitter` — `{ wanted(), emit(payload) }`, absent where nothing may be
+ *   emitted (a dry run); `nextSeq` — the run's counter;
+ * - `noticed` — the run's {@link notices}: a dropped or a cut call leaves one
+ *   debug line, since an invisible one is undebuggable;
+ * - `dryRun` — marks every line, and emits nothing.
+ *
+ * Pino's own argument order — `(fields, msg)` — is accepted too.
+ */
+function createMigrationLogger({
+  sink,
+  kind,
+  info,
+  direction,
+  emitter,
+  nextSeq,
+  noticed = notices(),
+  dryRun = false,
+}) {
+  const { event, line } = correlationOf(kind, info, direction);
+  if (dryRun) line.dryRun = true;
+  /** The one debug line for `what` (a key of `noticed`) — the call that caused it goes on */
+  const notice = (what, text) => {
+    if (noticed[what]) return;
+    noticed[what] = true;
+    sink.debug(text, { ...line });
+  };
+  const write = (level) => (first, second) => {
+    // Logging must never break a migration: a getter that throws, a message
+    // that cannot be stringified — the call is dropped, the run goes on.
+    try {
+      let msg = first;
+      let fields = second;
+      if (isPlainObject(first) && (second === undefined || typeof second === 'string')) {
+        msg = second;
+        fields = first;
+      }
+      const text = messageText(msg);
+      const plain = isPlainObject(fields);
+      if (plain) sink[level](text, { ...fields, ...line });
+      else if (fields === undefined) sink[level](text, { ...line });
+      else sink[level](text, { ...line, value: fields });
+      if (!plain || fields[USERLAND] !== true || !emitter || dryRun || !emitter.wanted()) return;
+      const data = redactBounded(fields, { omit: USERLAND });
+      // The event leaves the process: the values a server error quotes (an
+      // E11000's duplicate key) are masked too, as everywhere text leaves it.
+      // The log line keeps them — it is what a developer debugs with.
+      const message = redactOutbound(text);
+      const clipped = message.length > MAX_MESSAGE_LENGTH;
+      const truncated = data.truncated || clipped;
+      emitter.emit({
+        ...event,
+        level,
+        msg: clipped ? `${message.slice(0, MAX_MESSAGE_LENGTH)}…` : message,
+        data: data.value,
+        at: new Date(),
+        seq: nextSeq(),
+        ...(truncated ? { truncated: true } : {}),
+      });
+      if (truncated) {
+        notice(
+          'truncated',
+          'A ctx.logger call was cut to the bounds of its migration:log event ' +
+            `(message ${MAX_MESSAGE_LENGTH} characters; data ${BOUNDS.depth} levels, ` +
+            `${BOUNDS.entries} entries, ${BOUNDS.string}-character strings)`,
+        );
+      }
+    } catch (error) {
+      // See above: dropped — but said once, at debug level.
+      try {
+        notice('dropped', `A ctx.logger call was dropped: ${errorText(error)}`);
+      } catch {
+        // Not even that: the thrown value cannot be described.
+      }
+    }
+  };
+  return { debug: write('debug'), info: write('info'), warn: write('warn'), error: write('error') };
+}
+
+/**
+ * The log side of one ordinary run, made when it takes its id — what its
+ * migrations' contexts and the kit's own lines bind:
+ *
+ * - `job` — the queue job it works for (`{ id, groupId? }`), a copy, as a run
+ *   it drives inline hands it on; `fields` — the same as log fields
+ *   (`{ jobId?, groupId? }`), on every line of the kit's for the run;
+ * - `info(direction)` — `ctx.run` of the run itself (`beforeAll`/`afterAll`),
+ *   made once per direction (`redo` has two);
+ * - `logger(info)` — `ctx.logger` bound to `info`;
+ * - `attempt(direction, { migration, batch, attempt })` — what one attempt
+ *   of one migration adds to its context: `{ run, logger }`.
+ *
+ * Every logger of the run shares its counter and its notices, so `seq` runs
+ * across the whole run and a dropped call is said once. A logger keeps them
+ * after the run ended (a timed-out body's late call is still numbered).
+ */
+function createRunLog({ id, job, actor = {}, sink, emitter }) {
+  const ref =
+    job === undefined
+      ? undefined
+      : { id: job.id, ...(job.groupId !== undefined ? { groupId: job.groupId } : {}) };
+  const fields = jobFields(ref);
+  const base = { id, ...fields, ...actor };
+  const nextSeq = sequence();
+  const noticed = notices();
+  const infos = new Map();
+  const info = (direction) => {
+    let run = infos.get(direction);
+    if (run === undefined) {
+      run = runInfo(base, direction);
+      infos.set(direction, run);
+    }
+    return run;
+  };
+  const logger = (bound) =>
+    createMigrationLogger({ sink, kind: 'migration', info: bound, emitter, nextSeq, noticed });
+  return {
+    job: ref,
+    fields,
+    info,
+    logger,
+    attempt(direction, { migration, batch, attempt }) {
+      const run = migrationRunInfo(info(direction), { migration, batch, attempt });
+      return { run, logger: logger(run) };
+    },
+  };
+}
+
+/**
+ * `logs(info, direction)` for one background lane slice, live watcher or dry
+ * run: a `ctx.logger` per `ctx.background`, all sharing one counter and one
+ * set of notices. A dry run's mark every line and emit nothing.
+ */
+function backgroundLogs({ sink, emitter, dryRun = false }) {
+  const nextSeq = sequence();
+  const noticed = notices();
+  return (info, direction) =>
+    createMigrationLogger({
+      sink,
+      kind: 'background',
+      info,
+      direction,
+      ...(dryRun ? {} : { emitter }),
+      nextSeq,
+      noticed,
+      dryRun,
+    });
+}
+
+module.exports = {
+  MAX_MESSAGE_LENGTH,
+  MIGRATION_LOG_EVENT,
+  USERLAND,
+  backgroundLogs,
+  correlationOf,
+  createMigrationLogger,
+  createRunLog,
+  migrationRunInfo,
+  notices,
+  runInfo,
+  sequence,
+};