@alexify/migronaut 2.3.0 → 2.4.0

package/src/bullmq/background-processor.js

@@ -4,9 +4,14 @@ const { errorText } = require('../utils/error.js');
 const { redactOutbound } = require('../utils/redact.js');
 const { JOB_NAMES, buildLaneJob, isObjectLike, parseBackgroundJobData } = require('./jobs.js');
 const {
+  DEFAULT_USERLAND_LOG_ROWS,
   UNRECOVERABLE_ERROR_NAME,
+  assertUserlandLogRows,
   isRetryableError,
+  jobRefOf,
   prepareErrorForQueue,
+  userlandOverflowRow,
+  userlandRow,
 } = require('./processor.js');
 const {
   DEFAULT_STALL_MS,
@@ -75,6 +80,7 @@ const PROCESSOR_KEYS = new Set([
   'pollIntervalMs',
   'stallMs',
   'maxLaneRetries',
+  'userlandLogRows',
 ]);
 
 function resolveBackgroundProcessorOptions(options) {
@@ -96,6 +102,7 @@ function resolveBackgroundProcessorOptions(options) {
     pollIntervalMs = DEFAULTS.pollIntervalMs,
     stallMs = DEFAULT_STALL_MS,
     maxLaneRetries = DEFAULTS.maxLaneRetries,
+    userlandLogRows = DEFAULT_USERLAND_LOG_ROWS,
   } = options;
   if (kit !== undefined && config !== undefined) {
     throw new ConfigInvalidError('Pass either `kit` or `config`, not both');
@@ -132,7 +139,8 @@ function resolveBackgroundProcessorOptions(options) {
     });
   }
   assertBackgroundJobOptions(jobOptions);
-  return { sliceMs, children, pollIntervalMs, stallMs, maxLaneRetries };
+  assertUserlandLogRows(userlandLogRows);
+  return { sliceMs, children, pollIntervalMs, stallMs, maxLaneRetries, userlandLogRows };
 }
 
 /**
@@ -147,6 +155,13 @@ function createBackgroundProcessor(options = {}) {
   const shutdownController = new AbortController();
   const inFlight = new Set();
   let warnedUnmovable = false;
+  /**
+   * The lanes working a slice in this process, by job id — lanes run side by
+   * side, so a `migration:log` event's `jobId` is what says whose log it goes
+   * to. `writes` are its rows still in flight, drained when the slice ends;
+   * `rows`/`dropped` count them against `userlandLogRows`, per slice.
+   */
+  const lanes = new Map();
 
   /** A log row on the job — never allowed to fail it */
   async function log(job, row) {
@@ -190,6 +205,59 @@ function createBackgroundProcessor(options = {}) {
     throw moved(DELAYED_ERROR_NAME);
   }
 
+  /** A row on a lane's job, tracked so the slice's end can wait for it */
+  function laneLog(lane, row) {
+    const pending = log(lane.job, row);
+    lane.writes.add(pending);
+    pending.finally(() => lane.writes.delete(pending));
+  }
+
+  /**
+   * A lane's userland lines, into its own job's log — never allowed to fail
+   * it. A line with a group is a lane a migration job drives inline: its job
+   * is on the other queue, whatever its id.
+   */
+  function onUserland(event) {
+    if (event?.kind !== 'background' || event.jobId === undefined) return;
+    if (event.groupId !== undefined) return;
+    const lane = lanes.get(event.jobId);
+    if (lane === undefined) return;
+    if (lane.rows < settings.userlandLogRows) {
+      lane.rows += 1;
+      laneLog(lane, userlandRow(event));
+    } else {
+      lane.dropped += 1;
+    }
+  }
+  if (typeof kit.on === 'function') kit.on('migration:log', onUserland);
+
+  /**
+   * Run one slice as `job`'s: the slice gets `{ id }` for its lines and
+   * events, and its userland lines reach the job's log until the slice ends —
+   * drained before the job moves on, so none lands after it.
+   */
+  async function asLane(job, fn) {
+    const ref = jobRefOf(job);
+    if (ref === undefined) {
+      kit.logger.debug(
+        `Lane job ${job?.id} has no id a slice can carry — its lines name no job`,
+        {},
+      );
+      return fn(undefined);
+    }
+    const lane = { job, writes: new Set(), rows: 0, dropped: 0 };
+    lanes.set(ref.id, lane);
+    try {
+      return await fn(ref);
+    } finally {
+      if (lanes.get(ref.id) === lane) lanes.delete(ref.id);
+      if (lane.dropped > 0) {
+        laneLog(lane, userlandOverflowRow(lane.dropped, settings.userlandLogRows));
+      }
+      await Promise.allSettled([...lane.writes]);
+    }
+  }
+
   /** Heal from MongoDB: a coordinator for every background migration with work to do */
   async function heal(reason) {
     try {
@@ -327,10 +395,13 @@ function createBackgroundProcessor(options = {}) {
     if (shutdownController.signal.aborted) return later(ctx, 0, { ...base, outcome: 'stopped' });
     let slice;
     try {
-      slice = await kit.runBackgroundSlice(name, {
-        signal: ctx.abort,
-        ...(settings.sliceMs !== undefined ? { sliceMs: settings.sliceMs } : {}),
-      });
+      slice = await asLane(job, (ref) =>
+        kit.runBackgroundSlice(name, {
+          signal: ctx.abort,
+          ...(settings.sliceMs !== undefined ? { sliceMs: settings.sliceMs } : {}),
+          ...(ref ? { job: ref } : {}),
+        }),
+      );
     } catch (error) {
       if (shutdownController.signal.aborted) {
         return later(ctx, 0, { ...base, outcome: 'stopped' });
@@ -451,6 +522,7 @@ function createBackgroundProcessor(options = {}) {
   processor.close = async () => {
     processor.shutdown();
     await Promise.allSettled([...inFlight]);
+    if (typeof kit.off === 'function') kit.off('migration:log', onUserland);
     if (ownsKit) await kit.disconnect();
   };
 

package/src/bullmq/processor.js

@@ -9,8 +9,11 @@ const {
   RunAbortedError,
 } = require('../errors/index.js');
 const { pickActor } = require('../utils/actor.js');
+const { isPlainObject } = require('../utils/canonical.js');
 const { errorText } = require('../utils/error.js');
+const { jobRefIssue } = require('../utils/job-ref.js');
 const { redactDeep, redactOutbound } = require('../utils/redact.js');
+const { sanitize } = require('../utils/sanitize.js');
 const { JOB_NAMES, assertAllowed, isObjectLike, parseJobData, resolveAllow } = require('./jobs.js');
 const {
   assertBackgroundJobOptions,
@@ -79,14 +82,162 @@ function jobIds(ctx) {
   };
 }
 
+/**
+ * The job a run works for, as the kit's `job` option — `{ id, groupId? }` (a
+ * lane's slice: `{ id }`), or undefined when the job has no id the kit would
+ * take (it then logs nothing of the job; the run itself is unaffected). A
+ * group the kit would refuse is left out rather than the whole reference.
+ */
+function jobRefOf(job, groupId) {
+  if (job?.id === undefined || job.id === null) return undefined;
+  const id = String(job.id);
+  const ref = groupId !== undefined ? { id, groupId } : { id };
+  if (jobRefIssue(ref) === null) return ref;
+  return jobRefIssue({ id }) === null ? { id } : undefined;
+}
+
+/** The longest row a `migration:log` event becomes in a job's log */
+const USERLAND_ROW_MAX = 1024;
+
+/**
+ * How many `migration:log` rows one job's log takes by default (a lane: one
+ * slice) — the rest are counted in one closing row. A migration that logs in
+ * a loop must not write tens of thousands of entries into Redis per job.
+ */
+const DEFAULT_USERLAND_LOG_ROWS = 1000;
+
+/** Line breaks: a row is one line of the job's log, so they are shown, never acted on */
+const ROW_BREAKS = /\r\n|[\r\n\u2028\u2029]/g;
+
+/** Text as part of one row: line breaks shown as ⏎, control characters dropped */
+const oneLine = (text) => sanitize(String(text).replace(ROW_BREAKS, '⏎'));
+
+/** BigInts have no JSON form of their own; a log row shows their digits */
+const jsonValue = (_key, value) => (typeof value === 'bigint' ? value.toString() : value);
+
+/** Stops {@link boundedJson} once the row has what it can show */
+const ROW_FULL = Symbol('row full');
+
+/**
+ * `data` as JSON, written only until `max` characters — never the whole of
+ * it: an event's data may hold a thousand entries of 4 KB each, and the row
+ * shows a kilobyte. The JSON of what `JSON.stringify` would write, cut short.
+ */
+function boundedJson(data, max) {
+  let out = '';
+  const push = (text) => {
+    if (out.length + text.length > max) {
+      out += text.slice(0, max - out.length + 1);
+      throw ROW_FULL;
+    }
+    out += text;
+  };
+  const write = (value) => {
+    if (typeof value === 'bigint') return push(`"${value}"`);
+    // Arrays and plain objects are walked here; anything else is a leaf.
+    if (!Array.isArray(value) && !isPlainObject(value)) {
+      return push(JSON.stringify(value, jsonValue) ?? 'null');
+    }
+    if (Array.isArray(value)) {
+      push('[');
+      for (let index = 0; index < value.length; index++) {
+        if (index > 0) push(',');
+        write(value[index]);
+      }
+      return push(']');
+    }
+    push('{');
+    let first = true;
+    for (const key of Object.keys(value)) {
+      const item = value[key];
+      if (item === undefined || typeof item === 'function' || typeof item === 'symbol') continue;
+      push(`${first ? '' : ','}${JSON.stringify(key)}:`);
+      first = false;
+      write(item);
+    }
+    return push('}');
+  };
+  try {
+    write(data);
+  } catch (error) {
+    if (error !== ROW_FULL) throw error;
+  }
+  return out;
+}
+
+/** Whether `data` has a key to show — without listing them all */
+function hasEntries(data) {
+  if (data === null || typeof data !== 'object') return false;
+  for (const _key in data) return true;
+  return false;
+}
+
+/**
+ * A `migration:log` event as a row of the job's log, next to the processor's
+ * own lifecycle rows: `✎ <level: ><msg> <data as JSON>`, the attempt when a
+ * transaction was retried, the partition of a background lane. One line —
+ * a line break in the message cannot forge a row of its own — cut at
+ * {@link USERLAND_ROW_MAX}; redacted on its way out like every row.
+ */
+function userlandRow(event) {
+  const level = event.level === 'info' ? '' : `${event.level}: `;
+  const attempt = event.attempt > 1 ? ` (attempt ${event.attempt})` : '';
+  const partition =
+    event.kind === 'background' && event.partition ? ` [partition ${event.partition}]` : '';
+  const head = `✎ ${level}${event.msg}`;
+  let data = '';
+  if (hasEntries(event.data) && head.length < USERLAND_ROW_MAX) {
+    try {
+      data = ` ${boundedJson(event.data, USERLAND_ROW_MAX - head.length)}`;
+    } catch {
+      data = ' [data not serializable]';
+    }
+  }
+  const row = oneLine(`${head}${data}${attempt}${partition}`);
+  return row.length > USERLAND_ROW_MAX ? `${row.slice(0, USERLAND_ROW_MAX - 1)}…` : row;
+}
+
+/** The closing row of a job whose userland lines went past the limit */
+function userlandOverflowRow(dropped, limit) {
+  return `✎ … ${dropped} more line(s) past the limit of ${limit} not mirrored here — see migration:log`;
+}
+
+/** `userlandLogRows`: how many userland rows one job's log takes */
+function assertUserlandLogRows(value) {
+  if (value !== undefined && (!Number.isSafeInteger(value) || value < 0)) {
+    throw new ConfigInvalidError('userlandLogRows must be an integer of at least 0', {
+      userlandLogRows: value,
+    });
+  }
+}
+
+/**
+ * Whether a `migration:log` event is the job's own. Matched by run and job,
+ * not by the job in flight alone: a body that outlived its timeout may still
+ * log while the next job runs — the same job again, if it was put back in the
+ * queue — and its lines must not land in that run's log. A background
+ * migration the run drives inline logs from lanes of its own (their own run
+ * ids), named by the job and its group: a background queue's lanes have no
+ * group, and their ids may well repeat a migration job's.
+ */
+function ownLine(ctx, event) {
+  const ref = ctx.jobRef;
+  if (event.kind === 'background') {
+    return ref?.groupId !== undefined && event.jobId === ref.id && event.groupId === ref.groupId;
+  }
+  if (event.kind !== 'migration') return false;
+  if (ctx.runId === undefined || event.runId !== ctx.runId) return false;
+  return ref === undefined || event.jobId === ref.id;
+}
+
 /** A job in a few words, for log lines: `up 20260101-x.js (1/3)`, `converge`, `sync` */
 function describeJob(data) {
   if (data.kind !== 'migration') return data.kind;
   return `${data.direction} ${data.migration} (${data.index + 1}/${data.total})`;
 }
 
-/** The structured fields of a job's log lines */
-function jobFields(data) {
+/** The structured fields of a job's log lines: what the job is about */
+function jobSummary(data) {
   if (data.kind !== 'migration') return { kind: data.kind };
   return {
     migration: data.migration,
@@ -139,7 +290,16 @@ function resolveProcessorOptions(options) {
   if (!isObjectLike(options)) {
     throw new ConfigInvalidError('createMigrationProcessor options must be an object');
   }
-  const { kit, config, lockWait = {}, jobOptions, ordered = true, allow, background } = options;
+  const {
+    kit,
+    config,
+    lockWait = {},
+    jobOptions,
+    ordered = true,
+    allow,
+    background,
+    userlandLogRows = DEFAULT_USERLAND_LOG_ROWS,
+  } = options;
   if (kit !== undefined && config !== undefined) {
     throw new ConfigInvalidError('Pass either `kit` or `config`, not both');
   }
@@ -158,7 +318,8 @@ function resolveProcessorOptions(options) {
   assertLockWaitOptions(waitOptions);
   assertJobOptions(jobOptions);
   if (background !== undefined) assertBackgroundLink(background);
-  return { waitOptions, defaultOrdered: ordered, allow: resolveAllow(allow) };
+  assertUserlandLogRows(userlandLogRows);
+  return { waitOptions, defaultOrdered: ordered, allow: resolveAllow(allow), userlandLogRows };
 }
 
 /**
@@ -189,7 +350,7 @@ function assertBackgroundLink(background) {
  * signal only to processors whose `length` is at least 3.
  */
 function createMigrationProcessor(options = {}) {
-  const { waitOptions, defaultOrdered, allow } = resolveProcessorOptions(options);
+  const { waitOptions, defaultOrdered, allow, userlandLogRows } = resolveProcessorOptions(options);
   const { kit: injectedKit, config, kitOptions, queue, jobOptions, background } = options;
 
   const ownsKit = injectedKit === undefined;
@@ -235,6 +396,29 @@ function createMigrationProcessor(options = {}) {
     );
   const flush = (ctx) => Promise.allSettled([...ctx.writes]);
 
+  /** A userland line into the job's log — up to `userlandLogRows`, then only counted */
+  const mirror = (ctx, event) => {
+    if (ctx.userlandRows < userlandLogRows) {
+      ctx.userlandRows += 1;
+      log(ctx, userlandRow(event));
+    } else {
+      ctx.userlandDropped += 1;
+    }
+  };
+
+  /**
+   * Nothing more is written for the migration once the job settles: a row
+   * after BullMQ removed the job would leave its log behind in Redis. What
+   * went past the limit is said in one last row first.
+   */
+  const seal = (ctx) => {
+    if (ctx.sealed) return;
+    if (ctx.userlandDropped > 0) {
+      log(ctx, userlandOverflowRow(ctx.userlandDropped, userlandLogRows));
+    }
+    ctx.sealed = true;
+  };
+
   // Subscribed once, for the processor's lifetime: the kit emits per run, and
   // `current` says which job that run belongs to.
   const listeners = {
@@ -305,6 +489,10 @@ function createMigrationProcessor(options = {}) {
     'converge:end': (event) => {
       if (current && event.success) log(current, `✔ Converged ${event.changed} change(s)`);
     },
+    // What the migration itself logged for its users — see ownLine.
+    'migration:log': (event) => {
+      if (current && !current.sealed && ownLine(current, event)) mirror(current, event);
+    },
   };
   for (const [event, listener] of Object.entries(listeners)) kit.on(event, listener);
 
@@ -373,6 +561,15 @@ function createMigrationProcessor(options = {}) {
   async function runMigrationJob(ctx, signal) {
     const { data } = ctx;
     const ordered = data.ordered ?? defaultOrdered;
+    // The run's correlation names this job, so what the migration logs can be
+    // joined to it (and its userland lines routed into its log).
+    ctx.jobRef = jobRefOf(ctx.job, data.groupId);
+    if (ctx.jobRef === undefined) {
+      kit.logger.debug(`Job ${ctx.job?.id} has no id the run can carry — its lines name no job`, {
+        ...jobIds(ctx),
+      });
+    }
+    const jobField = ctx.jobRef ? { job: ctx.jobRef } : {};
     const attempt = () =>
       data.direction === JOB_NAMES.UP
         ? kit.up(data.migration, {
@@ -381,8 +578,13 @@ function createMigrationProcessor(options = {}) {
             ...(data.force ? { force: true } : {}),
             ...(data.checksum ? { checksum: data.checksum } : {}),
             ...pickActor(data),
+            ...jobField,
           })
-        : kit.down(data.migration, { ...(ordered ? { ordered: true } : {}), ...pickActor(data) });
+        : kit.down(data.migration, {
+            ...(ordered ? { ordered: true } : {}),
+            ...pickActor(data),
+            ...jobField,
+          });
 
     try {
       const { result, waitedMs } = await waitForLock(ctx, attempt, signal);
@@ -636,6 +838,9 @@ function createMigrationProcessor(options = {}) {
       started: false,
       writes: new Set(),
       registered: [],
+      userlandRows: 0,
+      userlandDropped: 0,
+      sealed: false,
     };
     const startedAt = Date.now();
     const signals = [shutdownController.signal];
@@ -658,22 +863,24 @@ function createMigrationProcessor(options = {}) {
       abort.addEventListener('abort', onAbort, { once: true });
       kit.logger.debug(`▶ Job ${job?.id} (${describeJob(ctx.data)})`, {
         ...jobIds(ctx),
-        ...jobFields(ctx.data),
+        ...jobSummary(ctx.data),
       });
       await kit.connect();
       let result;
       if (ctx.data.kind === 'sync') result = await runSyncJob(ctx);
       else if (ctx.data.kind === 'converge') result = await runConvergeJob(ctx, abort);
       else result = await runMigrationJob(ctx, abort);
+      seal(ctx);
       progress(ctx, 'completed', ctx.runId ? { runId: ctx.runId } : {});
       await flush(ctx);
       kit.logger.debug(`✔ Job ${job?.id} done`, {
         ...jobIds(ctx),
-        ...jobFields(ctx.data),
+        ...jobSummary(ctx.data),
         durationMs: Date.now() - startedAt,
       });
       return result;
     } catch (error) {
+      seal(ctx);
       const requeued = await requeueOnShutdown(ctx, error, token);
       if (requeued) throw requeued;
       // BullMQ only retries when the job was given more than one attempt — the
@@ -738,9 +945,15 @@ module.exports = {
   RETRYABLE_CODES,
   UNRECOVERABLE_ERROR_NAME,
   WAITING_ERROR_NAME,
+  DEFAULT_USERLAND_LOG_ROWS,
+  assertUserlandLogRows,
   createMigrationProcessor,
   isTransientForJob,
   isRetryableError,
+  jobRefOf,
+  ownLine,
   prepareErrorForQueue,
   resolveProcessorOptions,
+  userlandOverflowRow,
+  userlandRow,
 };

package/src/bullmq/service.js

@@ -222,6 +222,7 @@ class MigrationQueue {
       lockWait,
       allow,
       background,
+      userlandLogRows,
     } = options;
 
     if (!isObjectLike(bullmq)) {
@@ -307,6 +308,7 @@ class MigrationQueue {
       ...(config !== undefined ? { config } : {}),
       ...(lockWait !== undefined ? { lockWait } : {}),
       ...(allow !== undefined ? { allow } : {}),
+      ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
     });
     this.#allow = resolveAllow(allow);
     const backgroundSettings = resolveBackground(background, {
@@ -367,6 +369,7 @@ class MigrationQueue {
         kit: this.#kit,
         queue: this.#backgroundQueue,
         ...MigrationQueue.#backgroundProcessorOptions(backgroundSettings),
+        ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
       });
     }
     this.#processor = createMigrationProcessor({
@@ -377,6 +380,7 @@ class MigrationQueue {
       ...(lockWait !== undefined ? { lockWait } : {}),
       ...(jobOptions !== undefined ? { jobOptions } : {}),
       ...(allow !== undefined ? { allow } : {}),
+      ...(userlandLogRows !== undefined ? { userlandLogRows } : {}),
       // What an `up` registers starts on the background queue at once.
       ...(this.#backgroundQueue !== undefined
         ? {

package/src/core/background-dry-run.js

@@ -13,6 +13,7 @@ const {
 const { idRangePartitioner } = require('./background-partition.js');
 const { runSandbox } = require('./background-sandbox.js');
 const { toRelaxedEjson } = require('./bson-peer.js');
+const { backgroundLogs } = require('./migration-logger.js');
 const { READ_OPTIONS } = require('./server-info.js');
 
 /**
@@ -52,6 +53,12 @@ function sampleSize({ sample, first }) {
   return n;
 }
 
+/**
+ * `ctx.logger` in a dry run: the lines say `dryRun: true`, and nothing is
+ * emitted — a preview's logs are not the application's to keep.
+ */
+const dryLogs = (logger) => backgroundLogs({ sink: logger, dryRun: true });
+
 /** The job a dry run works with — no partition, no lease */
 function dryJob(name, loaded, direction, logger) {
   return {
@@ -64,6 +71,7 @@ function dryJob(name, loaded, direction, logger) {
     partitionId: 'dry-run',
     match: matchOf(loaded.spec, direction),
     logger,
+    logs: dryLogs(logger),
   };
 }
 
@@ -301,6 +309,7 @@ async function previewSteps(deps, name, loaded, options = {}) {
     generation: 0,
     partitionId: 'dry-run',
     logger: deps.logger,
+    logs: dryLogs(deps.logger),
   };
   const log = [];
   let stoppedBy = 'steps';

package/src/core/background-engine.js

@@ -70,17 +70,38 @@ function excludeBadIds(match, badIds = []) {
   return badIds.length > 0 ? { $and: [match, { _id: { $nin: badIds } }] } : match;
 }
 
-/** What a transformation sees besides the document */
-function transformContext(job, extra = {}) {
+/**
+ * `ctx.background`: which background migration, generation and partition —
+ * and, for `ctx.logger` and `migration:log`, the lane (`runId`, its owner),
+ * the queue job working it (and its group, when a run drives it inline) and
+ * the transaction attempt. Frozen, like an ordinary migration's `ctx.run`.
+ */
+function backgroundInfo(job, attempt) {
+  return Object.freeze({
+    name: job.name,
+    generation: job.generation,
+    partition: String(job.partitionId ?? ''),
+    ...(job.runId !== undefined ? { runId: job.runId } : {}),
+    ...(job.jobId !== undefined ? { jobId: job.jobId } : {}),
+    ...(job.groupId !== undefined ? { groupId: job.groupId } : {}),
+    attempt,
+  });
+}
+
+/**
+ * What a transformation sees besides the document. `attempt` counts the
+ * transactions a transactional batch or step went through — each one runs the
+ * user's code again — and is 1 everywhere else. `job.logs` binds a logger to
+ * `ctx.background`; a job without it (a test's) gets the plain logger.
+ */
+function transformContext(job, extra = {}, attempt = 1) {
+  const direction = job.direction ?? 'forward';
+  const background = backgroundInfo(job, attempt);
   return {
     signal: job.signal,
-    logger: job.logger,
-    direction: job.direction ?? 'forward',
-    background: {
-      name: job.name,
-      generation: job.generation,
-      partition: String(job.partitionId ?? ''),
-    },
+    logger: job.logs ? job.logs(background, direction) : job.logger,
+    direction,
+    background,
     ...extra,
   };
 }
@@ -228,14 +249,14 @@ function byIds(docs, partitioner) {
 async function applyBatch(
   job,
   docs,
-  { db, session, ctxExtra, abortOnConflict = false, strict = false, bare = false } = {},
+  { db, session, ctxExtra, abortOnConflict = false, strict = false, bare = false, attempt } = {},
 ) {
   const spec = job.spec;
   const collection = db.collection(spec.collection);
   const { source, target } = directionOf(spec, job.fns, job.direction);
   // `left`: documents read and not rewritten — a draining partition steps over them.
   const counts = { migrated: 0, skipped: 0, conflicts: 0, retried: 0, errors: [], left: [] };
-  const ctx = transformContext(job, ctxExtra);
+  const ctx = transformContext(job, ctxExtra, attempt);
   let pending = docs;
   for (let round = 0; pending.length > 0; round++) {
     const transformed = await transformAll(job, pending, ctx);
@@ -338,9 +359,12 @@ function documentError(entry) {
  * (and, in a transaction or a dry run, the session), its last checkpoint,
  * and the deadline it should return by.
  */
-function buildStepContext(job, { db, client, session, checkpoint, deadline, dryRun } = {}) {
+function buildStepContext(
+  job,
+  { db, client, session, checkpoint, deadline, dryRun, attempt } = {},
+) {
   return {
-    ...transformContext(job),
+    ...transformContext(job, {}, attempt),
     db,
     client,
     checkpoint: checkpoint ?? null,
@@ -427,20 +451,22 @@ async function runStep(job, ctx, cursor) {
     );
     return { next, counts };
   };
-  const stepContext = (session) =>
+  const stepContext = (session, attempt) =>
     buildStepContext(job, {
       db,
       client,
       checkpoint: cursor.checkpoint,
       deadline: ctx.deadline,
+      attempt,
       ...(session ? { session } : {}),
     });
-  if (!job.spec.transaction) return save(readStepResult(await fn(stepContext())));
+  if (!job.spec.transaction) return save(readStepResult(await fn(stepContext(undefined, 1))));
   for (let attempt = 0; ; attempt++) {
     const session = client.startSession();
     try {
       session.startTransaction(transactionOptions(job.spec));
-      const saved = await save(readStepResult(await fn(stepContext(session))), session);
+      const result = await fn(stepContext(session, attempt + 1));
+      const saved = await save(readStepResult(result), session);
       await commit(session);
       lease.touch();
       return saved;
@@ -602,9 +628,13 @@ async function transactionalBatch(job, ctx, cursor, batchSize) {
   let size = Math.min(ctx.txn.size, batchSize);
   let attempts = 0;
   let txnRetries = 0;
+  // Every turn of the loop is a transaction of its own that runs the
+  // transformations again: what they log says which one it was.
+  let transactions = 0;
   const excluded = new Map();
   for (;;) {
     if (signal?.aborted) throw signal.reason;
+    transactions += 1;
     const session = client.startSession();
     let docs = [];
     try {
@@ -628,6 +658,7 @@ async function transactionalBatch(job, ctx, cursor, batchSize) {
               ctxExtra: { session, db, client },
               abortOnConflict: true,
               strict: true,
+              attempt: transactions,
             })
           : { migrated: 0, skipped: 0, conflicts: 0, retried: 0, errors: [], left: [] };
       const errors = [...excluded.values()];