@ultimat3/jobs 29.4.0 → 30.1.0

package/src/events-pg.ts

@@ -10,6 +10,7 @@ import type { PgExecutor } from '@ultimat3/core';
 import { finiteOption, logger, uuidV7 } from '@ultimat3/core';
 import type { DurationInput } from './clock';
 import { finiteDurationMs } from './clock';
+import { SQL_EVENT_PUBLISH_WAKE } from './driver-pg-event-wake-sql';
 import {
   SQL_EVENT_FIND,
   SQL_EVENT_LIST,
@@ -38,6 +39,13 @@ export interface PostgresEventBusOptions {
   readonly defaultTtl?: DurationInput;
   /** Rows returned by `list()`. Diagnostics only; `find()` is what a step uses. */
   readonly listLimit?: number;
+  /**
+   * Publish with `SQL_EVENT_PUBLISH_WAKE`: the runs parked on the event are made due in the
+   * publishing statement, and their queues told at commit. Only over the database the queue's
+   * `x_jobs` lives in, so boot sets it for `postgresJobDriver` alone: another driver keeps no
+   * rows here, a wake would find none, and its waiters keep the 30 s poll. Default `false`.
+   */
+  readonly wakes?: boolean;
 }
 
 export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
@@ -51,6 +59,8 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
   );
   const listLimit = finiteOption('the pg event bus', 'listLimit', options.listLimit ?? 1_000);
   const exec = options.executor;
+  const wakes = options.wakes === true;
+  const publishStatement = wakes ? SQL_EVENT_PUBLISH_WAKE : SQL_EVENT_PUBLISH;
 
   // Awaited and never fired-and-forgotten: it was `void`ed with nothing calling it, so the table
   // only grew. A failure REJECTS — the caller is the retention sweep's step, which retries.
@@ -61,6 +71,7 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
 
   return {
     stored: true,
+    wakes,
     async now() {
       const rows = await exec.query<{ now: number | string }>(SQL_EVENT_NOW, []);
       return Number(rows[0]?.now);
@@ -75,7 +86,7 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
       const rows = await exec.query<{
         published_at: number | string;
         expires_at: number | string;
-      }>(SQL_EVENT_PUBLISH, [
+      }>(publishStatement, [
         id,
         name,
         JSON.stringify(payload ?? null),
@@ -99,16 +110,17 @@ export function postgresEventBus(options: PostgresEventBusOptions): EventBus {
       };
     },
 
-    /** Earliest match at or after `afterMs`, so a resumed step consumes events in order. */
-    async find(name, correlationKey, afterMs) {
-      const rows = await exec.query<{ payload: unknown; published_at: number | string }>(
-        SQL_EVENT_FIND,
-        [name, correlationKey ?? null, afterMs],
-      );
+    /** Earliest match at or after `afterMs` and past `afterId`, so a step consumes in order. */
+    async find(name, correlationKey, afterMs, afterId) {
+      const rows = await exec.query<{
+        id: string;
+        payload: unknown;
+        published_at: number | string;
+      }>(SQL_EVENT_FIND, [name, correlationKey ?? null, afterMs, afterId ?? null]);
       const row = rows[0];
       return row === undefined
         ? undefined
-        : { payload: row.payload, publishedAt: Number(row.published_at) };
+        : { id: row.id, payload: row.payload, publishedAt: Number(row.published_at) };
     },
 
     async list(name) {

package/src/events.ts

@@ -111,12 +111,21 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
 
     /**
      * Earliest matching event at or after `afterMs`, so a resumed step consumes events in
-     * publication order rather than jumping to the newest one.
+     * publication order rather than jumping to the newest one. Ties in one ms go to the earlier
+     * publish (the Map's insertion order), and `afterId` seeks past that event in the same order.
      */
-    find(name, correlationKey, afterMs) {
+    find(name, correlationKey, afterMs, afterId) {
       const at = nowMs(clock);
+      const cursor = afterId === undefined ? undefined : events.get(afterId);
+      // Until the cursor's own event is passed, an event in its millisecond came before it.
+      let passed = cursor === undefined;
       let best: JobEvent | undefined;
       for (const event of events.values()) {
+        if (event === cursor) {
+          passed = true;
+          continue;
+        }
+        if (!passed && event.publishedAt <= (cursor?.publishedAt ?? 0)) continue;
         if (event.name !== name) continue;
         if (event.expiresAt <= at) continue;
         if (event.publishedAt < afterMs) continue;
@@ -124,7 +133,9 @@ export function memoryEventBus(options: MemoryEventBusOptions = {}): EventBus {
         if (best === undefined || event.publishedAt < best.publishedAt) best = event;
       }
       return Promise.resolve(
-        best === undefined ? undefined : { payload: best.payload, publishedAt: best.publishedAt },
+        best === undefined
+          ? undefined
+          : { id: best.id, payload: best.payload, publishedAt: best.publishedAt },
       );
     },
 

package/src/execute-ctx.ts

@@ -0,0 +1,51 @@
+// What `executeJob` reads off the caller's context, defensively — a context can arrive across a cast
+// (`asCtx`, a test's `{} as Ctx`, an app's `WorkerOptions.context()`) without the fields the type
+// promises — and whether the worker's drain ended the run. Split from `execute.ts` at its ceiling.
+
+import type { Actor, Ctx, ServiceBag } from '@ultimat3/core';
+import { anonymousActor, isUltimateError } from '@ultimat3/core';
+
+/** Stands in for a caller with nothing to cancel, so the composition below has one shape. */
+const NEVER_ABORTED = new AbortController().signal;
+
+/**
+ * `Ctx.signal` is non-optional in the type and `ctxOf` always sets it — but a context can
+ * still arrive across a cast (`@ultimat3/http`'s `asCtx`, a test's `{} as Ctx`) without one, and
+ * a job that crashed on a missing field would be a far worse answer than a job with no caller to
+ * follow. Read it, do not assume it.
+ */
+export function callerSignal(ctx: Ctx): AbortSignal {
+  return ctx.signal instanceof AbortSignal ? ctx.signal : NEVER_ABORTED;
+}
+
+/**
+ * The same defensive read, for the same reason: `Ctx.actor` is non-optional in the type and
+ * `ctxOf` always sets it, but `WorkerOptions.context()` is the app's own function and a
+ * cast context (`{} as Ctx`) reaches here without one. Anonymous is the honest stand-in — it
+ * carries no org, so the job's declared tenant is the only thing that can put one on the run.
+ */
+export function callerActor(ctx: Ctx): Actor {
+  const actor: Actor | undefined = ctx.actor;
+  return actor === undefined ? anonymousActor() : actor;
+}
+
+const NO_SERVICES: ServiceBag = Object.freeze({});
+
+/**
+ * The same defensive read a third time, and this one is load-bearing rather than merely kind:
+ * `withChildContext` ITERATES the parent's bag to decide what carries forward, so a cast context
+ * with no `services` would fail the run with a `TypeError` before the body ever started.
+ */
+export function callerServices(ctx: Ctx): ServiceBag {
+  const services: ServiceBag | undefined = ctx.services;
+  return services === undefined ? NO_SERVICES : services;
+}
+
+/**
+ * The run was cancelled by the worker's drain: the code `JobDrainedError` carries, read off the
+ * signal. The FIRST reason wins on a controller, so a timeout that fired before the drain still
+ * reports as a timeout — the drain only claims an attempt it ended.
+ */
+export function drainedBy(signal: AbortSignal): boolean {
+  return signal.aborted && isUltimateError(signal.reason) && signal.reason.code === 'X_DRAINING';
+}

package/src/execute-unwind.ts

@@ -0,0 +1,191 @@
+// A `compensates: true` run that failed for good, settled: its compensators run, last-in first-out,
+// and only then is the row dead-lettered with the failure it unwound from. A compensator that fails
+// hands the row back while the unwind has attempts left (`isFinalUnwindAttempt`); past them, or on
+// an unknown `once` step, the row is dead with `X_JOB_COMPENSATION_FAILED`. Split from `execute.ts`.
+
+import type { Clock, Ctx } from '@ultimat3/core';
+import {
+  failureStack,
+  isUltimateError,
+  logger,
+  renderThrowable,
+  reportError,
+} from '@ultimat3/core';
+import { nowMs } from './clock';
+import type { ClaimedJob, JobDriver, SettleBy } from './driver';
+import { LEASE_LAPSED_FINAL_ATTEMPT } from './driver';
+import type { JobExecution, JobOutcome } from './execute';
+import type { AnyJobHandle } from './job';
+import { backoffDelayMs, isFinalUnwindAttempt } from './retry';
+import type { JobStopReason } from './retry-classification';
+import { failureForRow } from './retry-classification';
+import type { AnnounceOptions } from './settled';
+import { announceSettled, settledCode } from './settled';
+import type { StepRunner } from './steps';
+
+/** What the attempt that failed for good knows about its own failure. Absent: already unwinding. */
+export interface UnwindFailure {
+  readonly thrown: unknown;
+  /** As the row records it (`recordedFailure`). */
+  readonly recorded: string;
+  readonly stop: JobStopReason | undefined;
+}
+
+export interface UnwindSettleOptions {
+  readonly driver: JobDriver;
+  readonly claimed: ClaimedJob;
+  readonly handle: AnyJobHandle;
+  readonly runner: StepRunner;
+  readonly ctx: Ctx;
+  readonly clock?: Clock;
+  /** When the attempt began, on `clock`. */
+  readonly startedAt: number;
+  readonly by: () => SettleBy;
+  readonly landed: (did: boolean, settling: string) => boolean;
+  /** `executeJob`'s: the run's steps and replay trace onto the execution it answers. */
+  readonly settle: (execution: JobExecution) => Promise<JobExecution>;
+  /** The body's scope and parsed payload, for `onSettled`. */
+  readonly run: AnnounceOptions['run'];
+  /** The worker's drain ended this attempt — the compensator's failure is not its own. */
+  readonly drained: () => boolean;
+}
+
+type Ending = Pick<JobExecution, 'error' | 'resumeAt' | 'stopReason'>;
+
+/** The execution this attempt answers, and `onSettled` after a settle that landed. */
+function endings(options: UnwindSettleOptions) {
+  const { handle, claimed } = options;
+  return {
+    finish: (outcome: JobOutcome, extra: Ending): Promise<JobExecution> =>
+      options.settle({
+        outcome,
+        jobId: claimed.id,
+        job: handle.name,
+        attempt: claimed.attempt,
+        durationMs: nowMs(options.clock) - options.startedAt,
+        ...extra,
+        steps: [],
+        replayed: [],
+      }),
+    announce: (
+      outcome: 'dead-lettered' | 'dropped',
+      error: string,
+      code: string | undefined,
+    ): Promise<void> =>
+      announceSettled({
+        handle,
+        claimed,
+        ctx: options.ctx,
+        ...(options.run === undefined ? {} : { run: options.run }),
+        settlement: { outcome, error, code },
+      }),
+  };
+}
+
+/** An unknown `once` step stops the unwind on any attempt: only a person can answer it. */
+const isUnknownOutcome = (thrown: unknown): boolean =>
+  isUltimateError(thrown) &&
+  thrown.code === 'X_JOB_COMPENSATION_FAILED' &&
+  thrown.meta?.['outcome'] === 'unknown';
+
+export async function unwindAndSettle(
+  options: UnwindSettleOptions,
+  failure: UnwindFailure | undefined,
+): Promise<JobExecution> {
+  const { driver, claimed, handle, runner, by, landed } = options;
+  const { finish, announce } = endings(options);
+  const policy = handle.retry;
+  const keep = policy.deadLetter ?? true;
+  const settledAs = keep ? 'dead-lettered' : 'dropped';
+  let original: string;
+  try {
+    // A crash on the final attempt left no failure to read: the claim re-offered the row.
+    original = await runner.unwind(failure?.recorded ?? LEASE_LAPSED_FINAL_ATTEMPT);
+  } catch (thrown) {
+    const message = renderThrowable(thrown);
+    if (options.drained()) {
+      landed(
+        await driver.nack(claimed.id, { ...by(), delayMs: 0, countsAsAttempt: false }),
+        'interrupted',
+      );
+      return finish('interrupted', { error: message });
+    }
+    if (!isUnknownOutcome(thrown) && !isFinalUnwindAttempt(policy, claimed.attempt)) {
+      const delayMs = backoffDelayMs(policy, claimed.attempt);
+      // The failure the run unwinds from stays `last_error`: no error here keeps the row's own.
+      landed(
+        await driver.nack(claimed.id, {
+          ...by(),
+          delayMs,
+          ...(failure === undefined ? {} : { error: failure.recorded }),
+          countsAsAttempt: true,
+        }),
+        'retried',
+      );
+      logger.warn('jobs.unwind.retried', {
+        job: handle.name,
+        jobId: claimed.id,
+        attempt: claimed.attempt,
+        error: message,
+      });
+      return finish('retried', {
+        error: message,
+        resumeAt: nowMs(options.clock) + delayMs,
+      });
+    }
+    const recorded = failureForRow(thrown);
+    const stack = failureStack(thrown);
+    const settled = landed(
+      await driver.nack(claimed.id, {
+        ...by(),
+        delayMs: 0,
+        error: recorded,
+        ...(stack === undefined ? {} : { stack }),
+        countsAsAttempt: true,
+        deadLetter: keep,
+        fail: !keep,
+      }),
+      settledAs,
+    );
+    logger.warn('jobs.unwind.failed', { job: handle.name, jobId: claimed.id, error: message });
+    reportError(thrown, {
+      source: 'job',
+      severity: 'error',
+      scope: { operation: handle.name, extra: { jobId: claimed.id, runId: claimed.runId } },
+    });
+    if (settled) await announce(settledAs, recorded, settledCode(thrown));
+    return finish(settledAs, { error: message, stopReason: 'terminal' });
+  }
+
+  const stack = failure === undefined ? undefined : failureStack(failure.thrown);
+  const settled = landed(
+    await driver.nack(claimed.id, {
+      ...by(),
+      delayMs: 0,
+      error: original,
+      ...(stack === undefined ? {} : { stack }),
+      countsAsAttempt: true,
+      deadLetter: keep,
+      fail: !keep,
+    }),
+    settledAs,
+  );
+  logger.warn('jobs.unwind.completed', {
+    job: handle.name,
+    jobId: claimed.id,
+    attempt: claimed.attempt,
+    error: original,
+  });
+  if (failure !== undefined) {
+    reportError(failure.thrown, {
+      source: 'job',
+      severity: 'error',
+      scope: { operation: handle.name, extra: { jobId: claimed.id, runId: claimed.runId } },
+    });
+  }
+  if (settled) await announce(settledAs, original, settledCode(failure?.thrown));
+  return finish(settledAs, {
+    error: original,
+    stopReason: failure?.stop ?? 'attempts-exhausted',
+  });
+}

package/src/execute.ts

@@ -5,11 +5,9 @@
 // a deadline here means CANCEL: the nack that follows makes the job claimable again, so a body
 // still running past it would be a second copy of one job, racing the attempt that replaced it.
 
-import type { Actor, Clock, Ctx, ServiceBag } from '@ultimat3/core';
+import type { Clock, Ctx } from '@ultimat3/core';
 import {
-  anonymousActor,
   failureStack,
-  isUltimateError,
   logger,
   renderThrowable,
   reportError,
@@ -23,9 +21,12 @@ import type { ClaimedJob, JobDriver, SettleBy } from './driver';
 import { claimOf } from './driver';
 import { JobAbortedError } from './errors';
 import { eventBus } from './events';
+import { callerActor, callerServices, callerSignal, drainedBy } from './execute-ctx';
+import type { UnwindSettleOptions } from './execute-unwind';
+import { unwindAndSettle } from './execute-unwind';
 import type { AnyJobHandle } from './job';
 import { createProgressReporter } from './progress';
-import { isFinalAttempt } from './retry';
+import { isFinalAttempt, isUnwindAttempt } from './retry';
 import type { JobStopReason } from './retry-classification';
 import {
   failureForRow,
@@ -38,7 +39,8 @@ import { createRunSignal } from './run-signal';
 import { announceSettled, settledCode } from './settled';
 import type { EventLookup, StepRecord } from './steps';
 import { isStepSuspension, stepRunner } from './steps';
-import { jobRunActor } from './tenant';
+import { isUnwindReached } from './steps-suspension';
+import { jobRunActor, refuseRunTenant } from './tenant';
 
 /**
  * How one attempt ended. `interrupted` is the worker's drain cutting the attempt short: the job is
@@ -59,42 +61,6 @@ export type JobOutcome =
   | 'interrupted'
   | 'refused';
 
-/** Stands in for a caller with nothing to cancel, so the composition below has one shape. */
-const NEVER_ABORTED = new AbortController().signal;
-
-/**
- * `Ctx.signal` is non-optional in the type and `ctxOf` always sets it — but a context can
- * still arrive across a cast (`@ultimat3/http`'s `asCtx`, a test's `{} as Ctx`) without one, and
- * a job that crashed on a missing field would be a far worse answer than a job with no caller to
- * follow. Read it, do not assume it.
- */
-function callerSignal(ctx: Ctx): AbortSignal {
-  return ctx.signal instanceof AbortSignal ? ctx.signal : NEVER_ABORTED;
-}
-
-/**
- * The same defensive read, for the same reason: `Ctx.actor` is non-optional in the type and
- * `ctxOf` always sets it, but `WorkerOptions.context()` is the app's own function and a
- * cast context (`{} as Ctx`) reaches here without one. Anonymous is the honest stand-in — it
- * carries no org, so the job's declared tenant is the only thing that can put one on the run.
- */
-function callerActor(ctx: Ctx): Actor {
-  const actor: Actor | undefined = ctx.actor;
-  return actor === undefined ? anonymousActor() : actor;
-}
-
-const NO_SERVICES: ServiceBag = Object.freeze({});
-
-/**
- * The same defensive read a third time, and this one is load-bearing rather than merely kind:
- * `withChildContext` ITERATES the parent's bag to decide what carries forward, so a cast context
- * with no `services` would fail the run with a `TypeError` before the body ever started.
- */
-function callerServices(ctx: Ctx): ServiceBag {
-  const services: ServiceBag | undefined = ctx.services;
-  return services === undefined ? NO_SERVICES : services;
-}
-
 export interface JobExecution {
   readonly outcome: JobOutcome;
   readonly jobId: string;
@@ -167,6 +133,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     // no `job()` can reach — which both of these were until 2026-08.
     ...(handle.stepTimeoutMs === undefined ? {} : { stepTimeoutMs: handle.stepTimeoutMs }),
     ...(handle.eventPollMs === undefined ? {} : { eventPollMs: handle.eventPollMs }),
+    // Compensation is the job's declaration; an attempt past the policy's last is the claim
+    // re-offering a final attempt that crashed, and unwinds whatever its steps say.
+    compensates: handle.compensates,
+    unwind: handle.compensates && isUnwindAttempt(handle.retry, claimed.attempt),
     ...(options.clock === undefined ? {} : { clock: options.clock }),
     events: options.events ?? eventBus(),
   });
@@ -223,6 +193,22 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     return { ...outcome, steps, replayed: runner.replayedNames() };
   };
 
+  /** Everything `unwindAndSettle` needs of this attempt — built only when a run unwinds. */
+  const unwinder = (): UnwindSettleOptions => ({
+    driver,
+    claimed,
+    handle,
+    runner,
+    ctx,
+    startedAt,
+    by,
+    landed,
+    settle,
+    run: { scope: inRunScope, input: parsed },
+    drained: () => drainedBy(signal),
+    ...(options.clock === undefined ? {} : { clock: options.clock }),
+  });
+
   try {
     // Raised INSIDE the try so it takes the one failure path below — classified, nacked, logged
     // and reported exactly as if the body had thrown it — and the body never starts.
@@ -231,7 +217,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     parsed = { value: input };
     // The job's DECLARED tenant, on the actor the body runs as. `tenant: 'none'` strips the org
     // rather than inheriting the worker's, so a tenant-scoped read inside such a job fails closed.
-    const runActor = jobRunActor(callerActor(ctx), handle.tenantFor(input));
+    const declared = handle.tenantFor(input);
+    // A row queued for one org whose payload names another: refused before the body, terminally.
+    refuseRunTenant(handle.name, declared, claimed.tenantId);
+    const runActor = jobRunActor(callerActor(ctx), declared);
     // Installed as the AMBIENT context and not only handed over as a parameter. This is the whole
     // of the fix: `@ultimat3/entity`'s tenant guard derives from `tryUseContext()`, so a ctx passed
     // as an argument was read by nobody — `actorTenant` answered `undefined`, `scopedPlan` derived
@@ -266,6 +255,12 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     result = await (handle.timeoutMs === undefined
       ? work
       : raceTimeout(work, handle.timeoutMs, handle.name, cancel));
+    // An unwinding run's replay that RETURNED is still unwinding: the decision is durable.
+    // `return await`, at every unwind: the `finally` below ends the attempt, and the compensators
+    // must run — and write — before it does.
+    if (handle.compensates && (await runner.unwinding())) {
+      return await unwindAndSettle(unwinder(), undefined);
+    }
   } catch (error) {
     // Before ANY settle: the last value a body reported is written while the row is still this
     // worker's, which is the only time `recordProgress`'s fence lets it land.
@@ -292,6 +287,15 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     }
 
     const message = renderThrowable(error);
+    // Unwinding already: the replay stopped at its first incomplete step (`UnwindReached`) or
+    // ended any other way. Never retried forward — a drain still hands the attempt back.
+    if (
+      handle.compensates &&
+      !drainedBy(signal) &&
+      (isUnwindReached(error) || (await runner.unwinding().catch(() => false)))
+    ) {
+      return await unwindAndSettle(unwinder(), undefined);
+    }
     if (drainedBy(signal)) {
       // The worker's drain told this body to stop, and it did. Whatever it stopped WITH is the
       // framework's doing — the reason itself back from `fetch`, `throwIfAborted`'s `X_ABORTED`,
@@ -372,6 +376,10 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     // "do not keep" was re-claimed and re-run forever with its attempt counter climbing.
     const dropped = !decision.retry && !decision.deadLetter;
     const recorded = recordedFailure(failureForRow(error), decision);
+    // Failed for good, and the job undoes its steps: compensators first, the dead letter after.
+    if (!decision.retry && handle.compensates) {
+      return await unwindAndSettle(unwinder(), { thrown: error, recorded, stop });
+    }
     const stack = failureStack(error);
     const settled = landed(
       await driver.nack(claimed.id, {
@@ -484,12 +492,3 @@ export async function executeJob(options: ExecuteJobOptions): Promise<JobExecuti
     replayed: [],
   });
 }
-
-/**
- * The run was cancelled by the worker's drain: the code `JobDrainedError` carries, read off the
- * signal. The FIRST reason wins on a controller, so a timeout that fired before the drain still
- * reports as a timeout — the drain only claims an attempt it ended.
- */
-function drainedBy(signal: AbortSignal): boolean {
-  return signal.aborted && isUltimateError(signal.reason) && signal.reason.code === 'X_DRAINING';
-}

package/src/export-errors.ts

@@ -1,5 +1,5 @@
 // The two `X_EXPORT_*` codes an export pass can end on, apart from `errors.ts` for the reason
-// `backfill-errors.ts` and `webhook-errors.ts` are: one file, one job, and `errors.ts` holds the
+// `backfill-errors.ts` and `webhook-errors.ts` are: one file, one job, and `error-codes.ts` holds the
 // registry — the codes, the titles and the single `registerErrorCodes()` call.
 //
 // Both are TERMINAL and both are refusals of the DECLARATION rather than of the data: the same

package/src/failure-row.ts

@@ -0,0 +1,43 @@
+// A job row's `last_error` read for a CALLER: the code it was thrown with and the `callerFix` it
+// declared. One text column holds two audiences, split by the one tab no rendered field contains.
+
+import { hasErrorCode } from '@ultimat3/core';
+
+/**
+ * Between the operator's line and the caller's fix. `failureForRow` runs the operator's line
+ * through core's `singleLine`, which escapes every control character, so the first raw tab in a
+ * row is this one — a thrown message cannot write it, and so cannot forge a caller's fix.
+ */
+export const CALLER_FIX_SEPARATOR = '\tcaller fix: ';
+
+/** What a remote caller may be told about a failed job. Never the worker's text. */
+export interface CallerFailure {
+  /** A registered code the row's failure was THROWN with, else `X_INTERNAL`. */
+  readonly code: string;
+  /** The thrown error's declared `callerFix`; absent when it declared none. */
+  readonly callerFix?: string;
+}
+
+/** `<name>: <CODE>: …` — the leading position `renderThrowable` gives a coded error. */
+const LEADING_CODE = /^[A-Za-z]*: (X(?:_[A-Z0-9]+)+): /;
+
+/**
+ * The failure as a caller reads it. Only a code in the leading position counts — a `TypeError`
+ * whose message merely mentions one is not that code — and only a code this process registered;
+ * anything else is `X_INTERNAL`, http's name for the same case, and carries no fix of its own.
+ * The developer's `fix:` is on the operator's line and never crosses (core's `fixFor`).
+ */
+export function callerFailure(lastError: string | undefined): CallerFailure {
+  const row = lastError ?? '';
+  const code = LEADING_CODE.exec(row)?.[1];
+  if (code === undefined || !hasErrorCode(code)) return { code: 'X_INTERNAL' };
+  const at = row.indexOf(CALLER_FIX_SEPARATOR);
+  if (at === -1) return { code };
+  return { code, callerFix: row.slice(at + CALLER_FIX_SEPARATOR.length) };
+}
+
+/** `row` with `suffix` appended to the operator's half, the caller's fix (if any) kept last. */
+export function appendToOperatorLine(row: string, suffix: string): string {
+  const at = row.indexOf(CALLER_FIX_SEPARATOR);
+  return at === -1 ? `${row}${suffix}` : `${row.slice(0, at)}${suffix}${row.slice(at)}`;
+}