@ultimat3/jobs 29.3.2 → 30.0.0

package/src/steps-output.ts

@@ -0,0 +1,142 @@
+// What `step.run` answers: the JSON form of what the body returned, on the first run as on a
+// replay. The type states it and `jsonForm` produces it, so a body reading a `Date` off a step
+// result fails at compile time, and if forced, on attempt 1 rather than on the first resume.
+
+import { isJsonObject, isUltimateError, jsonbRefusal, renderThrowable } from '@ultimat3/core';
+import { StepOutputUnserializableError } from './errors-steps';
+
+type Callable = ((...args: never[]) => unknown) | (abstract new (...args: never[]) => unknown);
+
+/** The part of a member's type JSON leaves out: `{ a: undefined }` writes `{}`. */
+type NonJson<V> = V extends void ? V : V extends symbol | Callable ? V : never;
+
+/** `unknown` and `any` promise nothing about absence, so they stay what they are. */
+type IsOpen<V> = unknown extends V ? true : false;
+
+type AlwaysThere<V> = IsOpen<V> extends true ? true : [NonJson<V>] extends [never] ? true : false;
+
+type SometimesThere<V> =
+  AlwaysThere<V> extends true ? false : [Exclude<V, NonJson<V>>] extends [never] ? false : true;
+
+type JsonMembers<T> = {
+  [K in keyof T as K extends symbol
+    ? never
+    : AlwaysThere<T[K]> extends true
+      ? K
+      : never]: StepOutput<T[K]>;
+} & {
+  [K in keyof T as K extends symbol
+    ? never
+    : SometimesThere<T[K]> extends true
+      ? K
+      : never]?: StepOutput<Exclude<T[K], NonJson<T[K]>>>;
+};
+
+type Flat<T> = { [K in keyof T]: T[K] };
+
+/** Instances whose state lives in slots or on the prototype: `JSON.stringify` writes `{}`. */
+type WritesNoMembers =
+  | ReadonlyMap<unknown, unknown>
+  | ReadonlySet<unknown>
+  | RegExp
+  | ArrayBuffer
+  | DataView;
+
+/** `new Uint8Array([1])` writes `{"0":1}`, an object keyed by index, never an array. */
+type TypedArray = Exclude<ArrayBufferView, DataView>;
+
+type JsonOf<T> = T extends string | number | boolean | null
+  ? T
+  : T extends bigint
+    ? never
+    : T extends undefined
+      ? null
+      : T extends void
+        ? T
+        : T extends symbol | Callable
+          ? null
+          : T extends { toJSON(...args: never[]): infer R }
+            ? StepOutput<R>
+            : T extends WritesNoMembers
+              ? Record<string, never>
+              : T extends Error
+                ? Flat<JsonMembers<Omit<T, keyof Error>>>
+                : T extends TypedArray
+                  ? Record<string, number>
+                  : T extends readonly unknown[]
+                    ? { [K in keyof T]: StepOutput<T[K]> }
+                    : T extends object
+                      ? Flat<JsonMembers<T>>
+                      : T;
+
+/**
+ * The JSON form of `T`, exactly what `JSON.parse(JSON.stringify(value))` leaves. Primitives and
+ * brands kept; a `Date` or anything with a `toJSON()` becomes what that returns; function and
+ * symbol members dropped; `undefined` at the top or in an array is `null`, on a member the member
+ * is absent; `readonly` and optionality kept. `unknown` stays `unknown`, `void` stays `void`.
+ * `bigint` is `never`: the value raises `X_STEP_OUTPUT_UNSERIALIZABLE` instead of arriving, as do
+ * `NaN`, an infinity, an invalid `Date` and text `jsonb` refuses, which the type cannot see.
+ */
+export type StepOutput<T> = unknown extends T ? T : JsonOf<T>;
+
+/**
+ * The value both attempts see. One stringify and one parse per EXECUTED step, none on a replay; a
+ * step persists a cursor, not a page, which is what keeps that cheap (`backfill()`'s rule).
+ *
+ * `JSON.stringify` answers `undefined` for `undefined`, a function and a symbol, and every store
+ * writes those as `null` (`JSON.stringify(output ?? null)`), so that is the stored form.
+ *
+ * A number `JSON.stringify` would write as `null` is refused here rather than typed `number` and
+ * answered `null`. Text `jsonb` refuses is refused on every store: Postgres alone raised it, from
+ * `put`, with nothing recorded, so the body ran again on each attempt where memory ran it once.
+ */
+export function jsonForm<T>(value: T, at: { job: string; step: string }): StepOutput<T> {
+  const refused = (reason: string) => new StepOutputUnserializableError({ ...at, reason });
+  let text: string | undefined;
+  try {
+    text = JSON.stringify(value, function screen(this: unknown, key: string, written: unknown) {
+      if (typeof written === 'number' && !Number.isFinite(written)) {
+        throw refused('a NaN or an infinity, which JSON writes as null');
+      }
+      // Only the holder still has the `Date`: its `toJSON` has already answered `null`.
+      if (written === null && (this as Record<string, unknown>)[key] instanceof Date) {
+        throw refused('an invalid Date, which JSON writes as null');
+      }
+      return written;
+    });
+  } catch (error) {
+    if (isUltimateError(error) && error.code === 'X_STEP_OUTPUT_UNSERIALIZABLE') throw error;
+    throw refused(renderThrowable(error));
+  }
+  if (text === undefined) return null as StepOutput<T>;
+  const refusal = jsonbRefusal(text);
+  if (refusal !== undefined) throw refused(refusal);
+  return JSON.parse(text, inStoredOrder) as StepOutput<T>;
+}
+
+const UTF8 = new TextEncoder();
+
+/** `jsonb`'s own key order: shorter first by bytes, then bytewise. */
+function storedKeyOrder(a: Uint8Array, b: Uint8Array): number {
+  if (a.length !== b.length) return a.length - b.length;
+  for (let at = 0; at < a.length; at += 1) {
+    const difference = (a[at] ?? 0) - (b[at] ?? 0);
+    if (difference !== 0) return difference;
+  }
+  return 0;
+}
+
+/**
+ * A reviver: Postgres hands keys back in `jsonb`'s order and memory in the body's, so a body reading
+ * `Object.keys` saw two runs. Every attempt on every store now reads this one. Integer-like keys
+ * still come first, on both paths, because a JS object puts them there. `fromEntries` defines, so a
+ * `__proto__` key stays data.
+ */
+function inStoredOrder(_key: string, value: unknown): unknown {
+  if (!isJsonObject(value)) return value;
+  const entries = Object.entries(value);
+  if (entries.length < 2) return value;
+  const keyed = entries.map((entry) => ({ entry, bytes: UTF8.encode(entry[0]) }));
+  keyed.sort((a, b) => storedKeyOrder(a.bytes, b.bytes));
+  return Object.fromEntries(keyed.map(({ entry }) => entry));
+}

package/src/steps-store.ts

@@ -0,0 +1,68 @@
+// What a run's steps are stored as: the records, their status vocabulary, the fence a write is made
+// under, and the store seam every driver implements. Split from `steps.ts` at the file-size ceiling:
+// that file is the runner, this is what it writes through.
+
+/**
+ * The runtime list is the declaration and `StepStatus` is derived from it, the shape
+ * `BACKFILL_STATUSES` and `PRIMITIVE_KINDS` already have. A bare union cannot narrow a `text`
+ * column, so `driver-pg-rows.ts` cast one instead — and a cast that lands an unknown status on a
+ * record makes `stepRun`'s `existing?.status === 'completed'` false, which RE-EXECUTES a step
+ * this file promises runs once.
+ */
+export const STEP_STATUSES = [
+  'completed',
+  'sleeping',
+  'waiting',
+  'failed',
+  // A `once` step's record BEFORE its `fn` runs. Read back with no `completed`, it is an unknown
+  // outcome (`X_STEP_OUTCOME_UNKNOWN`); `cleared` is an operator's "it did not run": run it again.
+  'started',
+  'cleared',
+] as const;
+
+export type StepStatus = (typeof STEP_STATUSES)[number];
+
+/** Narrows a status read back out of a store. Never a cast — the list decides. */
+export const isStepStatus = (value: string): value is StepStatus =>
+  (STEP_STATUSES as readonly string[]).includes(value);
+
+export interface StepRecord {
+  readonly runId: string;
+  readonly name: string;
+  readonly status: StepStatus;
+  /**
+   * The memoized return value. Present when `status === 'completed'` — and on a `waiting` record
+   * holding its search floor (`waitFloor`).
+   */
+  readonly output?: unknown;
+  readonly startedAt: number;
+  readonly completedAt?: number;
+  /** Epoch ms at which a sleeping/waiting step becomes runnable. */
+  readonly wakeAt?: number;
+  readonly event?: string;
+  readonly correlationKey?: string;
+  readonly attempts: number;
+  readonly error?: string;
+}
+
+/** The claim a step write is made under: `claimOf(claimed)`, plus the row it names. */
+export interface StepFence {
+  readonly job: string;
+  readonly jobId: string;
+  readonly workerId: string;
+  readonly claim: number;
+}
+
+export interface StepStore {
+  get(runId: string, name: string): Promise<StepRecord | undefined>;
+  /**
+   * With `by`, the write lands only while that claim still holds the row (`running`, same worker,
+   * same claim ordinal — `SQL_ACK`'s fence) and is `X_JOB_LEASE_LOST` otherwise: the in-process
+   * signal cannot stop a body in ANOTHER process whose lease lapsed. Without it the write is
+   * unfenced — a transfer (`x jobs drain`) or a runner driven by hand, with no row to hold.
+   */
+  put(record: StepRecord, by?: StepFence): Promise<void>;
+  list(runId: string): Promise<readonly StepRecord[]>;
+  del(runId: string, name: string): Promise<void>;
+  clear(runId: string): Promise<void>;
+}

package/src/steps-suspension.ts

@@ -41,6 +41,57 @@ export function isWaitTimedOut(output: unknown): boolean {
   );
 }
 
+const WAIT_FLOOR_KEY = '~ultimate.waitFloor';
+const WAIT_FLOOR_ID_KEY = '~ultimate.waitFloorId';
+
+/** Where a wait's next search starts: at `afterMs`, and past the event `afterId` names. */
+export interface WaitFloor {
+  readonly afterMs: number;
+  readonly afterId?: string;
+}
+
+/**
+ * What a `waiting` record holds once its schema refused an event: where the next search starts.
+ * `startedAt` cannot carry it, because the timeout counts from there. `find` answers the EARLIEST
+ * event, so without this a repaired publisher's event sat behind the refused one until its TTL.
+ * The refused event's id rides with its instant: a floor of `publishedAt + 1` skipped every valid
+ * event stamped in the same millisecond. A record written before the id has `afterMs` alone.
+ */
+export const waitFloor = (floor: WaitFloor): Record<string, number | string> => ({
+  [WAIT_FLOOR_KEY]: floor.afterMs,
+  ...(floor.afterId === undefined ? {} : { [WAIT_FLOOR_ID_KEY]: floor.afterId }),
+});
+
+export function waitFloorOf(output: unknown): WaitFloor | undefined {
+  if (typeof output !== 'object' || output === null) return undefined;
+  const record = output as Record<string, unknown>;
+  const afterMs = record[WAIT_FLOOR_KEY];
+  if (typeof afterMs !== 'number') return undefined;
+  const afterId = record[WAIT_FLOOR_ID_KEY];
+  return typeof afterId === 'string' ? { afterMs, afterId } : { afterMs };
+}
+
+/**
+ * What an incomplete step throws while the run is UNWINDING: the body replays only to re-register
+ * its compensators, and the first step that never completed is where the forward path stops. The
+ * `StepSuspension` shape, and private: `executeJob` reads it, then calls `runner.unwind()`.
+ */
+export class UnwindReached extends Error {
+  static readonly brand = 'ultimate.jobs.unwind-reached';
+  readonly brand: string = UnwindReached.brand;
+  readonly step: string;
+
+  constructor(step: string) {
+    super(`step "${step}" never completed, and the run is unwinding`);
+    this.name = 'UnwindReached';
+    this.step = step;
+  }
+}
+
+export function isUnwindReached(error: unknown): error is UnwindReached {
+  return error instanceof Error && (error as { brand?: unknown }).brand === UnwindReached.brand;
+}
+
 export function isStepSuspension(error: unknown): error is StepSuspension {
   return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
 }

package/src/steps-timeout.ts

@@ -1,6 +1,8 @@
 // A step's own ceiling: the race between a step body and `stepTimeout`. Split from `steps.ts` at
 // the file-size ceiling — that file is the runner, and this is the one timer it arms.
 
+import { MAX_TIMER_MS } from '@ultimat3/core';
+
 /**
  * A step's own ceiling. It ABORTS before it rejects, in that order: `run()` fails the step on this
  * rejection and the job retries, so a body still holding a socket open past the deadline would be
@@ -14,11 +16,14 @@ export function withStepTimeout<T>(
 ): Promise<T> {
   if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
   return new Promise<T>((resolve, reject) => {
-    const timer = setTimeout(() => {
-      const failure = error();
-      deadline.abort(failure);
-      reject(failure);
-    }, timeoutMs);
+    const timer = setTimeout(
+      () => {
+        const failure = error();
+        deadline.abort(failure);
+        reject(failure);
+      },
+      Math.min(timeoutMs, MAX_TIMER_MS),
+    );
     Promise.resolve(work).then(
       (value) => {
         clearTimeout(timer);

package/src/steps-unwind.ts

@@ -0,0 +1,94 @@
+// Durable compensation: the stack a replay rebuilds, and the unwind that runs it. No closure is
+// stored — the body replays from the top, so each `step.run` re-registers its own — and every
+// compensator runs as an ordinary step `compensate:<name>`, so a replay skips what already ran.
+
+import { renderThrowable } from '@ultimat3/core';
+import { COMPENSATE_PREFIX, JobCompensationFailedError, UNWINDING_STEP } from './errors-saga';
+import type { RunnerCore } from './steps-core';
+import type { StepRecord } from './steps-store';
+
+/** Runs one step body and records it — `steps.ts`'s, shared so a compensator is a step like any. */
+export type StepExecutor = (
+  name: string,
+  existing: StepRecord | undefined,
+  fn: (signal: AbortSignal) => unknown,
+) => Promise<unknown>;
+
+interface Compensation {
+  readonly step: string;
+  /** Absent for a `once` step whose outcome is unknown: there is no output to undo from. */
+  readonly undo?: (signal: AbortSignal) => unknown;
+}
+
+export interface CompensationStack {
+  /** A completed step with a compensator, in the order it completed. */
+  done(step: string, undo: (signal: AbortSignal) => unknown): void;
+  /** A `once` step with a compensator whose outcome nobody knows. */
+  unknown(step: string): void;
+  /**
+   * Write `unwinding` (once, before the first compensator), then run every registered compensator
+   * last-in first-out. Answers the failure the run is unwinding FROM: the one `unwinding` already
+   * holds, or `failure` when this call wrote it. A compensator that throws, or an unknown `once`
+   * step, is `X_JOB_COMPENSATION_FAILED`.
+   */
+  unwind(failure: string): Promise<string>;
+}
+
+/** The failure an `unwinding` record carries, when it carries one. */
+export function unwoundFrom(record: StepRecord | undefined): string | undefined {
+  const output = record?.output;
+  if (typeof output !== 'object' || output === null) return undefined;
+  const error = (output as Record<string, unknown>)['error'];
+  return typeof error === 'string' ? error : undefined;
+}
+
+export function compensationStack(core: RunnerCore, execute: StepExecutor): CompensationStack {
+  const entries: Compensation[] = [];
+  return {
+    done: (step, undo) => entries.push({ step, undo }),
+    unknown: (step) => entries.push({ step }),
+    async unwind(failure) {
+      const mark = await core.load(UNWINDING_STEP);
+      const original = unwoundFrom(mark) ?? failure;
+      if (mark === undefined) {
+        const at = core.now();
+        await core.put({
+          runId: core.runId,
+          name: UNWINDING_STEP,
+          status: 'completed',
+          output: { error: original },
+          startedAt: at,
+          completedAt: at,
+          attempts: 1,
+        });
+      }
+      for (const entry of [...entries].reverse()) {
+        if (entry.undo === undefined) {
+          throw new JobCompensationFailedError({
+            job: core.jobName,
+            step: entry.step,
+            outcome: 'unknown',
+            reason: 'X_STEP_OUTCOME_UNKNOWN',
+          });
+        }
+        const name = `${COMPENSATE_PREFIX}${entry.step}`;
+        const existing = await core.load(name);
+        if (existing?.status === 'completed') {
+          core.replayed(name);
+          continue;
+        }
+        try {
+          await execute(name, existing, entry.undo);
+        } catch (error) {
+          throw new JobCompensationFailedError({
+            job: core.jobName,
+            step: entry.step,
+            outcome: 'failed',
+            reason: renderThrowable(error),
+          });
+        }
+      }
+      return original;
+    },
+  };
+}

package/src/steps-wait.ts

@@ -0,0 +1,163 @@
+// `step.waitForEvent`: the stored answer, a hit parsed by the wait's own schema, the timeout called
+// on the bus's clock, and the park. Split from `steps.ts` at the file-size ceiling — that file is
+// the runner; this is one of its three step kinds, over the `RunnerCore` it shares.
+
+import { logger, renderThrowable } from '@ultimat3/core';
+import type { InferOutput, StandardSchemaV1 } from '@ultimat3/schema';
+import { formatIssues, validateAsync } from '@ultimat3/schema';
+import { finiteDurationMs } from './clock';
+import { JobTimeoutError } from './errors';
+import { JobEventInvalidError } from './errors-steps';
+import type { WaitForEventOptions } from './steps-api';
+import type { RunnerCore } from './steps-core';
+import { jsonForm } from './steps-output';
+import type { StepRecord } from './steps-store';
+import {
+  isWaitTimedOut,
+  StepSuspension,
+  WAIT_TIMED_OUT,
+  waitFloor,
+  waitFloorOf,
+} from './steps-suspension';
+
+const DEFAULT_WAIT_MS = 86_400_000;
+
+type Hit = { readonly id: string; readonly payload: unknown; readonly publishedAt: number };
+
+export function waitForEventStep(core: RunnerCore) {
+  const { runId, jobName } = core;
+  return async function waitForEvent<S extends StandardSchemaV1>(
+    name: string,
+    event: string,
+    waitOptions: WaitForEventOptions<S>,
+  ): Promise<InferOutput<S> | undefined> {
+    /** The store keeps the raw payload; a hit and a replay parse it alike, so both see one value. */
+    const answer = async (payload: unknown): Promise<InferOutput<S>> => {
+      const parsed = await validateAsync(waitOptions.schema, payload);
+      if (parsed.issues === undefined) return parsed.value;
+      const issues = formatIssues(parsed.issues);
+      throw new JobEventInvalidError({ job: jobName, step: name, event, issues });
+    };
+
+    core.claim(name);
+    const existing = await core.load(name);
+    if (existing?.status === 'completed') {
+      core.replayed(name);
+      return isWaitTimedOut(existing.output) ? undefined : answer(existing.output);
+    }
+    await core.forwardOnly(name);
+
+    const at = core.now();
+    // A NEW wait is stamped by the bus, whose clock stamps every `publishedAt` it is compared
+    // with; the runner's own only when there is no bus to ask. A re-poll keeps what it persisted.
+    const startedAt = existing?.startedAt ?? (await core.events?.now()) ?? at;
+    const timeoutMs = finiteDurationMs(
+      waitOptions.timeout ?? DEFAULT_WAIT_MS,
+      'step.waitForEvent',
+      'timeout',
+    );
+    const deadline = startedAt + timeoutMs;
+    const correlationKey = waitOptions.match;
+    const attempts = (existing?.attempts ?? 0) + 1;
+    const keyed = correlationKey === undefined ? {} : { correlationKey };
+
+    const floor = waitFloorOf(existing?.output);
+    const parked = {
+      runId,
+      name,
+      status: 'waiting',
+      startedAt,
+      event,
+      attempts,
+      ...keyed,
+    } as const;
+
+    /** A found event, parsed before it is recorded: a refused one is no step's answer. */
+    const consume = async (hit: Hit): Promise<InferOutput<S>> => {
+      // The form a replay will parse, so a hit cannot answer what its replay refuses.
+      const stored = jsonForm(hit.payload, { job: jobName, step: name });
+      const value = await answer(stored).catch(async (refusal: unknown) => {
+        // The original error is what the caller sees; without the floor a retry is refused again.
+        await core
+          .put({
+            ...parked,
+            output: waitFloor({ afterMs: hit.publishedAt, afterId: hit.id }),
+            wakeAt: at,
+          })
+          .catch((unwritten: unknown) =>
+            logger.warn('jobs.step.floor-unrecorded', {
+              job: jobName,
+              step: name,
+              error: renderThrowable(unwritten),
+            }),
+          );
+        throw refusal;
+      });
+      const done: StepRecord = {
+        ...parked,
+        status: 'completed',
+        output: stored,
+        completedAt: at,
+      };
+      await core.put(done);
+      return value;
+    };
+
+    const hit = await core.events?.find(
+      event,
+      correlationKey,
+      floor?.afterMs ?? startedAt,
+      floor?.afterId,
+    );
+    if (hit !== undefined) return consume(hit);
+
+    // `deadline` is on the BUS's clock, so the bus calls the timeout: asked only once this runner
+    // believes the time is up, which costs a wait one statement at its end. A runner AHEAD of the
+    // bus used to give up early by the skew — on an answer still inside its window. (A runner
+    // BEHIND waits the skew longer; knowing that sooner would cost a round trip on every poll.)
+    let remaining = deadline - at;
+    if (remaining <= 0) remaining = deadline - ((await core.events?.now()) ?? at);
+    if (remaining <= 0) {
+      if (waitOptions.required === true) {
+        throw new JobTimeoutError({
+          job: jobName,
+          step: name,
+          timeoutMs,
+          fix: `step.waitForEvent('${name}', '${event}', { timeout: '1h' })   # raises the wait's timeout; or make the publisher of "${event}" send it sooner`,
+        });
+      }
+      logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
+      await core.put({
+        runId,
+        name,
+        status: 'completed',
+        output: WAIT_TIMED_OUT,
+        startedAt,
+        completedAt: at,
+        event,
+        attempts,
+      });
+      return undefined;
+    }
+
+    const resumeAt = at + Math.min(remaining, core.pollMs);
+    await core.put({
+      ...parked,
+      wakeAt: resumeAt,
+      ...(floor === undefined ? {} : { output: waitFloor(floor) }),
+    });
+    // A waking bus wakes only the runs it finds `waiting`, so an event committed between the first
+    // `find` and the write above woke nobody. Asked once more: one that commits after THIS read
+    // meets a `running` row, which the wake leaves alone, and is the backstop poll's.
+    if (core.events?.wakes === true) {
+      const late = await core.events.find(
+        event,
+        correlationKey,
+        floor?.afterMs ?? startedAt,
+        floor?.afterId,
+      );
+      if (late !== undefined) return consume(late);
+    }
+    throw new StepSuspension({ step: name, resumeAt, reason: 'event' });
+  };
+}