@ultimat3/jobs 1.0.0

package/src/steps.d.ts

@@ -0,0 +1,86 @@
+import type { Clock } from '@ultimat3/core';
+import type { DurationInput } from './clock';
+export type StepStatus = 'completed' | 'sleeping' | 'waiting' | 'failed';
+export interface StepRecord {
+    readonly runId: string;
+    readonly name: string;
+    readonly status: StepStatus;
+    /** The memoized return value. Present only when `status === 'completed'`. */
+    readonly output?: unknown;
+    readonly startedAt: number;
+    readonly completedAt?: number;
+    /** Epoch ms at which a sleeping/waiting step becomes runnable. */
+    readonly wakeAt?: number;
+    readonly event?: string;
+    readonly correlationKey?: string;
+    readonly attempts: number;
+    readonly error?: string;
+}
+export interface StepStore {
+    get(runId: string, name: string): Promise<StepRecord | undefined>;
+    put(record: StepRecord): Promise<void>;
+    list(runId: string): Promise<readonly StepRecord[]>;
+    del(runId: string, name: string): Promise<void>;
+    clear(runId: string): Promise<void>;
+}
+/** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
+export interface EventLookup {
+    find(event: string, correlationKey: string | undefined, afterMs: number): Promise<{
+        readonly payload: unknown;
+        readonly publishedAt: number;
+    } | undefined>;
+}
+export interface WaitForEventOptions {
+    /** How long to wait before giving up. Default 24h. */
+    readonly timeout?: DurationInput;
+    /** Correlation key the published event must carry — usually an entity id. */
+    readonly match?: string;
+    /** Default false: a timeout resolves `undefined`. `true` fails the job with X_JOB_TIMEOUT. */
+    readonly required?: boolean;
+}
+export interface StepApi {
+    /** Run once, ever. On replay the persisted output is returned and `fn` is not called. */
+    run<T>(name: string, fn: () => Promise<T> | T): Promise<T>;
+    /** Suspend the run. `sleep(duration)` derives the step name from the duration. */
+    sleep(name: string, duration: DurationInput): Promise<void>;
+    sleep(duration: DurationInput): Promise<void>;
+    waitForEvent<T>(name: string, event: string, options?: WaitForEventOptions): Promise<T | undefined>;
+}
+/**
+ * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
+ * Branded by string rather than `instanceof` so two copies of this module still agree.
+ */
+export declare class StepSuspension extends Error {
+    static readonly brand = "ultimate.jobs.suspension";
+    readonly brand: string;
+    readonly step: string;
+    readonly resumeAt: number;
+    readonly reason: 'sleep' | 'event';
+    constructor(input: {
+        step: string;
+        resumeAt: number;
+        reason: 'sleep' | 'event';
+    });
+}
+export declare function isStepSuspension(error: unknown): error is StepSuspension;
+export declare function createMemoryStepStore(): StepStore;
+export interface StepRunnerOptions {
+    readonly runId: string;
+    readonly jobName: string;
+    readonly store: StepStore;
+    readonly clock?: Clock;
+    readonly events?: EventLookup;
+    /** How long a waiting step stays parked between event polls. Default 30s. */
+    readonly eventPollMs?: number;
+    /** Per-step ceiling; the job-level timeout is enforced by the worker. */
+    readonly stepTimeoutMs?: number;
+}
+export interface StepRunner {
+    readonly step: StepApi;
+    /** Names used in THIS attempt, in order — the trace shown by `x jobs show`. */
+    usedNames(): readonly string[];
+    /** Names that were served from storage instead of executed. */
+    replayedNames(): readonly string[];
+}
+export declare function createStepRunner(options: StepRunnerOptions): StepRunner;
+//# sourceMappingURL=steps.d.ts.map

package/src/steps.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"steps.d.ts","sourceRoot":"","sources":["steps.ts"],"names":[],"mappings":"AAQA,OAAO,KAAK,EAAE,KAAK,EAAE,MAAM,gBAAgB,CAAC;AAE5C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,SAAS,CAAC;AAI7C,MAAM,MAAM,UAAU,GAAG,WAAW,GAAG,UAAU,GAAG,SAAS,GAAG,QAAQ,CAAC;AAEzE,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,MAAM,EAAE,UAAU,CAAC;IAC5B,6EAA6E;IAC7E,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC;IAC1B,QAAQ,CAAC,SAAS,EAAE,MAAM,CAAC;IAC3B,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,kEAAkE;IAClE,QAAQ,CAAC,MAAM,CAAC,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,QAAQ,CAAC,cAAc,CAAC,EAAE,MAAM,CAAC;IACjC,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;CACzB;AAED,MAAM,WAAW,SAAS;IACxB,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC,CAAC;IAClE,GAAG,CAAC,MAAM,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IACvC,IAAI,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,SAAS,UAAU,EAAE,CAAC,CAAC;IACpD,GAAG,CAAC,KAAK,EAAE,MAAM,EAAE,IAAI,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChD,KAAK,CAAC,KAAK,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;CACrC;AAED,mFAAmF;AACnF,MAAM,WAAW,WAAW;IAC1B,IAAI,CACF,KAAK,EAAE,MAAM,EACb,cAAc,EAAE,MAAM,GAAG,SAAS,EAClC,OAAO,EAAE,MAAM,GACd,OAAO,CAAC;QAAE,QAAQ,CAAC,OAAO,EAAE,OAAO,CAAC;QAAC,QAAQ,CAAC,WAAW,EAAE,MAAM,CAAA;KAAE,GAAG,SAAS,CAAC,CAAC;CACrF;AAED,MAAM,WAAW,mBAAmB;IAClC,sDAAsD;IACtD,QAAQ,CAAC,OAAO,CAAC,EAAE,aAAa,CAAC;IACjC,6EAA6E;IAC7E,QAAQ,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC;IACxB,8FAA8F;IAC9F,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC;CAC7B;AAED,MAAM,WAAW,OAAO;IACtB,yFAAyF;IACzF,GAAG,CAAC,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,EAAE,EAAE,MAAM,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,CAAC;IAC3D,kFAAkF;IAClF,KAAK,CAAC,IAAI,EAAE,MAAM,EAAE,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5D,KAAK,CAAC,QAAQ,EAAE,aAAa,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAC9C,YAAY,CAAC,CAAC,EACZ,IAAI,EAAE,MAAM,EACZ,KAAK,EAAE,MAAM,EACb,OAAO,CAAC,EAAE,mBAAmB,GAC5B,OAAO,CAAC,CAAC,GAAG,SAAS,CAAC,CAAC;CAC3B;AAED;;;GAGG;AACH,qBAAa,cAAe,SAAQ,KAAK;IACvC,MAAM,CAAC,QAAQ,CAAC,KAAK,8BAA8B;IACnD,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAwB;IAC9C,QAAQ,CAAC,IAAI,EAAE,MAAM,CAAC;IACtB,QAAQ,CAAC,QAAQ,EAAE,MAAM,CAAC;IAC1B,QAAQ,CAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAC;IAEnC,YAAY,KAAK,EAAE;QAAE,IAAI,EAAE,MAAM,CAAC;QAAC,QAAQ,EAAE,MAAM,CAAC;QAAC,MAAM,EAAE,OAAO,GAAG,OAAO,CAAA;KAAE,EAM/E;CACF;AAED,wBAAgB,gBAAgB,CAAC,KAAK,EAAE,OAAO,GAAG,KAAK,IAAI,cAAc,CAExE;AAED,wBAAgB,qBAAqB,IAAI,SAAS,CA+BjD;AAED,MAAM,WAAW,iBAAiB;IAChC,QAAQ,CAAC,KAAK,EAAE,MAAM,CAAC;IACvB,QAAQ,CAAC,OAAO,EAAE,MAAM,CAAC;IACzB,QAAQ,CAAC,KAAK,EAAE,SAAS,CAAC;IAC1B,QAAQ,CAAC,KAAK,CAAC,EAAE,KAAK,CAAC;IACvB,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IAC9B,6EAA6E;IAC7E,QAAQ,CAAC,WAAW,CAAC,EAAE,MAAM,CAAC;IAC9B,yEAAyE;IACzE,QAAQ,CAAC,aAAa,CAAC,EAAE,MAAM,CAAC;CACjC;AAED,MAAM,WAAW,UAAU;IACzB,QAAQ,CAAC,IAAI,EAAE,OAAO,CAAC;IACvB,+EAA+E;IAC/E,SAAS,IAAI,SAAS,MAAM,EAAE,CAAC;IAC/B,+DAA+D;IAC/D,aAAa,IAAI,SAAS,MAAM,EAAE,CAAC;CACpC;AAED,wBAAgB,gBAAgB,CAAC,OAAO,EAAE,iBAAiB,GAAG,UAAU,CAyKvE"}

package/src/steps.js

@@ -0,0 +1,227 @@
+// Durable steps — the heart of the package. Each step's result is PERSISTED before the next
+// one starts, so a retry replays completed steps from storage instead of re-executing them:
+// the welcome email is not sent twice because step 3 failed. That makes step names the replay
+// key, which is why they must be deterministic and unique within a run (X_STEP_DUPLICATE).
+//
+// Suspension (sleep / waitForEvent) unwinds the run by throwing a StepSuspension. The worker
+// catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
+import { logger } from '@ultimat3/core';
+import { nowMs, toMs } from './clock';
+import { JobTimeoutError, StepDuplicateError } from './errors';
+/**
+ * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
+ * Branded by string rather than `instanceof` so two copies of this module still agree.
+ */
+export class StepSuspension extends Error {
+    static brand = 'ultimate.jobs.suspension';
+    brand = StepSuspension.brand;
+    step;
+    resumeAt;
+    reason;
+    constructor(input) {
+        super(`step "${input.step}" suspended until ${new Date(input.resumeAt).toISOString()}`);
+        this.name = 'StepSuspension';
+        this.step = input.step;
+        this.resumeAt = input.resumeAt;
+        this.reason = input.reason;
+    }
+}
+export function isStepSuspension(error) {
+    return error instanceof Error && error.brand === StepSuspension.brand;
+}
+export function createMemoryStepStore() {
+    const byRun = new Map();
+    const runOf = (runId) => {
+        let run = byRun.get(runId);
+        if (run === undefined) {
+            run = new Map();
+            byRun.set(runId, run);
+        }
+        return run;
+    };
+    return {
+        get(runId, name) {
+            return Promise.resolve(byRun.get(runId)?.get(name));
+        },
+        put(record) {
+            runOf(record.runId).set(record.name, record);
+            return Promise.resolve();
+        },
+        list(runId) {
+            const records = [...(byRun.get(runId)?.values() ?? [])];
+            return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
+        },
+        del(runId, name) {
+            byRun.get(runId)?.delete(name);
+            return Promise.resolve();
+        },
+        clear(runId) {
+            byRun.delete(runId);
+            return Promise.resolve();
+        },
+    };
+}
+export function createStepRunner(options) {
+    const { runId, jobName, store } = options;
+    const used = [];
+    const replayed = [];
+    const clock = options.clock;
+    const pollMs = options.eventPollMs ?? 30_000;
+    const claimName = (name) => {
+        if (used.includes(name))
+            throw new StepDuplicateError({ job: jobName, step: name });
+        used.push(name);
+    };
+    const now = () => nowMs(clock);
+    async function run(name, fn) {
+        claimName(name);
+        const existing = await store.get(runId, name);
+        if (existing?.status === 'completed') {
+            replayed.push(name);
+            return existing.output;
+        }
+        const startedAt = existing?.startedAt ?? now();
+        const attempts = (existing?.attempts ?? 0) + 1;
+        try {
+            const output = await withStepTimeout(fn(), options.stepTimeoutMs, () => new JobTimeoutError({ job: jobName, step: name, timeoutMs: options.stepTimeoutMs ?? 0 }));
+            // Persist BEFORE returning: a crash one line later must not re-run this step.
+            await store.put({
+                runId,
+                name,
+                status: 'completed',
+                output,
+                startedAt,
+                completedAt: now(),
+                attempts,
+            });
+            return output;
+        }
+        catch (error) {
+            if (isStepSuspension(error))
+                throw error;
+            await store.put({
+                runId,
+                name,
+                status: 'failed',
+                startedAt,
+                attempts,
+                error: error instanceof Error ? error.message : String(error),
+            });
+            throw error;
+        }
+    }
+    async function sleep(a, b) {
+        // `sleep('3d')` — the duration doubles as the step name, which stays deterministic.
+        const name = b === undefined ? `sleep:${String(a)}` : String(a);
+        const duration = b === undefined ? a : b;
+        claimName(name);
+        const existing = await store.get(runId, name);
+        if (existing?.status === 'completed') {
+            replayed.push(name);
+            return;
+        }
+        const at = now();
+        if (existing?.status === 'sleeping' && existing.wakeAt !== undefined) {
+            if (existing.wakeAt <= at) {
+                await store.put({ ...existing, status: 'completed', completedAt: at });
+                return;
+            }
+            throw new StepSuspension({ step: name, resumeAt: existing.wakeAt, reason: 'sleep' });
+        }
+        const wakeAt = at + toMs(duration);
+        await store.put({
+            runId,
+            name,
+            status: 'sleeping',
+            startedAt: at,
+            wakeAt,
+            attempts: 1,
+        });
+        throw new StepSuspension({ step: name, resumeAt: wakeAt, reason: 'sleep' });
+    }
+    async function waitForEvent(name, event, waitOptions = {}) {
+        claimName(name);
+        const existing = await store.get(runId, name);
+        if (existing?.status === 'completed') {
+            replayed.push(name);
+            return existing.output;
+        }
+        const at = now();
+        const startedAt = existing?.startedAt ?? at;
+        const deadline = startedAt + toMs(waitOptions.timeout ?? 86_400_000);
+        const correlationKey = waitOptions.match;
+        const hit = await options.events?.find(event, correlationKey, startedAt);
+        if (hit !== undefined) {
+            await store.put({
+                runId,
+                name,
+                status: 'completed',
+                output: hit.payload,
+                startedAt,
+                completedAt: at,
+                event,
+                attempts: (existing?.attempts ?? 0) + 1,
+                ...(correlationKey === undefined ? {} : { correlationKey }),
+            });
+            return hit.payload;
+        }
+        if (at >= deadline) {
+            if (waitOptions.required === true) {
+                throw new JobTimeoutError({
+                    job: jobName,
+                    step: name,
+                    timeoutMs: toMs(waitOptions.timeout ?? 86_400_000),
+                });
+            }
+            logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
+            await store.put({
+                runId,
+                name,
+                status: 'completed',
+                output: undefined,
+                startedAt,
+                completedAt: at,
+                event,
+                attempts: (existing?.attempts ?? 0) + 1,
+            });
+            return undefined;
+        }
+        const resumeAt = Math.min(deadline, at + pollMs);
+        await store.put({
+            runId,
+            name,
+            status: 'waiting',
+            startedAt,
+            wakeAt: resumeAt,
+            event,
+            attempts: (existing?.attempts ?? 0) + 1,
+            ...(correlationKey === undefined ? {} : { correlationKey }),
+        });
+        throw new StepSuspension({ step: name, resumeAt, reason: 'event' });
+    }
+    const step = {
+        run,
+        sleep: sleep,
+        waitForEvent,
+    };
+    return {
+        step,
+        usedNames: () => [...used],
+        replayedNames: () => [...replayed],
+    };
+}
+function withStepTimeout(work, timeoutMs, error) {
+    if (timeoutMs === undefined || timeoutMs <= 0)
+        return Promise.resolve(work);
+    return new Promise((resolve, reject) => {
+        const timer = setTimeout(() => reject(error()), timeoutMs);
+        Promise.resolve(work).then((value) => {
+            clearTimeout(timer);
+            resolve(value);
+        }, (cause) => {
+            clearTimeout(timer);
+            reject(cause);
+        });
+    });
+}
+//# sourceMappingURL=steps.js.map

package/src/steps.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"steps.js","sourceRoot":"","sources":["steps.ts"],"names":[],"mappings":"AAAA,4FAA4F;AAC5F,4FAA4F;AAC5F,8FAA8F;AAC9F,2FAA2F;AAC3F,EAAE;AACF,6FAA6F;AAC7F,+FAA+F;AAG/F,OAAO,EAAE,MAAM,EAAE,MAAM,gBAAgB,CAAC;AAExC,OAAO,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,SAAS,CAAC;AACtC,OAAO,EAAE,eAAe,EAAE,kBAAkB,EAAE,MAAM,UAAU,CAAC;AA2D/D;;;GAGG;AACH,MAAM,OAAO,cAAe,SAAQ,KAAK;IACvC,MAAM,CAAU,KAAK,GAAG,0BAA0B,CAAC;IAC1C,KAAK,GAAW,cAAc,CAAC,KAAK,CAAC;IACrC,IAAI,CAAS;IACb,QAAQ,CAAS;IACjB,MAAM,CAAoB;IAEnC,YAAY,KAAoE;QAC9E,KAAK,CAAC,SAAS,KAAK,CAAC,IAAI,qBAAqB,IAAI,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;QACxF,IAAI,CAAC,IAAI,GAAG,gBAAgB,CAAC;QAC7B,IAAI,CAAC,IAAI,GAAG,KAAK,CAAC,IAAI,CAAC;QACvB,IAAI,CAAC,QAAQ,GAAG,KAAK,CAAC,QAAQ,CAAC;QAC/B,IAAI,CAAC,MAAM,GAAG,KAAK,CAAC,MAAM,CAAC;IAC7B,CAAC;CACF;AAED,MAAM,UAAU,gBAAgB,CAAC,KAAc;IAC7C,OAAO,KAAK,YAAY,KAAK,IAAK,KAA6B,CAAC,KAAK,KAAK,cAAc,CAAC,KAAK,CAAC;AACjG,CAAC;AAED,MAAM,UAAU,qBAAqB;IACnC,MAAM,KAAK,GAAG,IAAI,GAAG,EAAmC,CAAC;IACzD,MAAM,KAAK,GAAG,CAAC,KAAa,EAA2B,EAAE;QACvD,IAAI,GAAG,GAAG,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;QAC3B,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,GAAG,GAAG,IAAI,GAAG,EAAE,CAAC;YAChB,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,GAAG,CAAC,CAAC;QACxB,CAAC;QACD,OAAO,GAAG,CAAC;IACb,CAAC,CAAC;IACF,OAAO;QACL,GAAG,CAAC,KAAK,EAAE,IAAI;YACb,OAAO,OAAO,CAAC,OAAO,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC;QACtD,CAAC;QACD,GAAG,CAAC,MAAM;YACR,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,MAAM,CAAC,IAAI,EAAE,MAAM,CAAC,CAAC;YAC7C,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,IAAI,CAAC,KAAK;YACR,MAAM,OAAO,GAAG,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;YACxD,OAAO,OAAO,CAAC,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,SAAS,GAAG,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC;QAC5E,CAAC;QACD,GAAG,CAAC,KAAK,EAAE,IAAI;YACb,KAAK,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,CAAC,IAAI,CAAC,CAAC;YAC/B,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;QACD,KAAK,CAAC,KAAK;YACT,KAAK,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,OAAO,CAAC,OAAO,EAAE,CAAC;QAC3B,CAAC;KACF,CAAC;AACJ,CAAC;AAsBD,MAAM,UAAU,gBAAgB,CAAC,OAA0B;IACzD,MAAM,EAAE,KAAK,EAAE,OAAO,EAAE,KAAK,EAAE,GAAG,OAAO,CAAC;IAC1C,MAAM,IAAI,GAAa,EAAE,CAAC;IAC1B,MAAM,QAAQ,GAAa,EAAE,CAAC;IAC9B,MAAM,KAAK,GAAG,OAAO,CAAC,KAAK,CAAC;IAC5B,MAAM,MAAM,GAAG,OAAO,CAAC,WAAW,IAAI,MAAM,CAAC;IAE7C,MAAM,SAAS,GAAG,CAAC,IAAY,EAAQ,EAAE;QACvC,IAAI,IAAI,CAAC,QAAQ,CAAC,IAAI,CAAC;YAAE,MAAM,IAAI,kBAAkB,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,CAAC,CAAC;QACpF,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAClB,CAAC,CAAC;IAEF,MAAM,GAAG,GAAG,GAAW,EAAE,CAAC,KAAK,CAAC,KAAK,CAAC,CAAC;IAEvC,KAAK,UAAU,GAAG,CAAI,IAAY,EAAE,EAAwB;QAC1D,SAAS,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,OAAO,QAAQ,CAAC,MAAW,CAAC;QAC9B,CAAC;QAED,MAAM,SAAS,GAAG,QAAQ,EAAE,SAAS,IAAI,GAAG,EAAE,CAAC;QAC/C,MAAM,QAAQ,GAAG,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;QAC/C,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,eAAe,CAClC,EAAE,EAAE,EACJ,OAAO,CAAC,aAAa,EACrB,GAAG,EAAE,CACH,IAAI,eAAe,CAAC,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,SAAS,EAAE,OAAO,CAAC,aAAa,IAAI,CAAC,EAAE,CAAC,CAC3F,CAAC;YACF,8EAA8E;YAC9E,MAAM,KAAK,CAAC,GAAG,CAAC;gBACd,KAAK;gBACL,IAAI;gBACJ,MAAM,EAAE,WAAW;gBACnB,MAAM;gBACN,SAAS;gBACT,WAAW,EAAE,GAAG,EAAE;gBAClB,QAAQ;aACT,CAAC,CAAC;YACH,OAAO,MAAM,CAAC;QAChB,CAAC;QAAC,OAAO,KAAK,EAAE,CAAC;YACf,IAAI,gBAAgB,CAAC,KAAK,CAAC;gBAAE,MAAM,KAAK,CAAC;YACzC,MAAM,KAAK,CAAC,GAAG,CAAC;gBACd,KAAK;gBACL,IAAI;gBACJ,MAAM,EAAE,QAAQ;gBAChB,SAAS;gBACT,QAAQ;gBACR,KAAK,EAAE,KAAK,YAAY,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,KAAK,CAAC;aAC9D,CAAC,CAAC;YACH,MAAM,KAAK,CAAC;QACd,CAAC;IACH,CAAC;IAED,KAAK,UAAU,KAAK,CAAC,CAAyB,EAAE,CAAiB;QAC/D,oFAAoF;QACpF,MAAM,IAAI,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAC,SAAS,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC;QAChE,MAAM,QAAQ,GAAG,CAAC,KAAK,SAAS,CAAC,CAAC,CAAE,CAAmB,CAAC,CAAC,CAAC,CAAC,CAAC;QAC5D,SAAS,CAAC,IAAI,CAAC,CAAC;QAEhB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,OAAO;QACT,CAAC;QAED,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QACjB,IAAI,QAAQ,EAAE,MAAM,KAAK,UAAU,IAAI,QAAQ,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;YACrE,IAAI,QAAQ,CAAC,MAAM,IAAI,EAAE,EAAE,CAAC;gBAC1B,MAAM,KAAK,CAAC,GAAG,CAAC,EAAE,GAAG,QAAQ,EAAE,MAAM,EAAE,WAAW,EAAE,WAAW,EAAE,EAAE,EAAE,CAAC,CAAC;gBACvE,OAAO;YACT,CAAC;YACD,MAAM,IAAI,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;QACvF,CAAC;QAED,MAAM,MAAM,GAAG,EAAE,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC;QACnC,MAAM,KAAK,CAAC,GAAG,CAAC;YACd,KAAK;YACL,IAAI;YACJ,MAAM,EAAE,UAAU;YAClB,SAAS,EAAE,EAAE;YACb,MAAM;YACN,QAAQ,EAAE,CAAC;SACZ,CAAC,CAAC;QACH,MAAM,IAAI,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IAC9E,CAAC;IAED,KAAK,UAAU,YAAY,CACzB,IAAY,EACZ,KAAa,EACb,WAAW,GAAwB,EAAE;QAErC,SAAS,CAAC,IAAI,CAAC,CAAC;QAChB,MAAM,QAAQ,GAAG,MAAM,KAAK,CAAC,GAAG,CAAC,KAAK,EAAE,IAAI,CAAC,CAAC;QAC9C,IAAI,QAAQ,EAAE,MAAM,KAAK,WAAW,EAAE,CAAC;YACrC,QAAQ,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;YACpB,OAAO,QAAQ,CAAC,MAAuB,CAAC;QAC1C,CAAC;QAED,MAAM,EAAE,GAAG,GAAG,EAAE,CAAC;QACjB,MAAM,SAAS,GAAG,QAAQ,EAAE,SAAS,IAAI,EAAE,CAAC;QAC5C,MAAM,QAAQ,GAAG,SAAS,GAAG,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,UAAU,CAAC,CAAC;QACrE,MAAM,cAAc,GAAG,WAAW,CAAC,KAAK,CAAC;QAEzC,MAAM,GAAG,GAAG,MAAM,OAAO,CAAC,MAAM,EAAE,IAAI,CAAC,KAAK,EAAE,cAAc,EAAE,SAAS,CAAC,CAAC;QACzE,IAAI,GAAG,KAAK,SAAS,EAAE,CAAC;YACtB,MAAM,KAAK,CAAC,GAAG,CAAC;gBACd,KAAK;gBACL,IAAI;gBACJ,MAAM,EAAE,WAAW;gBACnB,MAAM,EAAE,GAAG,CAAC,OAAO;gBACnB,SAAS;gBACT,WAAW,EAAE,EAAE;gBACf,KAAK;gBACL,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC;gBACvC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;aAC5D,CAAC,CAAC;YACH,OAAO,GAAG,CAAC,OAAY,CAAC;QAC1B,CAAC;QAED,IAAI,EAAE,IAAI,QAAQ,EAAE,CAAC;YACnB,IAAI,WAAW,CAAC,QAAQ,KAAK,IAAI,EAAE,CAAC;gBAClC,MAAM,IAAI,eAAe,CAAC;oBACxB,GAAG,EAAE,OAAO;oBACZ,IAAI,EAAE,IAAI;oBACV,SAAS,EAAE,IAAI,CAAC,WAAW,CAAC,OAAO,IAAI,UAAU,CAAC;iBACnD,CAAC,CAAC;YACL,CAAC;YACD,MAAM,CAAC,IAAI,CAAC,wBAAwB,EAAE,EAAE,GAAG,EAAE,OAAO,EAAE,IAAI,EAAE,IAAI,EAAE,KAAK,EAAE,CAAC,CAAC;YAC3E,MAAM,KAAK,CAAC,GAAG,CAAC;gBACd,KAAK;gBACL,IAAI;gBACJ,MAAM,EAAE,WAAW;gBACnB,MAAM,EAAE,SAAS;gBACjB,SAAS;gBACT,WAAW,EAAE,EAAE;gBACf,KAAK;gBACL,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC;aACxC,CAAC,CAAC;YACH,OAAO,SAAS,CAAC;QACnB,CAAC;QAED,MAAM,QAAQ,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,EAAE,EAAE,GAAG,MAAM,CAAC,CAAC;QACjD,MAAM,KAAK,CAAC,GAAG,CAAC;YACd,KAAK;YACL,IAAI;YACJ,MAAM,EAAE,SAAS;YACjB,SAAS;YACT,MAAM,EAAE,QAAQ;YAChB,KAAK;YACL,QAAQ,EAAE,CAAC,QAAQ,EAAE,QAAQ,IAAI,CAAC,CAAC,GAAG,CAAC;YACvC,GAAG,CAAC,cAAc,KAAK,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,EAAE,cAAc,EAAE,CAAC;SAC5D,CAAC,CAAC;QACH,MAAM,IAAI,cAAc,CAAC,EAAE,IAAI,EAAE,IAAI,EAAE,QAAQ,EAAE,MAAM,EAAE,OAAO,EAAE,CAAC,CAAC;IACtE,CAAC;IAED,MAAM,IAAI,GAAY;QACpB,GAAG;QACH,KAAK,EAAE,KAAyB;QAChC,YAAY;KACb,CAAC;IAEF,OAAO;QACL,IAAI;QACJ,SAAS,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,IAAI,CAAC;QAC1B,aAAa,EAAE,GAAG,EAAE,CAAC,CAAC,GAAG,QAAQ,CAAC;KACnC,CAAC;AACJ,CAAC;AAED,SAAS,eAAe,CACtB,IAAoB,EACpB,SAA6B,EAC7B,KAAkB;IAElB,IAAI,SAAS,KAAK,SAAS,IAAI,SAAS,IAAI,CAAC;QAAE,OAAO,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC;IAC5E,OAAO,IAAI,OAAO,CAAI,CAAC,OAAO,EAAE,MAAM,EAAE,EAAE;QACxC,MAAM,KAAK,GAAG,UAAU,CAAC,GAAG,EAAE,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,EAAE,SAAS,CAAC,CAAC;QAC3D,OAAO,CAAC,OAAO,CAAC,IAAI,CAAC,CAAC,IAAI,CACxB,CAAC,KAAK,EAAE,EAAE;YACR,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,OAAO,CAAC,KAAK,CAAC,CAAC;QACjB,CAAC,EACD,CAAC,KAAK,EAAE,EAAE;YACR,YAAY,CAAC,KAAK,CAAC,CAAC;YACpB,MAAM,CAAC,KAAK,CAAC,CAAC;QAChB,CAAC,CACF,CAAC;IACJ,CAAC,CAAC,CAAC;AACL,CAAC"}

package/src/steps.ts

@@ -0,0 +1,339 @@
+// Durable steps — the heart of the package. Each step's result is PERSISTED before the next
+// one starts, so a retry replays completed steps from storage instead of re-executing them:
+// the welcome email is not sent twice because step 3 failed. That makes step names the replay
+// key, which is why they must be deterministic and unique within a run (X_STEP_DUPLICATE).
+//
+// Suspension (sleep / waitForEvent) unwinds the run by throwing a StepSuspension. The worker
+// catches it and re-queues the job for `resumeAt` instead of holding a process for three days.
+
+import type { Clock } from '@ultimat3/core';
+import { logger } from '@ultimat3/core';
+import type { DurationInput } from './clock';
+import { nowMs, toMs } from './clock';
+import { JobTimeoutError, StepDuplicateError } from './errors';
+
+export type StepStatus = 'completed' | 'sleeping' | 'waiting' | 'failed';
+
+export interface StepRecord {
+  readonly runId: string;
+  readonly name: string;
+  readonly status: StepStatus;
+  /** The memoized return value. Present only when `status === 'completed'`. */
+  readonly output?: unknown;
+  readonly startedAt: number;
+  readonly completedAt?: number;
+  /** Epoch ms at which a sleeping/waiting step becomes runnable. */
+  readonly wakeAt?: number;
+  readonly event?: string;
+  readonly correlationKey?: string;
+  readonly attempts: number;
+  readonly error?: string;
+}
+
+export interface StepStore {
+  get(runId: string, name: string): Promise<StepRecord | undefined>;
+  put(record: StepRecord): Promise<void>;
+  list(runId: string): Promise<readonly StepRecord[]>;
+  del(runId: string, name: string): Promise<void>;
+  clear(runId: string): Promise<void>;
+}
+
+/** The slice of the event bus a waiting step needs. Implemented by `events.ts`. */
+export interface EventLookup {
+  find(
+    event: string,
+    correlationKey: string | undefined,
+    afterMs: number,
+  ): Promise<{ readonly payload: unknown; readonly publishedAt: number } | undefined>;
+}
+
+export interface WaitForEventOptions {
+  /** How long to wait before giving up. Default 24h. */
+  readonly timeout?: DurationInput;
+  /** Correlation key the published event must carry — usually an entity id. */
+  readonly match?: string;
+  /** Default false: a timeout resolves `undefined`. `true` fails the job with X_JOB_TIMEOUT. */
+  readonly required?: boolean;
+}
+
+export interface StepApi {
+  /** Run once, ever. On replay the persisted output is returned and `fn` is not called. */
+  run<T>(name: string, fn: () => Promise<T> | T): Promise<T>;
+  /** Suspend the run. `sleep(duration)` derives the step name from the duration. */
+  sleep(name: string, duration: DurationInput): Promise<void>;
+  sleep(duration: DurationInput): Promise<void>;
+  waitForEvent<T>(
+    name: string,
+    event: string,
+    options?: WaitForEventOptions,
+  ): Promise<T | undefined>;
+}
+
+/**
+ * Control flow, not a failure: it must never be logged as an error or counted as an attempt.
+ * Branded by string rather than `instanceof` so two copies of this module still agree.
+ */
+export class StepSuspension extends Error {
+  static readonly brand = 'ultimate.jobs.suspension';
+  readonly brand: string = StepSuspension.brand;
+  readonly step: string;
+  readonly resumeAt: number;
+  readonly reason: 'sleep' | 'event';
+
+  constructor(input: { step: string; resumeAt: number; reason: 'sleep' | 'event' }) {
+    super(`step "${input.step}" suspended until ${new Date(input.resumeAt).toISOString()}`);
+    this.name = 'StepSuspension';
+    this.step = input.step;
+    this.resumeAt = input.resumeAt;
+    this.reason = input.reason;
+  }
+}
+
+export function isStepSuspension(error: unknown): error is StepSuspension {
+  return error instanceof Error && (error as { brand?: unknown }).brand === StepSuspension.brand;
+}
+
+export function createMemoryStepStore(): StepStore {
+  const byRun = new Map<string, Map<string, StepRecord>>();
+  const runOf = (runId: string): Map<string, StepRecord> => {
+    let run = byRun.get(runId);
+    if (run === undefined) {
+      run = new Map();
+      byRun.set(runId, run);
+    }
+    return run;
+  };
+  return {
+    get(runId, name) {
+      return Promise.resolve(byRun.get(runId)?.get(name));
+    },
+    put(record) {
+      runOf(record.runId).set(record.name, record);
+      return Promise.resolve();
+    },
+    list(runId) {
+      const records = [...(byRun.get(runId)?.values() ?? [])];
+      return Promise.resolve(records.sort((a, b) => a.startedAt - b.startedAt));
+    },
+    del(runId, name) {
+      byRun.get(runId)?.delete(name);
+      return Promise.resolve();
+    },
+    clear(runId) {
+      byRun.delete(runId);
+      return Promise.resolve();
+    },
+  };
+}
+
+export interface StepRunnerOptions {
+  readonly runId: string;
+  readonly jobName: string;
+  readonly store: StepStore;
+  readonly clock?: Clock;
+  readonly events?: EventLookup;
+  /** How long a waiting step stays parked between event polls. Default 30s. */
+  readonly eventPollMs?: number;
+  /** Per-step ceiling; the job-level timeout is enforced by the worker. */
+  readonly stepTimeoutMs?: number;
+}
+
+export interface StepRunner {
+  readonly step: StepApi;
+  /** Names used in THIS attempt, in order — the trace shown by `x jobs show`. */
+  usedNames(): readonly string[];
+  /** Names that were served from storage instead of executed. */
+  replayedNames(): readonly string[];
+}
+
+export function createStepRunner(options: StepRunnerOptions): StepRunner {
+  const { runId, jobName, store } = options;
+  const used: string[] = [];
+  const replayed: string[] = [];
+  const clock = options.clock;
+  const pollMs = options.eventPollMs ?? 30_000;
+
+  const claimName = (name: string): void => {
+    if (used.includes(name)) throw new StepDuplicateError({ job: jobName, step: name });
+    used.push(name);
+  };
+
+  const now = (): number => nowMs(clock);
+
+  async function run<T>(name: string, fn: () => Promise<T> | T): Promise<T> {
+    claimName(name);
+    const existing = await store.get(runId, name);
+    if (existing?.status === 'completed') {
+      replayed.push(name);
+      return existing.output as T;
+    }
+
+    const startedAt = existing?.startedAt ?? now();
+    const attempts = (existing?.attempts ?? 0) + 1;
+    try {
+      const output = await withStepTimeout(
+        fn(),
+        options.stepTimeoutMs,
+        () =>
+          new JobTimeoutError({ job: jobName, step: name, timeoutMs: options.stepTimeoutMs ?? 0 }),
+      );
+      // Persist BEFORE returning: a crash one line later must not re-run this step.
+      await store.put({
+        runId,
+        name,
+        status: 'completed',
+        output,
+        startedAt,
+        completedAt: now(),
+        attempts,
+      });
+      return output;
+    } catch (error) {
+      if (isStepSuspension(error)) throw error;
+      await store.put({
+        runId,
+        name,
+        status: 'failed',
+        startedAt,
+        attempts,
+        error: error instanceof Error ? error.message : String(error),
+      });
+      throw error;
+    }
+  }
+
+  async function sleep(a: string | DurationInput, b?: DurationInput): Promise<void> {
+    // `sleep('3d')` — the duration doubles as the step name, which stays deterministic.
+    const name = b === undefined ? `sleep:${String(a)}` : String(a);
+    const duration = b === undefined ? (a as DurationInput) : b;
+    claimName(name);
+
+    const existing = await store.get(runId, name);
+    if (existing?.status === 'completed') {
+      replayed.push(name);
+      return;
+    }
+
+    const at = now();
+    if (existing?.status === 'sleeping' && existing.wakeAt !== undefined) {
+      if (existing.wakeAt <= at) {
+        await store.put({ ...existing, status: 'completed', completedAt: at });
+        return;
+      }
+      throw new StepSuspension({ step: name, resumeAt: existing.wakeAt, reason: 'sleep' });
+    }
+
+    const wakeAt = at + toMs(duration);
+    await store.put({
+      runId,
+      name,
+      status: 'sleeping',
+      startedAt: at,
+      wakeAt,
+      attempts: 1,
+    });
+    throw new StepSuspension({ step: name, resumeAt: wakeAt, reason: 'sleep' });
+  }
+
+  async function waitForEvent<T>(
+    name: string,
+    event: string,
+    waitOptions: WaitForEventOptions = {},
+  ): Promise<T | undefined> {
+    claimName(name);
+    const existing = await store.get(runId, name);
+    if (existing?.status === 'completed') {
+      replayed.push(name);
+      return existing.output as T | undefined;
+    }
+
+    const at = now();
+    const startedAt = existing?.startedAt ?? at;
+    const deadline = startedAt + toMs(waitOptions.timeout ?? 86_400_000);
+    const correlationKey = waitOptions.match;
+
+    const hit = await options.events?.find(event, correlationKey, startedAt);
+    if (hit !== undefined) {
+      await store.put({
+        runId,
+        name,
+        status: 'completed',
+        output: hit.payload,
+        startedAt,
+        completedAt: at,
+        event,
+        attempts: (existing?.attempts ?? 0) + 1,
+        ...(correlationKey === undefined ? {} : { correlationKey }),
+      });
+      return hit.payload as T;
+    }
+
+    if (at >= deadline) {
+      if (waitOptions.required === true) {
+        throw new JobTimeoutError({
+          job: jobName,
+          step: name,
+          timeoutMs: toMs(waitOptions.timeout ?? 86_400_000),
+        });
+      }
+      logger.warn('jobs.step.wait-timeout', { job: jobName, step: name, event });
+      await store.put({
+        runId,
+        name,
+        status: 'completed',
+        output: undefined,
+        startedAt,
+        completedAt: at,
+        event,
+        attempts: (existing?.attempts ?? 0) + 1,
+      });
+      return undefined;
+    }
+
+    const resumeAt = Math.min(deadline, at + pollMs);
+    await store.put({
+      runId,
+      name,
+      status: 'waiting',
+      startedAt,
+      wakeAt: resumeAt,
+      event,
+      attempts: (existing?.attempts ?? 0) + 1,
+      ...(correlationKey === undefined ? {} : { correlationKey }),
+    });
+    throw new StepSuspension({ step: name, resumeAt, reason: 'event' });
+  }
+
+  const step: StepApi = {
+    run,
+    sleep: sleep as StepApi['sleep'],
+    waitForEvent,
+  };
+
+  return {
+    step,
+    usedNames: () => [...used],
+    replayedNames: () => [...replayed],
+  };
+}
+
+function withStepTimeout<T>(
+  work: Promise<T> | T,
+  timeoutMs: number | undefined,
+  error: () => Error,
+): Promise<T> {
+  if (timeoutMs === undefined || timeoutMs <= 0) return Promise.resolve(work);
+  return new Promise<T>((resolve, reject) => {
+    const timer = setTimeout(() => reject(error()), timeoutMs);
+    Promise.resolve(work).then(
+      (value) => {
+        clearTimeout(timer);
+        resolve(value);
+      },
+      (cause) => {
+        clearTimeout(timer);
+        reject(cause);
+      },
+    );
+  });
+}