@venturekit-pro/ai 0.0.48 → 0.0.50

package/dist/harness/harness.js

@@ -0,0 +1,1287 @@
+/**
+ * `createHarness` — the durable runtime for graphs.
+ *
+ * One task per node attempt (`{ runId, step, attempt }`):
+ *
+ *   lease (CAS queued → running, `node.started`)
+ *   → run the node (heartbeat renews the lease; a cancel poll and the
+ *     node timeout abort `ctx.signal`)
+ *   → ONE compare-and-set commit: output, next node, events, node-run row
+ *   → schedule the next task
+ *
+ * A gate is reached in the same commit (status `waiting`, nothing in
+ * flight) and left by `resume()`. A failed attempt is retried with
+ * backoff unless it is fatal (`NodeFatalError`, a gateway policy denial,
+ * a 4xx caller error) or the node's attempts are used up. Duplicate or
+ * late deliveries are no-ops (`stale` / `busy`). `sweep()` re-sends tasks
+ * that were lost or held back, and fails the attempt of a worker whose
+ * lease expired.
+ */
+import { randomUUID } from 'node:crypto';
+import { AgentRunError, runAgent } from '../agent-loop/run-agent.js';
+import { round8 } from '../chat/cost-util.js';
+import { GatewayPolicyError } from '../gateway/errors.js';
+import { attributes, inSpan, SPAN_KIND, SPAN_STATUS } from '../tracing/index.js';
+import { NodeFatalError } from './define.js';
+import { toRunRecord, } from './store.js';
+const DEFAULT_NODE_TIMEOUT_MS = 10 * 60_000;
+const DEFAULT_MAX_STEPS = 100;
+const DEFAULT_MAX_ATTEMPTS = 3;
+const DEFAULT_MAP_CONCURRENCY = 4;
+/** A due queued task is re-sent once it has been quiet this long (or `staleAfterMs`, if shorter). */
+const SWEEP_QUIET_MS = 15_000;
+const SWEEP_BATCH = 200;
+/** Misuse of the harness API (unknown run, wrong status, bad decision, lost race). */
+export class HarnessError extends Error {
+    code;
+    constructor(code, message) {
+        super(message);
+        this.name = 'HarnessError';
+        this.code = code;
+    }
+}
+class RunCancelledError extends Error {
+    constructor() {
+        super('Cancelled');
+        this.name = 'AbortError';
+    }
+}
+const iso = (ms) => new Date(ms).toISOString();
+const nowIso = () => new Date().toISOString();
+const messageOf = (err) => (err instanceof Error ? err.message : String(err));
+const sleep = (ms) => new Promise((resolve) => setTimeout(resolve, ms));
+function isFatal(err) {
+    if (err instanceof NodeFatalError || err instanceof GatewayPolicyError)
+        return true;
+    if (err instanceof AgentRunError && err.cause !== undefined && err.cause !== err)
+        return isFatal(err.cause);
+    const status = err?.statusCode;
+    return (typeof status === 'number' && status >= 400 && status < 500 && status !== 408 && status !== 429);
+}
+const timeoutError = (node, timeoutMs) => Object.assign(new Error(`'${node.label}' timed out after ${Math.round(timeoutMs / 1000)} s`), {
+    category: 'timeout',
+});
+const RETRY_CATEGORIES = [
+    'rate_limit',
+    'server',
+    'timeout',
+    'network',
+    'schema',
+    'other',
+];
+/** What kind of failure `err` is, for `retry.on` (an `AgentRunError` is judged by its cause). */
+export function retryCategory(err) {
+    const e = err instanceof AgentRunError && err.cause !== undefined ? err.cause : err;
+    const declared = e?.category;
+    if (typeof declared === 'string' && RETRY_CATEGORIES.includes(declared))
+        return declared;
+    const message = messageOf(e);
+    const statusCode = e?.statusCode ??
+        e?.status;
+    const status = typeof statusCode === 'number'
+        ? statusCode
+        : Number(/\]\s+(\d{3})\b/.exec(message)?.[1] ?? NaN);
+    if (status === 429)
+        return 'rate_limit';
+    if (status === 408 || status === 504)
+        return 'timeout';
+    if (status >= 500 && status < 600)
+        return 'server';
+    const name = e?.name;
+    if (name === 'AbortError' || name === 'TimeoutError' || /timed out|timeout/i.test(message))
+        return 'timeout';
+    if (/ECONNRESET|ECONNREFUSED|ENOTFOUND|EAI_AGAIN|ETIMEDOUT|EPIPE|socket hang up|fetch failed|network/i.test(message))
+        return 'network';
+    return 'other';
+}
+/** Whether the node's `retry.on` lets this (non-fatal) failure retry. */
+function retryAllowed(node, err) {
+    const on = node.retry?.on;
+    if (!on)
+        return true;
+    const category = retryCategory(err);
+    return typeof on === 'function' ? on(err, category) : on.includes(category);
+}
+function errorCode(err, fatal) {
+    const code = err?.code;
+    if (typeof code === 'string' && code)
+        return code;
+    if (err instanceof AgentRunError && err.cause !== undefined)
+        return errorCode(err.cause, fatal);
+    return fatal ? 'node_fatal' : 'node_failed';
+}
+export function createHarness(options) {
+    const { store, scheduler } = options;
+    const generateId = options.generateId ?? randomUUID;
+    const progressIntervalMs = options.progressIntervalMs ?? 1000;
+    const cancelPollMs = Math.max(1, options.cancelPollMs ?? 2000);
+    const leaseMs = options.leaseMs ?? 60_000;
+    const backoffBase = options.retryBackoffMs ?? 5000;
+    const backoffMax = options.maxRetryBackoffMs ?? 5 * 60_000;
+    const correlationId = options.correlationId ?? ((run) => run.id);
+    const registry = new Map();
+    for (const def of options.graphs) {
+        const versions = registry.get(def.key) ?? new Map();
+        if (versions.has(def.version))
+            throw new Error(`[ai/harness] graph ${def.key}@${def.version} is registered twice`);
+        versions.set(def.version, { def, nodes: new Map(def.nodes.map((n) => [n.id, n])) });
+        registry.set(def.key, versions);
+    }
+    const graphOf = (run) => registry.get(run.graphKey)?.get(run.graphVersion);
+    const event = (run, type, label, extra = {}) => ({
+        type,
+        nodeId: run.currentNode,
+        step: run.step,
+        attempt: run.attempt,
+        label,
+        detail: null,
+        data: null,
+        ...extra,
+    });
+    // ─── Transitions ─────────────────────────────────────────────────
+    /** Leave `node` with `output` towards `next`: the patch and events of the commit. */
+    function transition(run, graph, node, output, next, addedCost) {
+        const outputs = { ...run.outputs, [node.id]: output };
+        const costUsd = round8(run.costUsd + addedCost);
+        if (next === null) {
+            const patch = {
+                status: 'completed',
+                outputs,
+                currentNode: null,
+                leaseUntil: null,
+                notBefore: null,
+                error: null,
+                costUsd,
+                completedAt: nowIso(),
+            };
+            return {
+                patch,
+                events: [event(run, 'run.completed', 'Done', { nodeId: null })],
+                outcome: 'completed',
+            };
+        }
+        const nextNode = graph.nodes.get(next);
+        if (!nextNode)
+            throw new NodeFatalError(`Node '${node.id}' goes to unknown node '${next}'`, 'unknown_node');
+        const maxSteps = graph.def.maxSteps ?? DEFAULT_MAX_STEPS;
+        if (run.step + 1 > maxSteps)
+            throw new NodeFatalError(`The run passed its limit of ${maxSteps} steps`, 'max_steps');
+        const step = run.step + 1;
+        const base = {
+            outputs,
+            currentNode: next,
+            step,
+            attempt: 1,
+            attemptBase: 0,
+            notBefore: null,
+            leaseUntil: null,
+            error: null,
+            costUsd,
+        };
+        if (nextNode.kind === 'gate') {
+            return {
+                patch: { ...base, status: 'waiting' },
+                events: [
+                    {
+                        type: 'gate.waiting',
+                        nodeId: next,
+                        step,
+                        attempt: 1,
+                        label: nextNode.label,
+                        detail: null,
+                        data: { decisions: nextNode.decisions },
+                    },
+                ],
+                outcome: 'waiting',
+            };
+        }
+        return {
+            patch: { ...base, status: 'queued' },
+            events: [],
+            outcome: 'advanced',
+        };
+    }
+    const resolveNext = (node, output, ctx) => typeof node.next === 'function' ? node.next({ output, ctx }) : (node.next ?? null);
+    async function finishCancelled(run, attempt, detail = null) {
+        const nodeRun = attempt
+            ? {
+                step: run.step,
+                nodeId: attempt.node.id,
+                attempt: run.attempt,
+                status: 'cancelled',
+                startedAt: attempt.startedAt,
+                finishedAt: nowIso(),
+                output: null,
+                error: null,
+                costUsd: attempt.nodeCost,
+            }
+            : undefined;
+        const committed = await store.updateRun(run.id, run.version, {
+            status: 'cancelled',
+            cancelRequested: true,
+            leaseUntil: null,
+            notBefore: null,
+            completedAt: nowIso(),
+            costUsd: round8(run.costUsd + (attempt?.nodeCost ?? 0)),
+        }, {
+            events: [event(run, 'run.cancelled', 'Cancelled', { detail })],
+            ...(nodeRun ? { nodeRun } : {}),
+        });
+        return committed ? 'cancelled' : 'stale';
+    }
+    async function handleFailure(run, node, err, startedAt, nodeCost) {
+        const message = messageOf(err);
+        const fatal = isFatal(err);
+        const used = run.attempt - run.attemptBase;
+        const maxAttempts = node.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+        const error = {
+            message,
+            code: errorCode(err, fatal),
+            nodeId: node.id,
+            attempt: run.attempt,
+        };
+        const nodeRun = {
+            step: run.step,
+            nodeId: node.id,
+            attempt: run.attempt,
+            status: 'failed',
+            startedAt,
+            finishedAt: nowIso(),
+            output: null,
+            error,
+            costUsd: nodeCost,
+        };
+        if (!fatal && retryAllowed(node, err) && used < maxAttempts) {
+            const delayMs = Math.min(backoffMax, (node.retry?.backoffMs ?? backoffBase) * 2 ** Math.max(0, used - 1));
+            const committed = await store.updateRun(run.id, run.version, {
+                status: 'queued',
+                attempt: run.attempt + 1,
+                notBefore: iso(Date.now() + delayMs),
+                leaseUntil: null,
+                costUsd: round8(run.costUsd + nodeCost),
+            }, {
+                events: [
+                    event(run, 'node.retrying', node.label, {
+                        detail: message,
+                        data: { nextAttempt: run.attempt + 1, delayMs, code: error.code },
+                    }),
+                ],
+                nodeRun,
+            });
+            if (!committed)
+                return 'stale';
+            await scheduler.enqueue({ runId: run.id, step: run.step, attempt: run.attempt + 1 }, { delayMs });
+            return 'retrying';
+        }
+        const committed = await store.updateRun(run.id, run.version, { status: 'failed', leaseUntil: null, error, costUsd: round8(run.costUsd + nodeCost) }, {
+            events: [
+                event(run, 'node.failed', node.label, { detail: message, data: { code: error.code } }),
+                event(run, 'run.failed', 'Failed', { detail: message, data: { code: error.code } }),
+            ],
+            nodeRun,
+        });
+        return committed ? 'failed' : 'stale';
+    }
+    // ─── Node context ────────────────────────────────────────────────
+    function makeContext(run, graph, node, signal, cost, item) {
+        let lastProgress = 0;
+        let calls = 0;
+        const gatewayContext = (opts) => ({
+            tenantId: run.context.tenantId ?? null,
+            feature: opts?.feature ?? `${run.graphKey}/${node.id}`,
+            ...(run.context.actor ? { actor: run.context.actor } : {}),
+            ...(run.context.target ? { target: run.context.target } : {}),
+            correlationId: correlationId(run),
+            labels: {
+                ...run.context.labels,
+                ...opts?.labels,
+                graph: `${run.graphKey}@${run.graphVersion}`,
+                node: node.id,
+                step: run.step,
+                attempt: run.attempt,
+                ...(item ? { item: item.index } : {}),
+            },
+        });
+        /** Idempotency key base: one per node attempt, or per item attempt of a fanned-out map. */
+        const keyBase = `${run.id}:${run.step}:${run.attempt}${item ? `:i${item.index}.${item.attempt}` : ''}`;
+        const remainingBudget = () => {
+            const limits = [
+                node.maxCostUsd !== undefined ? node.maxCostUsd - cost.value : undefined,
+                graph.def.maxCostUsd !== undefined
+                    ? graph.def.maxCostUsd - run.costUsd - cost.value
+                    : undefined,
+            ].filter((v) => v !== undefined);
+            return limits.length ? Math.min(...limits) : undefined;
+        };
+        const beforeCall = () => {
+            if (signal.aborted)
+                throw new RunCancelledError();
+            const left = remainingBudget();
+            if (left !== undefined && left <= 0)
+                throw new NodeFatalError(`The spending limit for '${node.id}' is used up`, 'budget_exhausted');
+            if (!options.gateway)
+                throw new NodeFatalError('This harness has no LLM gateway', 'no_gateway');
+            calls += 1;
+            return { gateway: options.gateway, key: `${keyBase}:${calls}` };
+        };
+        const append = (events) => store.appendEvents(run.id, events);
+        const progress = async (detail, data, opts) => {
+            const now = Date.now();
+            if (!opts?.force && now - lastProgress < progressIntervalMs)
+                return;
+            lastProgress = now;
+            await append([
+                {
+                    type: 'node.progress',
+                    nodeId: node.id,
+                    step: run.step,
+                    attempt: run.attempt,
+                    label: node.label,
+                    detail,
+                    data: data ?? null,
+                },
+            ]);
+        };
+        /** Spend in USD, kept to the ledger's 8 decimals. */
+        const spend = (usd) => {
+            cost.value = round8(cost.value + usd);
+        };
+        /** The node's tier on requests that name none. */
+        const withTier = (request) => node.tier && request.tier === undefined ? { ...request, tier: node.tier } : request;
+        return {
+            run: {
+                id: run.id,
+                graphKey: run.graphKey,
+                graphVersion: run.graphVersion,
+                context: run.context,
+                createdAt: run.createdAt,
+            },
+            nodeId: node.id,
+            step: run.step,
+            attempt: run.attempt,
+            signal,
+            get costUsd() {
+                return cost.value;
+            },
+            input: () => run.input,
+            output: (id) => run.outputs[id],
+            require(id) {
+                if (!(id in run.outputs))
+                    throw new NodeFatalError(`Node '${node.id}' needs the output of '${id}', which has not run`, 'missing_output');
+                return run.outputs[id];
+            },
+            progress,
+            state: (state, opts) => progress(node.detail?.(state) ?? '', state, opts),
+            async event(type, label, data, detail) {
+                await append([
+                    {
+                        type,
+                        nodeId: node.id,
+                        step: run.step,
+                        attempt: run.attempt,
+                        label,
+                        detail: detail ?? null,
+                        data: data ?? null,
+                    },
+                ]);
+            },
+            async generate(request, opts) {
+                const { gateway, key } = beforeCall();
+                const result = await gateway.generate(gatewayContext(opts), {
+                    ...withTier(request),
+                    signal,
+                    idempotencyKey: request.idempotencyKey ?? key,
+                });
+                spend(result.cost?.costUsd ?? 0);
+                return result;
+            },
+            async *stream(request, opts) {
+                const { gateway, key } = beforeCall();
+                const call = {
+                    ...withTier(request),
+                    signal,
+                    idempotencyKey: request.idempotencyKey ?? key,
+                };
+                if (!gateway.stream) {
+                    // A gateway that can't stream: the whole answer as one chunk.
+                    const result = await gateway.generate(gatewayContext(opts), call);
+                    spend(result.cost?.costUsd ?? 0);
+                    if (result.text)
+                        yield { type: 'text', delta: result.text };
+                    yield { type: 'done', result };
+                    return;
+                }
+                for await (const e of gateway.stream(gatewayContext(opts), call)) {
+                    if (signal.aborted)
+                        throw new RunCancelledError();
+                    if (e.type === 'done')
+                        spend(e.result.cost?.costUsd ?? 0);
+                    yield e;
+                }
+            },
+            async runAgent(agentOptions, opts) {
+                const { gateway, key } = beforeCall();
+                if (!gateway.client)
+                    throw new NodeFatalError('The LLM gateway has no client() for agents', 'no_gateway');
+                const left = remainingBudget();
+                const maxCostUsd = [agentOptions.limits?.maxCostUsd, left].filter((v) => v !== undefined);
+                const onEvent = async (e) => {
+                    if (e.type === 'tool.completed') {
+                        const r = e.result;
+                        await append([
+                            {
+                                type: 'tool.called',
+                                nodeId: node.id,
+                                step: run.step,
+                                attempt: run.attempt,
+                                label: r.name,
+                                detail: r.isError ? r.content.slice(0, 300) : null,
+                                data: {
+                                    callId: r.callId,
+                                    isError: r.isError,
+                                    durationMs: r.durationMs,
+                                    ...(r.data !== undefined ? { data: r.data } : {}),
+                                },
+                            },
+                        ]).catch(() => undefined);
+                    }
+                    await agentOptions.onEvent?.(e);
+                };
+                try {
+                    const tracer = agentOptions.tracer ?? options.tracer;
+                    const result = await runAgent({
+                        ...agentOptions,
+                        request: withTier(agentOptions.request),
+                        name: agentOptions.name ?? `${run.graphKey}/${node.id}`,
+                        ...(tracer ? { tracer } : {}),
+                        ...(maxCostUsd.length
+                            ? { limits: { ...agentOptions.limits, maxCostUsd: Math.min(...maxCostUsd) } }
+                            : {}),
+                        client: gateway.client(gatewayContext(opts)),
+                        signal,
+                        idempotencyKey: key,
+                        onEvent,
+                    });
+                    // Tools in a node spend through ctx.generate / ctx.runAgent, which
+                    // count already: add only the agent's own model turns.
+                    spend((result.cost?.costUsd ?? 0) - (result.toolCost?.costUsd ?? 0));
+                    return result;
+                }
+                catch (err) {
+                    if (err instanceof AgentRunError)
+                        spend((err.snapshot.cost?.costUsd ?? 0) - (err.snapshot.toolCost?.costUsd ?? 0));
+                    throw err;
+                }
+            },
+        };
+    }
+    async function runNode(node, ctx, run) {
+        if (node.kind === 'gate')
+            throw new NodeFatalError(`Gate '${node.id}' cannot run`, 'gate');
+        if (node.kind !== 'map')
+            return node.run(ctx);
+        const items = await node.items(ctx);
+        const done = await store.getMapItems(run.id, run.step);
+        const results = new Array(items.length);
+        for (const [i, out] of done)
+            if (i < items.length)
+                results[i] = out;
+        const todo = items.map((_, i) => i).filter((i) => !done.has(i));
+        let cursor = 0;
+        let failure = null;
+        const worker = async () => {
+            while (cursor < todo.length && !failure && !ctx.signal.aborted) {
+                const index = todo[cursor++];
+                try {
+                    const out = await node.runItem(items[index], index, ctx);
+                    await store.putMapItem(run.id, run.step, index, out ?? null);
+                    results[index] = out;
+                }
+                catch (error) {
+                    failure ??= { error };
+                }
+            }
+        };
+        await Promise.all(Array.from({ length: Math.min(Math.max(1, node.concurrency ?? DEFAULT_MAP_CONCURRENCY), todo.length) }, worker));
+        if (failure)
+            throw failure.error;
+        if (ctx.signal.aborted)
+            throw new RunCancelledError();
+        return results;
+    }
+    /** One node attempt, in a `harness.node` span when a tracer is set. */
+    function runClaimed(run, graph, node, startedAt) {
+        const cost = { value: 0 };
+        return inSpan(options.tracer, `harness.node ${run.graphKey}/${node.id}`, {
+            kind: SPAN_KIND.INTERNAL,
+            attributes: attributes({
+                'vk.harness.run_id': run.id,
+                'vk.harness.graph': `${run.graphKey}@${run.graphVersion}`,
+                'vk.harness.node': node.id,
+                'vk.harness.node_kind': node.kind,
+                'vk.harness.step': run.step,
+                'vk.harness.attempt': run.attempt,
+                'vk.ai.tenant_id': run.context.tenantId,
+                'vk.ai.tier': node.tier,
+            }),
+        }, async (span) => {
+            const outcome = isFanOut(node)
+                ? await fanOut(run, graph, node, startedAt)
+                : await attemptNode(run, graph, node, startedAt, cost);
+            span?.setAttributes({ 'vk.harness.outcome': outcome, 'vk.ai.cost_usd': cost.value });
+            if (outcome === 'failed' || outcome === 'retrying') {
+                span?.setStatus({ code: SPAN_STATUS.ERROR, message: `'${node.label}' ${outcome}` });
+            }
+            return outcome;
+        });
+    }
+    async function attemptNode(run, graph, node, startedAt, cost) {
+        const controller = new AbortController();
+        let abortReason = null;
+        const abort = (reason) => {
+            if (abortReason)
+                return;
+            abortReason = reason;
+            controller.abort();
+        };
+        const timeoutMs = node.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
+        const heartbeat = setInterval(() => {
+            void store
+                .extendLease(run.id, run.version, iso(Date.now() + leaseMs))
+                .catch(() => undefined);
+        }, Math.max(1000, Math.floor(leaseMs / 3)));
+        const poll = setInterval(() => {
+            void store
+                .isCancelRequested(run.id)
+                .then((requested) => {
+                if (requested)
+                    abort('cancel');
+            })
+                .catch(() => undefined);
+        }, cancelPollMs);
+        const timer = setTimeout(() => abort('timeout'), timeoutMs);
+        const aborted = new Promise((_, reject) => {
+            controller.signal.addEventListener('abort', () => reject(abortReason === 'timeout' ? timeoutError(node, timeoutMs) : new RunCancelledError()), { once: true });
+        });
+        aborted.catch(() => undefined);
+        const ctx = makeContext(run, graph, node, controller.signal, cost);
+        try {
+            const work = Promise.resolve().then(() => runNode(node, ctx, run));
+            work.catch(() => undefined);
+            const output = await Promise.race([work, aborted]);
+            if (abortReason === 'cancel')
+                throw new RunCancelledError();
+            const t = transition(run, graph, node, output, resolveNext(node, output, ctx), cost.value);
+            const committed = await store.updateRun(run.id, run.version, t.patch, {
+                events: [
+                    event(run, 'node.completed', node.label, {
+                        nodeId: node.id,
+                        data: { costUsd: cost.value },
+                    }),
+                    ...t.events,
+                ],
+                nodeRun: {
+                    step: run.step,
+                    nodeId: node.id,
+                    attempt: run.attempt,
+                    status: 'succeeded',
+                    startedAt,
+                    finishedAt: nowIso(),
+                    output: output ?? null,
+                    error: null,
+                    costUsd: cost.value,
+                },
+            });
+            if (!committed)
+                return 'stale';
+            if (t.outcome === 'advanced')
+                await scheduler.enqueue({ runId: run.id, step: committed.step, attempt: 1 });
+            return t.outcome;
+        }
+        catch (err) {
+            if (abortReason === 'cancel' || err instanceof RunCancelledError)
+                return finishCancelled(run, { node, startedAt, nodeCost: cost.value });
+            return handleFailure(run, node, err, startedAt, cost.value);
+        }
+        finally {
+            clearInterval(heartbeat);
+            clearInterval(poll);
+            clearTimeout(timer);
+        }
+    }
+    // ─── Fan-out (map nodes with `dispatch: 'tasks'`) ─────────────────
+    const isFanOut = (node) => node.kind === 'map' && node.dispatch === 'tasks';
+    function mapStore() {
+        const s = store;
+        const methods = [
+            'listMapItems',
+            'beginMap',
+            'claimMapItem',
+            'extendMapItemLease',
+            'completeMapItem',
+            'releaseMapItem',
+            'claimNextMapItem',
+            'findMapItemsToSweep',
+            'touchMapItem',
+        ];
+        if (methods.some((m) => typeof s[m] !== 'function'))
+            throw new NodeFatalError("This harness store can't fan map items out (dispatch: 'tasks')", 'no_fan_out');
+        return s;
+    }
+    const itemTask = (run, index) => ({
+        runId: run.id,
+        step: run.step,
+        attempt: run.attempt,
+        item: index,
+    });
+    /**
+     * The node attempt of a fanned-out map: list the items, write them with
+     * the join counter, release the run's lease and send the first
+     * `concurrency` item tasks. The run stays `running` (lease null) until
+     * the last item joins.
+     */
+    async function fanOut(run, graph, node, startedAt) {
+        const cost = { value: 0 };
+        try {
+            const maps = mapStore();
+            const ctx = makeContext(run, graph, node, new AbortController().signal, cost);
+            const items = await node.items(ctx);
+            const existing = new Map((await maps.listMapItems(run.id, run.step)).map((r) => [r.index, r]));
+            let slots = Math.max(1, node.concurrency ?? DEFAULT_MAP_CONCURRENCY);
+            const plan = items.map((input, index) => {
+                const done = existing.get(index)?.status === 'done';
+                const status = !done && slots > 0 ? (slots--, 'queued') : 'pending';
+                return { index, input: input ?? null, status, done };
+            });
+            const begun = await maps.beginMap(run.id, run.version, {
+                step: run.step,
+                items: plan.map(({ done: _d, ...it }) => (void _d, it)),
+                addCostUsd: cost.value,
+                events: [
+                    event(run, 'map.started', node.label, {
+                        nodeId: node.id,
+                        data: {
+                            total: items.length,
+                            done: plan.filter((p) => p.done).length,
+                            concurrency: node.concurrency ?? DEFAULT_MAP_CONCURRENCY,
+                        },
+                    }),
+                ],
+            });
+            if (!begun)
+                return 'stale';
+            if (begun.done >= begun.total)
+                return await joinMap(run.id, graph, node, begun.total, startedAt);
+            for (const p of plan)
+                if (!p.done && p.status === 'queued')
+                    await scheduler.enqueue(itemTask(run, p.index));
+            return 'fanned_out';
+        }
+        catch (err) {
+            return handleFailure(run, node, err, startedAt, cost.value);
+        }
+    }
+    /** One item task: claim the item, run it under its own lease, timeout and cancel poll. */
+    async function executeMapItem(task, executeOptions) {
+        const index = task.item;
+        const maps = mapStore();
+        const load = async () => {
+            const run = await store.getRun(task.runId);
+            if (!run ||
+                run.step !== task.step ||
+                run.attempt !== task.attempt ||
+                run.status !== 'running' ||
+                run.leaseUntil !== null)
+                return null;
+            return run;
+        };
+        let run = await load();
+        if (!run)
+            return 'stale';
+        const graph = graphOf(run);
+        const node = run.currentNode ? graph?.nodes.get(run.currentNode) : undefined;
+        if (!graph || !node || !isFanOut(node))
+            return 'stale';
+        if (run.cancelRequested)
+            return finishCancelled(run, null);
+        const row = (await maps.listMapItems(run.id, run.step)).find((r) => r.index === index);
+        if (!row)
+            return 'stale';
+        if (row.status === 'running')
+            return 'busy';
+        if (row.status !== 'queued')
+            return 'stale';
+        if (!executeOptions.ignoreDelay && row.notBefore) {
+            const wait = Date.parse(row.notBefore) - Date.now();
+            if (wait > 0) {
+                if (wait > (executeOptions.maxWaitMs ?? 0))
+                    return 'deferred';
+                const cancelled = await waitOut(run.id, wait);
+                run = await load();
+                if (!run)
+                    return 'stale';
+                if (cancelled || run.cancelRequested)
+                    return finishCancelled(run, null);
+            }
+        }
+        const claimed = await maps.claimMapItem(run.id, run.step, index, iso(Date.now() + leaseMs));
+        if (!claimed)
+            return 'busy';
+        return runMapItem(run, graph, node, claimed);
+    }
+    async function runMapItem(run, graph, node, item) {
+        const maps = mapStore();
+        const controller = new AbortController();
+        let abortReason = null;
+        const abort = (reason) => {
+            if (abortReason)
+                return;
+            abortReason = reason;
+            controller.abort();
+        };
+        const cost = { value: 0 };
+        const timeoutMs = node.timeoutMs ?? DEFAULT_NODE_TIMEOUT_MS;
+        const heartbeat = setInterval(() => {
+            void maps
+                .extendMapItemLease(run.id, run.step, item.index, item.attempt, iso(Date.now() + leaseMs))
+                .catch(() => undefined);
+        }, Math.max(1000, Math.floor(leaseMs / 3)));
+        const poll = setInterval(() => {
+            void store
+                .isCancelRequested(run.id)
+                .then((requested) => {
+                if (requested)
+                    abort('cancel');
+            })
+                .catch(() => undefined);
+        }, cancelPollMs);
+        const timer = setTimeout(() => abort('timeout'), timeoutMs);
+        const aborted = new Promise((_, reject) => {
+            controller.signal.addEventListener('abort', () => reject(abortReason === 'timeout' ? timeoutError(node, timeoutMs) : new RunCancelledError()), { once: true });
+        });
+        aborted.catch(() => undefined);
+        const ctx = makeContext(run, graph, node, controller.signal, cost, {
+            index: item.index,
+            attempt: item.attempt,
+        });
+        try {
+            const work = Promise.resolve().then(() => node.runItem(item.input, item.index, ctx));
+            work.catch(() => undefined);
+            const output = await Promise.race([work, aborted]);
+            if (abortReason === 'cancel')
+                throw new RunCancelledError();
+            const counter = await maps.completeMapItem(run.id, run.step, item.index, item.attempt, {
+                output: output ?? null,
+                costUsd: cost.value,
+            });
+            if (!counter)
+                return 'stale';
+            await store
+                .appendEvents(run.id, [
+                {
+                    type: 'node.progress',
+                    nodeId: node.id,
+                    step: run.step,
+                    attempt: run.attempt,
+                    label: node.label,
+                    detail: node.detail?.({ done: counter.done, total: counter.total, item: item.index }) ??
+                        `${counter.done} of ${counter.total}`,
+                    data: { done: counter.done, total: counter.total, item: item.index },
+                },
+            ])
+                .catch(() => undefined);
+            if (counter.done >= counter.total)
+                return await joinMap(run.id, graph, node, counter.total, run.updatedAt);
+            const next = await maps.claimNextMapItem(run.id, run.step);
+            if (next !== null)
+                await scheduler.enqueue(itemTask(run, next));
+            return 'item_done';
+        }
+        catch (err) {
+            if (abortReason === 'cancel' || err instanceof RunCancelledError) {
+                await maps
+                    .releaseMapItem(run.id, run.step, item.index, item.attempt, {
+                    status: 'queued',
+                    notBefore: null,
+                    error: {
+                        message: 'Cancelled',
+                        code: 'cancelled',
+                        nodeId: node.id,
+                        attempt: run.attempt,
+                    },
+                    costUsd: cost.value,
+                })
+                    .catch(() => undefined);
+                const fresh = await store.getRun(run.id);
+                return fresh && fresh.status === 'running' ? finishCancelled(fresh, null) : 'cancelled';
+            }
+            return failMapItem(run, node, item, err, cost.value);
+        }
+        finally {
+            clearInterval(heartbeat);
+            clearInterval(poll);
+            clearTimeout(timer);
+        }
+    }
+    /** A failed item: retry it with backoff, or fail the run once its attempts are used up. */
+    async function failMapItem(run, node, item, err, itemCost) {
+        const maps = mapStore();
+        const message = messageOf(err);
+        const fatal = isFatal(err);
+        const used = item.attempt - item.attemptBase;
+        const maxAttempts = node.retry?.maxAttempts ?? DEFAULT_MAX_ATTEMPTS;
+        const error = {
+            message,
+            code: errorCode(err, fatal),
+            nodeId: node.id,
+            attempt: run.attempt,
+        };
+        if (!fatal && retryAllowed(node, err) && used < maxAttempts) {
+            const delayMs = Math.min(backoffMax, (node.retry?.backoffMs ?? backoffBase) * 2 ** Math.max(0, used - 1));
+            const released = await maps.releaseMapItem(run.id, run.step, item.index, item.attempt, {
+                status: 'queued',
+                notBefore: iso(Date.now() + delayMs),
+                error,
+                costUsd: itemCost,
+            });
+            if (!released)
+                return 'stale';
+            await store
+                .appendEvents(run.id, [
+                event(run, 'node.retrying', node.label, {
+                    nodeId: node.id,
+                    detail: message,
+                    data: { item: item.index, nextAttempt: used + 1, delayMs, code: error.code },
+                }),
+            ])
+                .catch(() => undefined);
+            await scheduler.enqueue(itemTask(run, item.index), { delayMs });
+            return 'retrying';
+        }
+        const released = await maps.releaseMapItem(run.id, run.step, item.index, item.attempt, {
+            status: 'failed',
+            notBefore: null,
+            error,
+            costUsd: itemCost,
+        });
+        if (!released)
+            return 'stale';
+        const fresh = await store.getRun(run.id);
+        if (!fresh ||
+            fresh.status !== 'running' ||
+            fresh.step !== run.step ||
+            fresh.attempt !== run.attempt)
+            return 'stale';
+        const spent = round8((await maps.listMapItems(run.id, run.step)).reduce((sum, r) => sum + r.costUsd, 0));
+        const runError = { ...error, message: `Item ${item.index + 1}: ${message}` };
+        const committed = await store.updateRun(fresh.id, fresh.version, {
+            status: 'failed',
+            leaseUntil: null,
+            error: runError,
+            costUsd: round8(fresh.costUsd + spent),
+        }, {
+            events: [
+                event(fresh, 'node.failed', node.label, {
+                    nodeId: node.id,
+                    detail: runError.message,
+                    data: { code: error.code, item: item.index },
+                }),
+                event(fresh, 'run.failed', 'Failed', {
+                    nodeId: node.id,
+                    detail: runError.message,
+                    data: { code: error.code },
+                }),
+            ],
+            nodeRun: {
+                step: fresh.step,
+                nodeId: node.id,
+                attempt: fresh.attempt,
+                status: 'failed',
+                startedAt: fresh.updatedAt,
+                finishedAt: nowIso(),
+                output: null,
+                error: runError,
+                costUsd: spent,
+            },
+        });
+        return committed ? 'failed' : 'stale';
+    }
+    /** The join: every item is done — commit the node with the outputs in item order and the items' spend. */
+    async function joinMap(runId, graph, node, total, startedAt) {
+        const maps = mapStore();
+        const run = await store.getRun(runId);
+        if (!run || run.status !== 'running' || run.leaseUntil !== null)
+            return 'stale';
+        if (run.cancelRequested)
+            return finishCancelled(run, null);
+        const rows = await maps.listMapItems(run.id, run.step);
+        const byIndex = new Map(rows.map((r) => [r.index, r]));
+        const outputs = Array.from({ length: total }, (_, i) => byIndex.get(i)?.output ?? null);
+        const spent = round8(rows.filter((r) => r.index < total).reduce((sum, r) => sum + r.costUsd, 0));
+        try {
+            const ctx = makeContext(run, graph, node, new AbortController().signal, { value: 0 });
+            const t = transition(run, graph, node, outputs, resolveNext(node, outputs, ctx), spent);
+            const committed = await store.updateRun(run.id, run.version, t.patch, {
+                events: [
+                    event(run, 'node.completed', node.label, {
+                        nodeId: node.id,
+                        data: { costUsd: spent, items: total },
+                    }),
+                    ...t.events,
+                ],
+                nodeRun: {
+                    step: run.step,
+                    nodeId: node.id,
+                    attempt: run.attempt,
+                    status: 'succeeded',
+                    startedAt,
+                    finishedAt: nowIso(),
+                    output: outputs,
+                    error: null,
+                    costUsd: spent,
+                },
+            });
+            if (!committed)
+                return 'stale';
+            if (t.outcome === 'advanced')
+                await scheduler.enqueue({ runId: run.id, step: committed.step, attempt: 1 });
+            return t.outcome;
+        }
+        catch (err) {
+            return handleFailure(run, node, err, startedAt, spent);
+        }
+    }
+    /** Wait in place for a delayed task; true when the run was cancelled meanwhile. */
+    async function waitOut(runId, ms) {
+        const until = Date.now() + ms;
+        while (Date.now() < until) {
+            await sleep(Math.min(cancelPollMs, until - Date.now()));
+            if (await store.isCancelRequested(runId))
+                return true;
+        }
+        return false;
+    }
+    const requireRun = async (runId) => {
+        const run = await store.getRun(runId);
+        if (!run)
+            throw new HarnessError('not_found', `Run '${runId}' not found`);
+        return run;
+    };
+    // ─── Public API ──────────────────────────────────────────────────
+    return {
+        graphs: () => [...registry.values()].flatMap((versions) => [...versions.values()].map((g) => g.def)),
+        async start(startOptions) {
+            const versions = registry.get(startOptions.graph);
+            if (!versions)
+                throw new HarnessError('unknown_graph', `Graph '${startOptions.graph}' is not registered`);
+            const version = startOptions.version ?? Math.max(...versions.keys());
+            const graph = versions.get(version);
+            if (!graph)
+                throw new HarnessError('unknown_graph', `Graph '${startOptions.graph}@${version}' is not registered`);
+            const startNode = graph.nodes.get(graph.def.start);
+            const at = nowIso();
+            const delayMs = Math.max(0, startOptions.delayMs ?? 0);
+            const waiting = startNode.kind === 'gate';
+            const run = {
+                id: generateId(),
+                graphKey: graph.def.key,
+                graphVersion: version,
+                status: waiting ? 'waiting' : 'queued',
+                context: startOptions.context ?? {},
+                input: startOptions.input ?? null,
+                outputs: {},
+                currentNode: startNode.id,
+                step: 1,
+                attempt: 1,
+                attemptBase: 0,
+                notBefore: !waiting && delayMs > 0 ? iso(Date.now() + delayMs) : null,
+                error: null,
+                cancelRequested: false,
+                costUsd: 0,
+                createdAt: at,
+                updatedAt: at,
+                completedAt: null,
+                version: 0,
+                leaseUntil: null,
+            };
+            const events = [
+                {
+                    type: 'run.started',
+                    nodeId: null,
+                    step: null,
+                    attempt: null,
+                    label: `${graph.def.key}@${version}`,
+                    detail: null,
+                    data: delayMs ? { delayMs } : null,
+                },
+            ];
+            if (waiting)
+                events.push({
+                    type: 'gate.waiting',
+                    nodeId: startNode.id,
+                    step: 1,
+                    attempt: 1,
+                    label: startNode.label,
+                    detail: null,
+                    data: { decisions: startNode.kind === 'gate' ? startNode.decisions : [] },
+                });
+            await store.insertRun(run, events);
+            if (!waiting)
+                await scheduler.enqueue({ runId: run.id, step: 1, attempt: 1 }, delayMs > 0 ? { delayMs } : undefined);
+            return toRunRecord(run);
+        },
+        async execute(task, executeOptions = {}) {
+            const result = (outcome) => ({ outcome, runId: task.runId });
+            if (task.item !== undefined)
+                return result(await executeMapItem(task, executeOptions));
+            const load = async () => {
+                const run = await store.getRun(task.runId);
+                if (!run || run.step !== task.step || run.attempt !== task.attempt)
+                    return { run: null, outcome: 'stale' };
+                if (run.status === 'running')
+                    return { run: null, outcome: 'busy' };
+                if (run.status !== 'queued')
+                    return { run: null, outcome: 'stale' };
+                return { run, outcome: null };
+            };
+            let loaded = await load();
+            if (!loaded.run)
+                return result(loaded.outcome);
+            let run = loaded.run;
+            if (run.cancelRequested)
+                return result(await finishCancelled(run, null));
+            if (!executeOptions.ignoreDelay && run.notBefore) {
+                const wait = Date.parse(run.notBefore) - Date.now();
+                if (wait > 0) {
+                    if (wait > (executeOptions.maxWaitMs ?? 0))
+                        return result('deferred');
+                    const cancelled = await waitOut(run.id, wait);
+                    loaded = await load();
+                    if (!loaded.run)
+                        return result(loaded.outcome);
+                    run = loaded.run;
+                    if (cancelled || run.cancelRequested)
+                        return result(await finishCancelled(run, null));
+                }
+            }
+            const graph = graphOf(run);
+            const node = run.currentNode ? graph?.nodes.get(run.currentNode) : undefined;
+            if (!graph || !node) {
+                const message = `Graph ${run.graphKey}@${run.graphVersion} or its node '${run.currentNode}' is not registered`;
+                const failed = await store.updateRun(run.id, run.version, { status: 'failed', error: { message, code: 'unknown_graph' } }, { events: [event(run, 'run.failed', 'Failed', { detail: message })] });
+                return result(failed ? 'failed' : 'stale');
+            }
+            if (node.kind === 'gate') {
+                const waiting = await store.updateRun(run.id, run.version, { status: 'waiting', notBefore: null }, {
+                    events: [
+                        event(run, 'gate.waiting', node.label, { data: { decisions: node.decisions } }),
+                    ],
+                });
+                return result(waiting ? 'waiting' : 'busy');
+            }
+            const startedAt = nowIso();
+            const claimed = await store.updateRun(run.id, run.version, { status: 'running', leaseUntil: iso(Date.now() + leaseMs), notBefore: null }, {
+                events: [event(run, 'node.started', node.label, { data: { kind: node.kind } })],
+            });
+            if (!claimed)
+                return result('busy');
+            return result(await runClaimed(claimed, graph, node, startedAt));
+        },
+        async resume(runId, decision) {
+            const run = await requireRun(runId);
+            if (run.status !== 'waiting')
+                throw new HarnessError('not_waiting', `Run '${runId}' is not waiting for a decision (status: ${run.status})`);
+            const graph = graphOf(run);
+            const node = run.currentNode ? graph?.nodes.get(run.currentNode) : undefined;
+            if (!graph || !node || node.kind !== 'gate')
+                throw new HarnessError('not_waiting', `Run '${runId}' is not at a gate`);
+            if (!node.decisions.includes(decision.decision)) {
+                throw new HarnessError('invalid_decision', `'${decision.decision}' is not a decision of '${node.id}' (${node.decisions.join(', ')})`);
+            }
+            const output = {
+                decision: decision.decision,
+                ...(decision.data !== undefined ? { data: decision.data } : {}),
+                ...(decision.actor ? { actor: decision.actor } : {}),
+                at: nowIso(),
+            };
+            const ctx = makeContext(run, graph, node, new AbortController().signal, { value: 0 });
+            const t = transition(run, graph, node, output, resolveNext(node, output, ctx), 0);
+            const committed = await store.updateRun(run.id, run.version, t.patch, {
+                events: [
+                    event(run, 'gate.decided', decision.decision, {
+                        data: { actor: decision.actor ?? null },
+                    }),
+                    ...t.events,
+                ],
+                nodeRun: {
+                    step: run.step,
+                    nodeId: node.id,
+                    attempt: run.attempt,
+                    status: 'succeeded',
+                    startedAt: run.updatedAt,
+                    finishedAt: nowIso(),
+                    output,
+                    error: null,
+                    costUsd: 0,
+                },
+            });
+            if (!committed)
+                throw new HarnessError('conflict', `Run '${runId}' changed meanwhile; try again`);
+            if (t.outcome === 'advanced')
+                await scheduler.enqueue({ runId: run.id, step: committed.step, attempt: 1 });
+            return toRunRecord(committed);
+        },
+        async cancel(runId, cancelOptions = {}) {
+            const detail = cancelOptions.reason ?? null;
+            const actor = cancelOptions.actor ?? null;
+            for (let tries = 0; tries < 3; tries++) {
+                const run = await requireRun(runId);
+                if (run.status === 'completed' || run.status === 'cancelled')
+                    return toRunRecord(run);
+                if (run.status === 'running') {
+                    await store.requestCancel(run.id);
+                    await store.appendEvents(run.id, [
+                        event(run, 'run.cancel_requested', 'Stopping', { detail, data: { actor } }),
+                    ]);
+                    // The worker is gone (lease expired), or a fanned-out map holds no
+                    // lease (its items stop at their next cancel poll): finish it now.
+                    if (run.leaseUntil === null || Date.parse(run.leaseUntil) < Date.now()) {
+                        const fresh = await requireRun(runId);
+                        if (fresh.status === 'running')
+                            await finishCancelled(fresh, null, detail);
+                    }
+                    return toRunRecord(await requireRun(runId));
+                }
+                const committed = await store.updateRun(run.id, run.version, {
+                    status: 'cancelled',
+                    cancelRequested: true,
+                    leaseUntil: null,
+                    notBefore: null,
+                    completedAt: nowIso(),
+                }, { events: [event(run, 'run.cancelled', 'Cancelled', { detail, data: { actor } })] });
+                if (committed)
+                    return toRunRecord(committed);
+            }
+            throw new HarnessError('conflict', `Run '${runId}' kept changing; try again`);
+        },
+        async retry(runId, retryOptions = {}) {
+            const run = await requireRun(runId);
+            if (run.status !== 'failed')
+                throw new HarnessError('not_failed', `Run '${runId}' is not failed (status: ${run.status})`);
+            const attempt = run.attempt + 1;
+            const committed = await store.updateRun(run.id, run.version, {
+                status: 'queued',
+                attempt,
+                attemptBase: run.attempt,
+                error: null,
+                notBefore: null,
+                leaseUntil: null,
+                cancelRequested: false,
+                completedAt: null,
+            }, {
+                events: [
+                    {
+                        ...event(run, 'run.retried', 'Retrying', {
+                            data: { actor: retryOptions.actor ?? null },
+                        }),
+                        attempt,
+                    },
+                ],
+            });
+            if (!committed)
+                throw new HarnessError('conflict', `Run '${runId}' changed meanwhile; try again`);
+            await scheduler.enqueue({ runId: run.id, step: committed.step, attempt });
+            return toRunRecord(committed);
+        },
+        async sweep(sweepOptions = {}) {
+            const staleAfterMs = sweepOptions.staleAfterMs ?? 2 * 60_000;
+            const now = Date.now();
+            const runs = await store.findRunsToSweep({
+                now: iso(now),
+                staleBefore: iso(now - staleAfterMs),
+                quietBefore: iso(now - Math.min(staleAfterMs, SWEEP_QUIET_MS)),
+                limit: SWEEP_BATCH,
+            });
+            let touched = 0;
+            for (const run of runs) {
+                try {
+                    if (run.cancelRequested) {
+                        if ((await finishCancelled(run, null)) === 'cancelled')
+                            touched += 1;
+                        continue;
+                    }
+                    if (run.status === 'queued') {
+                        // Refresh `updated_at` so the next sweep waits again, then re-send.
+                        const bumped = await store.updateRun(run.id, run.version, {});
+                        if (!bumped)
+                            continue;
+                        await scheduler.enqueue({ runId: run.id, step: run.step, attempt: run.attempt });
+                        touched += 1;
+                    }
+                    else if (run.status === 'running') {
+                        const node = run.currentNode ? graphOf(run)?.nodes.get(run.currentNode) : undefined;
+                        if (!node)
+                            continue;
+                        const outcome = await handleFailure(run, node, new Error('The worker stopped before this step finished'), run.updatedAt, 0);
+                        if (outcome !== 'stale')
+                            touched += 1;
+                    }
+                }
+                catch {
+                    // One bad run must not stop the sweep.
+                }
+            }
+            // Fanned-out map items: re-send lost item tasks, fail the attempt of a dead item worker.
+            if (typeof store.findMapItemsToSweep === 'function') {
+                const maps = mapStore();
+                const items = await maps.findMapItemsToSweep({
+                    now: iso(now),
+                    quietBefore: iso(now - Math.min(staleAfterMs, SWEEP_QUIET_MS)),
+                    limit: SWEEP_BATCH,
+                });
+                for (const it of items) {
+                    try {
+                        const run = await store.getRun(it.runId);
+                        if (!run || run.status !== 'running')
+                            continue;
+                        if (run.cancelRequested) {
+                            if ((await finishCancelled(run, null)) === 'cancelled')
+                                touched += 1;
+                            continue;
+                        }
+                        if (it.status === 'queued') {
+                            if (!(await maps.touchMapItem(run.id, it.step, it.index)))
+                                continue;
+                            await scheduler.enqueue(itemTask(run, it.index));
+                            touched += 1;
+                        }
+                        else {
+                            const node = run.currentNode ? graphOf(run)?.nodes.get(run.currentNode) : undefined;
+                            const row = (await maps.listMapItems(run.id, it.step)).find((r) => r.index === it.index);
+                            if (!node || !isFanOut(node) || !row)
+                                continue;
+                            const outcome = await failMapItem(run, node, row, new Error('The worker stopped before this item finished'), 0);
+                            if (outcome !== 'stale')
+                                touched += 1;
+                        }
+                    }
+                    catch {
+                        // One bad item must not stop the sweep.
+                    }
+                }
+            }
+            return touched;
+        },
+        async getRun(runId) {
+            const run = await store.getRun(runId);
+            return run ? toRunRecord(run) : null;
+        },
+        async events(runId, eventOptions = {}) {
+            return store.listEvents(runId, {
+                ...(eventOptions.afterSeq !== undefined ? { afterSeq: eventOptions.afterSeq } : {}),
+                limit: Math.min(Math.max(eventOptions.limit ?? 500, 1), 5000),
+            });
+        },
+    };
+}
+//# sourceMappingURL=harness.js.map