@fabric-harness/sdk 1.14.0 → 2.0.1

package/dist/submission-runner.js

@@ -0,0 +1,767 @@
+/**
+ * Durable submission runner (v2 migration, phase A5).
+ *
+ * The execution engine over an {@link AgentSubmissionStore}: a claim loop
+ * that takes runnable session heads with lease-fenced CAS claims, an
+ * executor seam that applies the input to the instance session, and a
+ * reconciler that classifies interrupted work after a crash and produces
+ * exactly-once observable outcomes — completed work settles, repairable
+ * work resumes, unstarted work requeues, exhausted work fails with a
+ * terminal advisory. Ported from flue's node agent coordinator +
+ * `reconcileInterruptedSubmission`, adapted to the harness session model
+ * (SessionEntry DAG; `session.prompt({ submission })` is idempotent by
+ * submission id, so re-execution after a crash never double-applies input).
+ */
+import { LEASE_DURATION_MS, createDispatchAgentSubmissionInput, submissionStoreSessionId, } from './submission-store.js';
+import { materializeMessageAttachments, messageHasDataAttachments } from './attachment-store.js';
+import { submissionSettlementEntryId } from './submission-state.js';
+import { runWithSubmissionContext } from './submission-context.js';
+import { emitSubmissionTelemetry } from './submission-telemetry.js';
+// ─── Errors ─────────────────────────────────────────────────────────────────
+export class SubmissionAbortedError extends Error {
+    code = 'SUBMISSION_ABORTED';
+    constructor() {
+        super('The agent submission was aborted.');
+        this.name = 'SubmissionAbortedError';
+    }
+}
+export class SubmissionTimeoutError extends Error {
+    code = 'SUBMISSION_TIMEOUT';
+    constructor() {
+        super('The agent submission exceeded its execution timeout.');
+        this.name = 'SubmissionTimeoutError';
+    }
+}
+export class SubmissionRetryExhaustedError extends Error {
+    code = 'SUBMISSION_RETRY_EXHAUSTED';
+    interruptedTools;
+    constructor(options) {
+        super(`The agent submission exceeded its maximum recovery attempts (${options.attemptCount}/${options.maxAttempts}).`);
+        this.name = 'SubmissionRetryExhaustedError';
+        this.interruptedTools = options.interruptedTools;
+    }
+}
+export class SubmissionInterruptedError extends Error {
+    code = 'SUBMISSION_INTERRUPTED';
+    phase;
+    constructor(phase, detail) {
+        super(phase === 'retry_exhausted_before_input'
+            ? `The agent submission was interrupted repeatedly before its input could be applied${detail ? ` (${detail})` : ''}.`
+            : 'The agent submission was interrupted after its input was applied and could not be recovered.');
+        this.name = 'SubmissionInterruptedError';
+        this.phase = phase;
+    }
+}
+export function createSubmissionRunner(options) {
+    const { submissions, executor } = options;
+    const onError = options.onError ??
+        ((context, error) => {
+            console.error(`[fabric-harness:submission-runner] ${context}:`, error);
+        });
+    const heartbeatIntervalMs = options.heartbeatIntervalMs ?? 10_000;
+    const leaseScanIntervalMs = options.leaseScanIntervalMs ?? 15_000;
+    /** Owner id for lease-based submission ownership. */
+    const ownerId = crypto.randomUUID();
+    const activeSubmissions = new Map();
+    /** Local settlement notifications for same-process waiters. */
+    const localSettlements = new Map();
+    const settlementListeners = new Map();
+    let heartbeatInterval = null;
+    let leaseScanInterval = null;
+    // ── Wake machinery (see flue's claim loop for the mode discussion) ──
+    let wakeResolve = null;
+    let wakePromise = null;
+    let claimPassRunning = false;
+    let wakeRequested = false;
+    let loopRunning = false;
+    let stopping = false;
+    function resetWakePromise() {
+        wakePromise = new Promise((resolve) => {
+            wakeResolve = resolve;
+        });
+    }
+    function wake() {
+        if (claimPassRunning) {
+            wakeRequested = true;
+            return;
+        }
+        if (wakeResolve) {
+            const resolve = wakeResolve;
+            wakeResolve = null;
+            resolve();
+        }
+    }
+    function notifySettled(submissionId, settlement, submission) {
+        if (submission) {
+            emitSubmissionTelemetry(options.telemetry, {
+                type: 'submission_settled',
+                ...telemetryBase(submission),
+                ...(submission.attemptId !== undefined ? { attemptId: submission.attemptId } : {}),
+                outcome: settlement.outcome,
+                ...(settlement.error !== undefined ? { error: settlement.error } : {}),
+                durationMs: Math.max(0, Date.now() - submission.acceptedAt),
+                attemptCount: submission.attemptCount,
+            }, telemetryError);
+        }
+        localSettlements.set(submissionId, settlement);
+        const listeners = settlementListeners.get(submissionId);
+        if (listeners) {
+            settlementListeners.delete(submissionId);
+            for (const listener of listeners) {
+                try {
+                    listener(settlement);
+                }
+                catch (error) {
+                    onError('settlement listener failed', error);
+                }
+            }
+        }
+        try {
+            options.onSettled?.(submissionId, settlement);
+        }
+        catch (error) {
+            onError('onSettled callback failed', error);
+        }
+    }
+    function serializeError(error) {
+        if (error instanceof Error)
+            return error.message;
+        return String(error);
+    }
+    function attemptRefOf(submission) {
+        if (!submission.attemptId)
+            return null;
+        return { submissionId: submission.submissionId, attemptId: submission.attemptId };
+    }
+    function telemetryBase(submission) {
+        return {
+            submissionId: submission.submissionId,
+            kind: submission.kind,
+            agent: submission.input.agent,
+            instanceId: submission.input.id,
+            session: submission.input.session,
+            at: Date.now(),
+            ...(submission.input.tenantId !== undefined ? { tenantId: submission.input.tenantId } : {}),
+            ...(submission.input.actor !== undefined ? { actor: submission.input.actor } : {}),
+        };
+    }
+    const telemetryError = (error) => onError('telemetry sink failed', error);
+    // ── Settlement ───────────────────────────────────────────────────────
+    /**
+     * Two-phase settlement for direct submissions: reserve the exact canonical
+     * record as an obligation (running → terminalizing CAS), append the
+     * canonical entry (idempotent by deterministic id), then finalize
+     * (terminalizing → settled, obligation deleted). A crash between the
+     * phases is repaired by the pending-settlement pass in reconciliation.
+     */
+    async function settleDirect(submission, attempt, outcome, result, error) {
+        const recordId = submissionSettlementEntryId(submission.submissionId);
+        const pending = (await submissions.listPendingSubmissionSettlements()).find((obligation) => obligation.submissionId === submission.submissionId);
+        const record = pending?.record ??
+            {
+                type: 'submission_settled',
+                submissionId: submission.submissionId,
+                outcome,
+                ...(outcome === 'completed' && result !== undefined
+                    ? { resultText: typeof result === 'string' ? result : JSON.stringify(result) }
+                    : {}),
+                ...(outcome !== 'completed' ? { error: serializeError(error ?? new Error('unknown error')) } : {}),
+            };
+        const obligation = pending ?? (await submissions.reserveSubmissionSettlement(attempt, { recordId, record }));
+        if (!obligation)
+            return;
+        await executor.appendSettlement(submission, obligation);
+        await submissions.finalizeSubmissionSettlement(attempt, recordId);
+        notifySettled(submission.submissionId, {
+            outcome: obligation.record.outcome,
+            ...(result !== undefined ? { result } : {}),
+            ...(obligation.record.error !== undefined ? { error: obligation.record.error } : {}),
+        }, submission);
+    }
+    async function settleDispatch(submission, attempt, outcome, result, error) {
+        if (outcome === 'completed') {
+            if (!(await submissions.completeSubmission(attempt)))
+                return;
+        }
+        else if (!(await submissions.failSubmission(attempt, error ?? new SubmissionAbortedError()))) {
+            return;
+        }
+        notifySettled(submission.submissionId, {
+            outcome,
+            ...(result !== undefined ? { result } : {}),
+            ...(outcome !== 'completed' ? { error: serializeError(error ?? new SubmissionAbortedError()) } : {}),
+        }, submission);
+    }
+    async function settle(submission, attempt, outcome, result, error) {
+        if (submission.kind === 'direct') {
+            await settleDirect(submission, attempt, outcome, result, error);
+        }
+        else {
+            await settleDispatch(submission, attempt, outcome, result, error);
+        }
+    }
+    /** Settle as the distinct aborted outcome, with a visible terminal advisory. */
+    async function settleAborted(submission, attempt) {
+        const error = new SubmissionAbortedError();
+        try {
+            await executor.recordTerminal(submission, {
+                submissionId: submission.submissionId,
+                kind: submission.kind,
+                reason: 'aborted',
+                message: error.message,
+            });
+        }
+        catch (advisoryError) {
+            onError(`abort advisory failed for ${submission.submissionId}`, advisoryError);
+        }
+        await settle(submission, attempt, 'aborted', undefined, error);
+    }
+    async function failInterrupted(submission, attempt, reason, createError) {
+        // Best-effort terminal record: a persistent save failure must not wedge
+        // the submission in an infinite reconciliation loop.
+        let interruptedTools;
+        try {
+            interruptedTools = await executor.recordTerminal(submission, {
+                submissionId: submission.submissionId,
+                kind: submission.kind,
+                reason,
+                message: createError(undefined).message,
+            });
+        }
+        catch (terminalError) {
+            onError(`terminal record failed for ${submission.submissionId}`, terminalError);
+        }
+        const error = createError(interruptedTools?.length ? interruptedTools : undefined);
+        await settle(submission, attempt, 'failed', undefined, error);
+    }
+    // ── Processing ───────────────────────────────────────────────────────
+    async function processClaimed(claimed, signal) {
+        const attempt = attemptRefOf(claimed);
+        if (!attempt)
+            return;
+        const persisted = await submissions.getSubmission(claimed.submissionId);
+        if (persisted?.status !== 'running' || persisted.attemptId !== attempt.attemptId)
+            return;
+        await submissions.insertAttemptMarker(attempt);
+        emitSubmissionTelemetry(options.telemetry, { type: 'submission_started', ...telemetryBase(claimed), attemptId: attempt.attemptId, attemptCount: persisted.attemptCount }, telemetryError);
+        try {
+            // Pre-execution abort: a queued submission that was abort-flagged is
+            // still claimed (attempt-based settlement is uniform); settle it as
+            // aborted before running any model work.
+            if (persisted.abortRequestedAt !== undefined) {
+                await settleAborted(claimed, attempt);
+                return;
+            }
+            let result;
+            try {
+                // Ambient correlation: tool wrappers / telemetry inside the turn read
+                // the owning submission via currentSubmissionContext().
+                result = await runWithSubmissionContext({
+                    submissionId: claimed.submissionId,
+                    attemptId: attempt.attemptId,
+                    agent: claimed.input.agent,
+                    instanceId: claimed.input.id,
+                    session: claimed.input.session,
+                    ...(claimed.input.tenantId !== undefined ? { tenantId: claimed.input.tenantId } : {}),
+                    ...(claimed.input.actor !== undefined ? { actor: claimed.input.actor } : {}),
+                }, () => executor.execute(claimed, {
+                    attempt,
+                    signal,
+                    onInputApplied: async () => {
+                        if (!(await submissions.markSubmissionInputApplied(attempt))) {
+                            throw new Error('[fabric-harness] Submission attempt lost ownership before input application.');
+                        }
+                    },
+                }));
+            }
+            catch (error) {
+                // Graceful shutdown: the submission stays running; its expired
+                // lease triggers reclamation on next startup.
+                if (stopping && isAbortError(error))
+                    throw error;
+                if (signal.reason instanceof SubmissionAbortedError) {
+                    await settleAborted(claimed, attempt);
+                    return;
+                }
+                await settle(claimed, attempt, 'failed', undefined, error);
+                throw error;
+            }
+            await settle(claimed, attempt, 'completed', result);
+        }
+        finally {
+            await submissions.deleteAttemptMarker(attempt).catch((error) => {
+                onError(`attempt marker cleanup failed for ${claimed.submissionId}`, error);
+            });
+        }
+    }
+    function spawnSubmissionTask(claimed) {
+        const controller = new AbortController();
+        const task = processClaimed(claimed, controller.signal)
+            .catch((error) => {
+            if (isAbortError(error))
+                return;
+            onError(`processing failed for ${claimed.submissionId}`, error);
+        })
+            .finally(() => {
+            activeSubmissions.delete(claimed.submissionId);
+            wake();
+        });
+        activeSubmissions.set(claimed.submissionId, { task, abort: controller });
+    }
+    // ── Claim loop ───────────────────────────────────────────────────────
+    async function runClaimPass() {
+        await markUnreadyCanonical();
+        await periodicLeaseScan();
+        const runnable = await submissions.listRunnableSubmissions();
+        let progressed = false;
+        for (const submission of runnable) {
+            if (activeSubmissions.has(submission.submissionId))
+                continue;
+            const claimed = await submissions.claimSubmission({
+                submissionId: submission.submissionId,
+                attemptId: crypto.randomUUID(),
+                ownerId,
+                leaseExpiresAt: Date.now() + LEASE_DURATION_MS,
+            });
+            if (!claimed)
+                continue;
+            progressed = true;
+            spawnSubmissionTask(claimed);
+        }
+        return progressed;
+    }
+    async function claimLoop() {
+        while (!stopping) {
+            claimPassRunning = true;
+            try {
+                let progressed;
+                do {
+                    wakeRequested = false;
+                    progressed = await runClaimPass();
+                } while (progressed || wakeRequested);
+            }
+            catch (error) {
+                onError('claim pass failed, retrying', error);
+                await new Promise((resolve) => setTimeout(resolve, 1000));
+                wakeRequested = true;
+            }
+            finally {
+                // Reset the sleep promise BEFORE clearing claimPassRunning so a wake
+                // arriving in the gap resolves the new promise, not a stale one.
+                resetWakePromise();
+                claimPassRunning = false;
+            }
+            if (wakeRequested) {
+                wakeRequested = false;
+                continue;
+            }
+            await wakePromise;
+        }
+    }
+    function ensureClaimLoop() {
+        if (loopRunning)
+            return;
+        loopRunning = true;
+        void claimLoop().catch((error) => {
+            onError('fatal claim loop error', error);
+            loopRunning = false;
+        });
+        if (!heartbeatInterval) {
+            heartbeatInterval = setInterval(() => {
+                const ids = [...activeSubmissions.keys()];
+                if (ids.length === 0)
+                    return;
+                submissions.renewLeases(ownerId, ids).catch((error) => {
+                    onError('lease renewal failed', error);
+                });
+            }, heartbeatIntervalMs);
+            heartbeatInterval.unref?.();
+        }
+        if (!leaseScanInterval) {
+            leaseScanInterval = setInterval(() => wake(), leaseScanIntervalMs);
+            leaseScanInterval.unref?.();
+        }
+    }
+    // ── Reconciliation ───────────────────────────────────────────────────
+    let lastLeaseScanAt = 0;
+    let reconcilePassInFlight = null;
+    async function periodicLeaseScan() {
+        const now = Date.now();
+        if (now - lastLeaseScanAt < leaseScanIntervalMs)
+            return;
+        lastLeaseScanAt = now;
+        await reconcileRunningSubmissions();
+    }
+    /** Single-flight expired-lease reconciliation (see flue for the race this closes). */
+    function reconcileRunningSubmissions() {
+        reconcilePassInFlight ??= runReconciliationPass().finally(() => {
+            reconcilePassInFlight = null;
+        });
+        return reconcilePassInFlight;
+    }
+    /** Admission crash-window repair: canonical readiness for unready queued rows. */
+    async function markUnreadyCanonical() {
+        for (const submission of await submissions.listUnreadySubmissions()) {
+            try {
+                await submissions.markSubmissionCanonicalReady(submission.submissionId);
+            }
+            catch (error) {
+                onError(`canonical readiness failed for ${submission.submissionId}`, error);
+            }
+        }
+    }
+    async function runReconciliationPass() {
+        await markUnreadyCanonical();
+        // Finalize settlements that crashed between reserve and finalize.
+        for (const pending of await submissions.listPendingSubmissionSettlements()) {
+            const submission = await submissions.getSubmission(pending.submissionId);
+            if (!submission || submission.kind !== 'direct')
+                continue;
+            if (activeSubmissions.has(submission.submissionId) || submission.leaseExpiresAt > Date.now())
+                continue;
+            try {
+                const attempt = { submissionId: pending.submissionId, attemptId: pending.attemptId };
+                await executor.appendSettlement(submission, pending);
+                await submissions.finalizeSubmissionSettlement(attempt, pending.recordId);
+                notifySettled(submission.submissionId, {
+                    outcome: pending.record.outcome,
+                    ...(pending.record.error !== undefined ? { error: pending.record.error } : {}),
+                }, submission);
+            }
+            catch (error) {
+                onError(`pending settlement finalization failed for ${pending.submissionId}`, error);
+            }
+        }
+        for (const submission of await submissions.listExpiredSubmissions()) {
+            // Never reconcile our own live submission (transient heartbeat failure
+            // can expire a lease mid-flight); a second concurrent task for the same
+            // session is exactly the corruption leases exist to prevent.
+            if (activeSubmissions.has(submission.submissionId))
+                continue;
+            try {
+                const replacement = await reconcileInterrupted(submission);
+                if (replacement)
+                    spawnSubmissionTask(replacement);
+            }
+            catch (error) {
+                onError(`reconciliation failed for ${submission.submissionId}`, error);
+            }
+        }
+    }
+    /**
+     * Shared reconciliation decision tree for an interrupted running
+     * submission (flue's `reconcileInterruptedSubmission`, adapted). Returns
+     * the replacement submission when a new attempt was claimed; `undefined`
+     * for every other outcome (all durable side effects already applied).
+     */
+    async function reconcileInterrupted(submission) {
+        const attempt = attemptRefOf(submission);
+        if (!attempt)
+            return undefined;
+        try {
+            return await reconcileInterruptedAttempt(submission, attempt);
+        }
+        finally {
+            // The interrupted attempt is dead on every branch (settled, requeued,
+            // or replaced by a fresh attempt that inserts its own marker) — drop
+            // its durable attempt marker so markers never leak across crashes.
+            await submissions.deleteAttemptMarker(attempt).catch((error) => {
+                onError(`stale attempt marker cleanup failed for ${submission.submissionId}`, error);
+            });
+        }
+    }
+    async function reconcileInterruptedAttempt(submission, attempt) {
+        // Completed canonical work settles as success unconditionally — the
+        // retry budget and timeout gate only the retry/requeue branches below,
+        // and must never discard finished provider work.
+        const state = await executor.inspect(submission);
+        if (state === 'completed') {
+            await settle(submission, attempt, 'completed');
+            return undefined;
+        }
+        // A crash-interrupted abort is never resurrected; and the budget/timeout
+        // cannot pre-empt the distinct aborted outcome.
+        if (submission.abortRequestedAt !== undefined) {
+            await settleAborted(submission, attempt);
+            return undefined;
+        }
+        if (submission.attemptCount >= submission.maxRetry) {
+            await failInterrupted(submission, attempt, 'exhausted_retry_budget', (interruptedTools) => submission.inputAppliedAt === undefined
+                ? new SubmissionInterruptedError('retry_exhausted_before_input', `${submission.attemptCount}/${submission.maxRetry} attempts`)
+                : new SubmissionRetryExhaustedError({
+                    attemptCount: submission.attemptCount,
+                    maxAttempts: submission.maxRetry,
+                    ...(interruptedTools ? { interruptedTools } : {}),
+                }));
+            return undefined;
+        }
+        if (submission.timeoutAt > 0 && Date.now() >= submission.timeoutAt) {
+            await failInterrupted(submission, attempt, 'exceeded_timeout', () => new SubmissionTimeoutError());
+            return undefined;
+        }
+        // Canonical input exists but the input-applied marker did not land (the
+        // crash window between the entry write and the marker CAS). Re-acquire
+        // the attempt, install the marker, and resume from canonical state.
+        if (submission.inputAppliedAt === undefined && state !== 'absent') {
+            const replacement = await submissions.replaceSubmissionAttempt(attempt, crypto.randomUUID(), {
+                ownerId,
+                leaseExpiresAt: Date.now() + LEASE_DURATION_MS,
+            });
+            if (replacement?.attemptId) {
+                const replacementAttempt = { submissionId: replacement.submissionId, attemptId: replacement.attemptId };
+                if (!(await submissions.markSubmissionInputApplied(replacementAttempt)))
+                    return undefined;
+                return replacement;
+            }
+            return undefined;
+        }
+        // Resumable progress ('continuable': trailing tool batch repaired at the
+        // next prompt) or the one accepted provider-redispatch window
+        // ('uncertain': nothing observable persisted). Acquire the replacement
+        // attempt (the fencing CAS) before any recovery side effect.
+        if (state === 'continuable' || state === 'uncertain') {
+            const replacement = await submissions.replaceSubmissionAttempt(attempt, crypto.randomUUID(), {
+                ownerId,
+                leaseExpiresAt: Date.now() + LEASE_DURATION_MS,
+            });
+            if (!replacement?.attemptId)
+                return undefined;
+            return replacement;
+        }
+        // Only 'absent' remains. No canonical input: either requeue for a clean
+        // first attempt (marker never landed) or fail (marker landed but the
+        // canonical input could not be persisted before the crash).
+        if (submission.inputAppliedAt === undefined) {
+            await submissions.requeueSubmissionBeforeInputApplied(attempt);
+            return undefined;
+        }
+        await failInterrupted(submission, attempt, 'interrupted_after_input_application', () => new SubmissionInterruptedError('after_input_application'));
+        return undefined;
+    }
+    // ── Attachment materialization (pre-admission) ───────────────────────
+    /**
+     * Replace inline base64 attachments with durable refs before the payload
+     * is persisted. `admissionId` seeds the deterministic attachment ids
+     * (`att_<admissionId>_<index>`), so an exact redelivery of the same
+     * dispatch/submission id materializes to an identical payload.
+     */
+    async function materializeForAdmission(input, scope, admissionId) {
+        const store = options.attachments?.store;
+        if (!store || !messageHasDataAttachments(input.message))
+            return input;
+        const message = await materializeMessageAttachments(input.message, store, {
+            scope,
+            idPrefix: `att_${admissionId}`,
+        });
+        return { ...input, message };
+    }
+    // ── Public interface ─────────────────────────────────────────────────
+    const runner = {
+        async admitDispatch(input) {
+            if (stopping)
+                throw new Error('[fabric-harness] Submission runner is shutting down.');
+            await options.validateInput?.(createDispatchAgentSubmissionInput(input));
+            let admissionInput = await materializeForAdmission(input, submissionStoreSessionId(input), input.dispatchId);
+            if (admissionInput !== input) {
+                // Also point the legacy `input` field at the materialized message so
+                // the persisted payload does not retain a base64 copy through it.
+                admissionInput = { ...admissionInput, input: admissionInput.message };
+            }
+            const admission = await submissions.admitDispatch(admissionInput);
+            if (admission.kind !== 'submission')
+                return admission;
+            let submission = admission.submission;
+            if (submission.canonicalReadyAt === null) {
+                submission = (await submissions.markSubmissionCanonicalReady(submission.submissionId)) ?? submission;
+                emitSubmissionTelemetry(options.telemetry, { type: 'submission_accepted', ...telemetryBase(submission) }, telemetryError);
+            }
+            ensureClaimLoop();
+            wake();
+            return { kind: 'submission', submission };
+        },
+        async admitDirect(input) {
+            if (stopping)
+                throw new Error('[fabric-harness] Submission runner is shutting down.');
+            await options.validateInput?.(input);
+            const admissionInput = await materializeForAdmission(input, submissionStoreSessionId(input), input.submissionId);
+            const admitted = await submissions.admitDirect(admissionInput);
+            if (admitted.canonicalReadyAt === null) {
+                await submissions.markSubmissionCanonicalReady(admitted.submissionId);
+                emitSubmissionTelemetry(options.telemetry, { type: 'submission_accepted', ...telemetryBase(admitted) }, telemetryError);
+            }
+            ensureClaimLoop();
+            wake();
+            return admitted;
+        },
+        async requestAbort(storeSessionId) {
+            const affected = await submissions.requestSessionAbort(storeSessionId);
+            if (affected.length === 0)
+                return affected;
+            let hasInactive = false;
+            for (const submissionId of affected) {
+                const active = activeSubmissions.get(submissionId);
+                if (active)
+                    active.abort.abort(new SubmissionAbortedError());
+                else
+                    hasInactive = true;
+            }
+            ensureClaimLoop();
+            wake();
+            if (hasInactive) {
+                void reconcileRunningSubmissions().catch((error) => {
+                    onError('reconcile after abort failed', error);
+                });
+            }
+            return affected;
+        },
+        async reconcile() {
+            if (!(await submissions.hasUnsettledSubmissions()))
+                return;
+            ensureClaimLoop();
+            await reconcileRunningSubmissions();
+            await runner.waitForIdle();
+        },
+        async waitForSettlement(submissionId, waitOptions = {}) {
+            const local = localSettlements.get(submissionId);
+            if (local)
+                return local;
+            const pollIntervalMs = waitOptions.pollIntervalMs ?? 250;
+            return new Promise((resolve, reject) => {
+                let done = false;
+                let timer = null;
+                const finish = (settlement) => {
+                    if (done)
+                        return;
+                    done = true;
+                    if (timer)
+                        clearTimeout(timer);
+                    waitOptions.signal?.removeEventListener('abort', onAbort);
+                    resolve(settlement);
+                };
+                const onAbort = () => {
+                    if (done)
+                        return;
+                    done = true;
+                    if (timer)
+                        clearTimeout(timer);
+                    listeners?.delete(finish);
+                    reject(waitOptions.signal?.reason instanceof Error ? waitOptions.signal.reason : new Error('waitForSettlement aborted'));
+                };
+                let listeners = settlementListeners.get(submissionId);
+                if (!listeners) {
+                    listeners = new Set();
+                    settlementListeners.set(submissionId, listeners);
+                }
+                listeners.add(finish);
+                waitOptions.signal?.addEventListener('abort', onAbort, { once: true });
+                // Cross-process fallback: poll the store; outcome precision comes
+                // from the canonical settlement record when one exists.
+                const poll = async () => {
+                    if (done)
+                        return;
+                    try {
+                        const submission = await submissions.getSubmission(submissionId);
+                        if (submission?.status === 'settled') {
+                            const canonical = await executor.getSettlement(submission).catch(() => null);
+                            finish(canonical
+                                ? {
+                                    outcome: canonical.outcome,
+                                    ...(canonical.resultText !== undefined ? { result: canonical.resultText } : {}),
+                                    ...(canonical.error !== undefined ? { error: canonical.error } : {}),
+                                }
+                                : submission.error !== undefined
+                                    ? { outcome: 'failed', error: submission.error }
+                                    : { outcome: 'completed' });
+                            return;
+                        }
+                    }
+                    catch (error) {
+                        onError(`settlement poll failed for ${submissionId}`, error);
+                    }
+                    if (!done)
+                        timer = setTimeout(poll, pollIntervalMs);
+                    if (timer && typeof timer === 'object' && 'unref' in timer)
+                        timer.unref();
+                };
+                void poll();
+            });
+        },
+        async waitForIdle() {
+            while (true) {
+                if (stopping)
+                    return;
+                if (activeSubmissions.size > 0) {
+                    await Promise.allSettled([...activeSubmissions.values()].map((entry) => entry.task));
+                }
+                if (stopping)
+                    return;
+                // Let the claim loop's wake → claim pass cycle pick up newly
+                // runnable work from session-head advancement.
+                await new Promise((resolve) => setTimeout(resolve, 10));
+                if (activeSubmissions.size === 0) {
+                    const runnable = await submissions.listRunnableSubmissions();
+                    if (runnable.length === 0)
+                        break;
+                    if (stopping)
+                        return;
+                    wake();
+                }
+            }
+        },
+        async shutdown(timeoutMs = 30_000) {
+            if (stopping)
+                return;
+            stopping = true;
+            wake();
+            for (const { abort } of activeSubmissions.values()) {
+                abort.abort(new DOMException('Submission runner shutting down.', 'AbortError'));
+            }
+            if (activeSubmissions.size > 0) {
+                const settlement = Promise.allSettled([...activeSubmissions.values()].map((entry) => entry.task));
+                const timeout = new Promise((resolve) => {
+                    const timer = setTimeout(resolve, timeoutMs);
+                    void settlement.finally(() => clearTimeout(timer));
+                });
+                await Promise.race([settlement, timeout]);
+            }
+            if (heartbeatInterval) {
+                clearInterval(heartbeatInterval);
+                heartbeatInterval = null;
+            }
+            if (leaseScanInterval) {
+                clearInterval(leaseScanInterval);
+                leaseScanInterval = null;
+            }
+            if (activeSubmissions.size > 0) {
+                onError('shutdown timeout', new Error(`${activeSubmissions.size} submission(s) did not settle within ${timeoutMs}ms; leases will be reclaimed on next startup.`));
+            }
+        },
+        wake,
+        createDispatchQueue() {
+            return {
+                enqueue: async (input) => {
+                    const admission = await runner.admitDispatch(input);
+                    if (admission.kind === 'retained_receipt') {
+                        return {
+                            dispatchId: admission.receipt.submissionId,
+                            acceptedAt: new Date(admission.receipt.acceptedAt).toISOString(),
+                        };
+                    }
+                    if (admission.kind === 'conflict') {
+                        throw new Error(`[fabric-harness] dispatch target agent "${input.agent}" rejected a conflicting dispatch replay.`);
+                    }
+                    return { dispatchId: admission.submission.submissionId, acceptedAt: input.acceptedAt };
+                },
+            };
+        },
+    };
+    return runner;
+}
+/** Store-session FIFO key of a submission (re-exported convenience). */
+export function submissionSessionKey(input) {
+    return submissionStoreSessionId(input);
+}
+function isAbortError(error) {
+    return error instanceof DOMException && error.name === 'AbortError';
+}
+//# sourceMappingURL=submission-runner.js.map