@karowanorg/orc-core 0.1.0

package/dist/supervisor.js

@@ -0,0 +1,1072 @@
+/**
+ * The run supervisor: owns one run end-to-end — program VM, leaf dispatch with
+ * maxParallel, the WAL journal (fsync before delivery), the trace sidecar,
+ * approvals via control.jsonl, cancellation, replay, and fail-forward resume
+ * (write leaves re-dispatch as re-orienting attempts, never blind re-runs).
+ */
+import * as fs from "node:fs";
+import * as path from "node:path";
+import { randomUUID } from "node:crypto";
+import { DEFAULT_POLICY, DivergenceError, PolicyError, } from "./contracts.js";
+import { boundString, canonicalJson, digestJson, sha256Hex } from "./canonical.js";
+import { validateAgainstSchema } from "./jsonschema.js";
+import { compileProgram } from "./compile.js";
+import { ProgramVM } from "./engine.js";
+import { JsonlAppender, acquireLock, createRunDir, newRunId, readControl, readJournal, readManifest, readResult, readTraces, runPaths, writeResult, } from "./rundir.js";
+import { openApprovals, statusForRun } from "./status.js";
+const READ_ONLY_SHUTDOWN_GRACE_MS = 1_000;
+export const ORC_VERSION = "0.1.0";
+/** Static pre-scan: does the program source declare any write leaf? */
+export function sourceRequestsWrite(source) {
+    return /readOnly\s*:\s*false/.test(source);
+}
+export async function prepareRun(opts, registry) {
+    const { bundle, sha256 } = await compileProgram(opts.programPath);
+    // For local runs, canonicalize + validate the cwd against the local FS. For
+    // remote (host) runs the cwd lives on the remote host — keep it as an
+    // absolute path and let `orc doctor --host` / the leaf validate existence.
+    let cwd;
+    if (opts.host) {
+        cwd = opts.cwd ?? ".";
+    }
+    else {
+        cwd = fs.realpathSync(path.resolve(opts.cwd ?? process.cwd()));
+        if (!fs.statSync(cwd).isDirectory())
+            throw new Error(`cwd is not a directory: ${cwd}`);
+    }
+    const brief = opts.brief?.trim();
+    if (!brief)
+        throw new Error("brief is required");
+    if (opts.budgetUsd !== undefined && !(opts.budgetUsd > 0)) {
+        throw new Error("budget must be a positive USD amount");
+    }
+    const manifest = {
+        runId: newRunId(opts.name ?? path.basename(opts.programPath).replace(/\.[^.]+$/, "")),
+        name: opts.name,
+        programPath: path.resolve(opts.programPath),
+        programSha256: sha256,
+        cwd,
+        host: opts.host,
+        brief,
+        allowWrites: opts.allowWrites ?? false,
+        approvalMode: opts.approvalMode ?? "auto",
+        sandbox: opts.sandbox ?? false,
+        sandboxDirs: opts.sandboxDirs ?? [],
+        maxParallel: Math.min(opts.maxParallel ?? DEFAULT_POLICY.maxParallel, 64),
+        idleTimeoutMs: opts.idleTimeout ?? 15 * 60_000,
+        budgetUsd: opts.budgetUsd,
+        defaultHarness: opts.defaultHarness ?? registry.defaultHarness,
+        createdAtMs: Date.now(),
+        orcVersion: ORC_VERSION,
+    };
+    createRunDir(manifest, bundle);
+    return manifest;
+}
+export async function superviseRun(runId, registry, hooks = {}, policy = DEFAULT_POLICY) {
+    const manifest = readManifest(runId);
+    const paths = runPaths(runId);
+    const lock = await acquireLock(paths);
+    let journalOut;
+    let traceOut;
+    let controlWatcher;
+    try {
+        journalOut = new JsonlAppender(paths.journal);
+        traceOut = new JsonlAppender(paths.traces);
+        const sup = new Supervisor(manifest, paths, registry, hooks, policy, journalOut, traceOut);
+        // Watch control.jsonl so approval responses / cancels are picked up within
+        // milliseconds instead of on the next poll tick — responsive UI approvals.
+        fs.closeSync(fs.openSync(paths.control, "a")); // touch: create if absent, never modify
+        try {
+            controlWatcher = fs.watch(paths.control, () => sup.wakeForControl());
+        }
+        catch {
+            /* fs.watch unsupported here; the 1s poll still covers it */
+        }
+        let ownershipError;
+        void lock.lost.catch((err) => {
+            ownershipError = err instanceof Error ? err : new Error(String(err));
+            sup.loseOwnership(ownershipError);
+        });
+        const status = await sup.run();
+        if (ownershipError)
+            throw ownershipError;
+        return status;
+    }
+    finally {
+        controlWatcher?.close();
+        journalOut?.close();
+        traceOut?.close();
+        await lock.release();
+        hooks.onUpdate?.(runId);
+    }
+}
+class Supervisor {
+    manifest;
+    paths;
+    registry;
+    hooks;
+    policy;
+    journalOut;
+    traceOut;
+    vm;
+    newCalls = [];
+    callRecords = []; // full history incl. journal-loaded
+    specBySeq = new Map();
+    dispatchQueue = [];
+    inflight = new Map();
+    completed = [];
+    completionSignal = null;
+    attemptBySeq = new Map();
+    pendingApprovals = new Map();
+    controlConsumed = 0;
+    cancelled = false;
+    terminalError = null;
+    terminalErrorSeq;
+    /** Latest cumulative cost for each durable leaf attempt. */
+    costByAttempt = new Map();
+    leafTasks = new Set();
+    writeLeafTasks = new Set();
+    stopping = false;
+    replayCallCursor = 0; // verified against callRecords during replay
+    phaseNames = [];
+    constructor(manifest, paths, registry, hooks, policy, journalOut, traceOut) {
+        this.manifest = manifest;
+        this.paths = paths;
+        this.registry = registry;
+        this.hooks = hooks;
+        this.policy = policy;
+        this.journalOut = journalOut;
+        this.traceOut = traceOut;
+    }
+    async run() {
+        const journal = readJournal(this.manifest.runId);
+        const alreadyFinished = lastFinish(journal);
+        let latestFinish;
+        for (const record of journal) {
+            if (record.t === "finish")
+                latestFinish = record;
+        }
+        const controls = readControl(this.manifest.runId);
+        // A finish durably records the old control epoch. This also survives a
+        // resume that wrote its retry marker and then crashed.
+        this.controlConsumed =
+            latestFinish?.controlOffset === undefined
+                ? alreadyFinished
+                    ? controls.length // legacy finish record
+                    : 0
+                : Math.min(latestFinish.controlOffset, controls.length);
+        const bundle = fs.readFileSync(this.paths.program, "utf8");
+        if (sha256Hex(bundle) !== this.manifest.programSha256) {
+            throw new DivergenceError("program bundle does not match manifest hash", {});
+        }
+        // Attempt starts and spend are durable so crashes cannot reset retry
+        // allowance or make prior attempts free. Completion records preserve
+        // compatibility with runs created before attempt-start records existed.
+        for (const rec of journal) {
+            if (rec.t === "attempt" || rec.t === "done") {
+                this.attemptBySeq.set(rec.seq, Math.max(this.attemptBySeq.get(rec.seq) ?? 0, rec.attempt));
+            }
+            else if (rec.t === "cost") {
+                this.costByAttempt.set(attemptKey(rec.seq, rec.attempt), rec.costUsd);
+            }
+        }
+        // Old runs recorded cost only in traces. Recover the latest revision for
+        // each attempt, but never override a durable cost record.
+        if (this.manifest.budgetUsd !== undefined) {
+            const legacyCosts = new Map();
+            for (const tr of readTraces(this.manifest.runId)) {
+                if (tr.t !== "leaf" || tr.costUsd === undefined)
+                    continue;
+                const key = attemptKey(tr.seq, tr.attempt);
+                const current = legacyCosts.get(key);
+                if (!current || tr.rev >= current.rev)
+                    legacyCosts.set(key, { rev: tr.rev, cost: tr.costUsd });
+            }
+            for (const [key, value] of legacyCosts) {
+                if (!this.costByAttempt.has(key))
+                    this.costByAttempt.set(key, value.cost);
+            }
+            this.checkBudget();
+        }
+        // ---- fail-forward resume bookkeeping -----------------------------------
+        const priorCalls = journal.filter((r) => r.t === "call");
+        this.callRecords = [...priorCalls];
+        const effective = effectiveCompletions(journal);
+        let retrySeqs = [];
+        let persistRetry = false;
+        if (alreadyFinished && alreadyFinished.status !== "completed") {
+            if (alreadyFinished.status === "failed") {
+                // A failure followed by another call was observed and handled by the
+                // program; rewriting it would rewrite history. The only safe automatic
+                // retry is the exact leaf recorded as the terminal cause. Call sequence
+                // is dispatch order, so a concurrent causal failure need not be the
+                // highest sequence number.
+                if (alreadyFinished.errorSeq !== undefined &&
+                    effective.get(alreadyFinished.errorSeq)?.record.status === "error") {
+                    retrySeqs = [alreadyFinished.errorSeq];
+                }
+                else if (alreadyFinished.errorSeq === undefined) {
+                    // Legacy finishes lack causal metadata. Keep their conservative
+                    // final-call string check rather than guessing across concurrency.
+                    const finalSeq = priorCalls.at(-1)?.seq;
+                    const finalCompletion = finalSeq === undefined ? undefined : effective.get(finalSeq)?.record;
+                    if (finalSeq !== undefined &&
+                        finalCompletion?.status === "error" &&
+                        finalCompletion.error !== undefined &&
+                        (alreadyFinished.error === finalCompletion.error ||
+                            alreadyFinished.error?.startsWith(`${finalCompletion.error}\n`))) {
+                        retrySeqs = [finalSeq];
+                    }
+                }
+                if (retrySeqs.length === 0) {
+                    throw new Error(`run ${this.manifest.runId} has no unambiguous terminal leaf failure to retry; launch a new run`);
+                }
+            }
+            for (const seq of retrySeqs)
+                effective.delete(seq);
+            persistRetry = true;
+        }
+        else if (alreadyFinished?.status === "completed") {
+            throw new Error(`run ${this.manifest.runId} already completed`);
+        }
+        // A hard-killed owner cannot resolve approvals in its finally path. Close
+        // those historical requests before a resumed attempt can ask again.
+        for (const approval of openApprovals(readTraces(this.manifest.runId))) {
+            this.traceEvent({
+                kind: "approval-resolved",
+                approvalId: approval.id,
+                decision: { behavior: "deny", message: "approval expired when the supervisor restarted" },
+                by: "supervisor",
+            });
+        }
+        const replaying = priorCalls.length > 0;
+        try {
+            // ---- boot the VM (the initial drain can already trip the step budget)
+            this.vm = await ProgramVM.create(bundle, this.policy, {
+                onCall: (seq, spec) => this.onCall(seq, spec, replaying),
+                onLog: (m) => this.traceEvent({ kind: "log", message: m }),
+                onPhase: (name) => {
+                    if (!this.phaseNames.includes(name))
+                        this.phaseNames.push(name);
+                    this.traceEvent({ kind: "phase", name });
+                },
+            });
+            return await this.execute(effective, retrySeqs, persistRetry);
+        }
+        catch (err) {
+            // Policy violations (write gate, caps, step budget) are TERMINAL RUN
+            // OUTCOMES, not supervisor crashes. Divergence stays a loud throw.
+            if (err instanceof PolicyError) {
+                this.finishFailed(err.message);
+                return statusForRun(this.manifest.runId);
+            }
+            throw err;
+        }
+        finally {
+            await this.stopAll("run ended");
+            this.vm?.dispose();
+        }
+    }
+    loseOwnership(error) {
+        this.terminalError = `supervisor ownership lost: ${error.message}`;
+        this.abortAll(this.terminalError);
+        this.signal();
+    }
+    async execute(effective, retrySeqs, persistRetry) {
+        {
+            // ---- replay: deliver effective completions in journal order ----------
+            const deliveryOrder = [...effective.values()].sort((a, b) => a.index - b.index);
+            for (const { record } of deliveryOrder) {
+                this.checkNewCallsAgainstJournal();
+                if (!this.vm.pendingSeqs().includes(record.seq)) {
+                    throw new DivergenceError(`journal has a completion for seq ${record.seq} the program never requested (dangling completion)`, { seq: record.seq });
+                }
+                this.deliver(record.seq, this.materializeOutcome(record));
+            }
+            this.checkNewCallsAgainstJournal();
+            // Unconsumed suffix check: every prior call must have been re-made.
+            if (this.replayCallCursor < this.callRecords.length) {
+                const pendingReplayable = this.callRecords.slice(this.replayCallCursor).map((c) => c.seq);
+                const vmPending = new Set(this.vm.pendingSeqs());
+                const nevermade = pendingReplayable.filter((s) => !vmPending.has(s) && !this.specBySeq.has(s));
+                if (nevermade.length > 0 && this.vm.state().state !== "pending") {
+                    throw new DivergenceError(`program completed replay with unconsumed journal calls: seq ${nevermade.join(",")}`, {});
+                }
+            }
+            // Persist only after the candidate history replayed cleanly. A failed
+            // resume must not mutate the journal into a permanently re-armed state.
+            if (persistRetry) {
+                this.journalOut.append({ t: "retry", seqs: retrySeqs, atMs: Date.now() });
+            }
+            // ---- re-dispatch undelivered calls (re-orient for writes) ------------
+            for (const seq of this.vm.pendingSeqs()) {
+                if (!this.dispatchQueue.includes(seq) && !this.inflight.has(seq)) {
+                    const spec = this.specBySeq.get(seq);
+                    if (spec) {
+                        const reorient = !spec.readOnly || retrySeqs.includes(seq);
+                        this.dispatchQueue.push(seq);
+                        if (reorient)
+                            this.markReorient(seq);
+                    }
+                }
+            }
+            // ---- live loop -------------------------------------------------------
+            await this.liveLoop();
+        }
+        return statusForRun(this.manifest.runId);
+    }
+    // -------------------------------------------------------------------------
+    reorientSeqs = new Set();
+    markReorient(seq) {
+        this.reorientSeqs.add(seq);
+    }
+    onCall(seq, spec, verifyAgainstJournal) {
+        // ext leaves: readOnly comes from the registration, not the program.
+        if (spec.kind.startsWith("ext:")) {
+            const ext = this.registry.extensions.get(spec.kind.slice(4));
+            if (ext)
+                spec.readOnly = ext.readOnly;
+        }
+        this.specBySeq.set(seq, spec);
+        this.newCalls.push({ seq, spec });
+        void verifyAgainstJournal; // verification happens in checkNewCallsAgainstJournal
+    }
+    /** Journal (or verify) calls the VM made during the last drain. */
+    checkNewCallsAgainstJournal() {
+        for (const { seq, spec } of this.newCalls) {
+            const digest = digestJson(spec);
+            if (this.replayCallCursor < this.callRecords.length) {
+                const expected = this.callRecords[this.replayCallCursor];
+                if (expected.seq !== seq || expected.specDigest !== digest) {
+                    throw new DivergenceError(`call ${seq} does not match the journal (expected seq ${expected.seq})`, { seq, expected: expected.specDigest, got: digest });
+                }
+                this.replayCallCursor++;
+            }
+            else {
+                if (seq >= this.policy.maxCommands) {
+                    throw new PolicyError(`program exceeded maxCommands (${this.policy.maxCommands})`);
+                }
+                if (spec.prompt && Buffer.byteLength(spec.prompt) > this.policy.maxPromptBytes) {
+                    throw new PolicyError(`prompt for call ${seq} exceeds ${this.policy.maxPromptBytes} bytes`);
+                }
+                // Extension payloads are subject to the same byte cap as prompts, and
+                // are validated against the extension's declared inputSchema.
+                if (spec.kind.startsWith("ext:")) {
+                    const name = spec.kind.slice(4);
+                    const ext = this.registry.extensions.get(name);
+                    if (!ext)
+                        throw new PolicyError(`call ${seq} uses unregistered extension ext.${name}`);
+                    const payloadBytes = Buffer.byteLength(canonicalJson(spec.payload ?? null));
+                    if (payloadBytes > this.policy.maxPromptBytes) {
+                        throw new PolicyError(`ext.${name} payload for call ${seq} exceeds ${this.policy.maxPromptBytes} bytes`);
+                    }
+                    if (ext.inputSchema !== undefined) {
+                        const problem = validateAgainstSchema(spec.payload ?? null, ext.inputSchema);
+                        if (problem)
+                            throw new PolicyError(`ext.${name} payload for call ${seq} fails inputSchema: ${problem}`);
+                    }
+                }
+                if (!spec.readOnly && !this.manifest.allowWrites) {
+                    throw new PolicyError(`write-declared leaf (seq ${seq}) but the run was launched without allow_writes — fail-closed`);
+                }
+                const rec = {
+                    t: "call",
+                    seq,
+                    kind: spec.kind,
+                    id: spec.id,
+                    phase: spec.phase,
+                    readOnly: spec.readOnly,
+                    specDigest: digest,
+                };
+                this.journalOut.append(rec); // WAL: call journaled before dispatch
+                this.callRecords.push(rec);
+                this.replayCallCursor++;
+                this.dispatchQueue.push(seq);
+            }
+        }
+        this.newCalls = [];
+    }
+    materializeOutcome(rec) {
+        if (rec.status === "ok" && rec.resultSha) {
+            return { status: "ok", value: readResult(this.paths, rec.resultSha) };
+        }
+        return { status: "error", error: rec.error ?? "unknown leaf error" };
+    }
+    // -------------------------------------------------------------------------
+    async liveLoop() {
+        for (;;) {
+            this.pollControl();
+            this.checkNewCallsAgainstJournal();
+            if (this.terminalError) {
+                this.finishFailed(this.terminalError);
+                return;
+            }
+            if (this.cancelled) {
+                this.abortAll("cancelled");
+                this.journalOut.append({
+                    t: "finish",
+                    status: "cancelled",
+                    error: "cancelled by operator",
+                    controlOffset: this.controlConsumed,
+                });
+                this.hooks.onUpdate?.(this.manifest.runId);
+                return;
+            }
+            const state = this.vm.state();
+            if (state.state !== "pending" &&
+                this.inflight.size === 0 &&
+                this.dispatchQueue.length === 0 &&
+                this.completed.length === 0) {
+                this.finish(state);
+                return;
+            }
+            this.pumpDispatch();
+            if (state.state === "pending" &&
+                this.inflight.size === 0 &&
+                this.dispatchQueue.length === 0 &&
+                this.completed.length === 0 &&
+                this.newCalls.length === 0) {
+                this.finishFailed("program is awaiting something orc will never resolve (deadlock)");
+                return;
+            }
+            if (this.completed.length > 0) {
+                // Deliver exactly one completion per quiescent drain (frozen policy).
+                const next = this.completed.shift();
+                this.deliverLive(next);
+                continue;
+            }
+            await this.waitForSignal(1_000);
+            this.checkIdleTimeouts();
+        }
+    }
+    pumpDispatch() {
+        while (this.inflight.size < this.manifest.maxParallel && this.dispatchQueue.length > 0) {
+            const seq = this.dispatchQueue.shift();
+            const spec = this.specBySeq.get(seq);
+            if (!spec)
+                continue;
+            const attempt = (this.attemptBySeq.get(seq) ?? 0) + 1;
+            this.attemptBySeq.set(seq, attempt);
+            this.journalOut.append({ t: "attempt", seq, attempt, atMs: Date.now() });
+            const leaf = {
+                seq,
+                attempt,
+                spec,
+                abort: new AbortController(),
+                lastEventAtMs: Date.now(),
+                idleTimeoutMs: spec.idleTimeoutMs ?? this.manifest.idleTimeoutMs,
+                groupId: spec.groupId,
+            };
+            this.inflight.set(seq, leaf);
+            let task;
+            task = this.executeLeaf(leaf)
+                .then((outcome) => this.onLeafDone(leaf, outcome))
+                .catch((err) => this.onLeafDone(leaf, {
+                status: "error",
+                error: boundString(String(err instanceof Error ? err.stack ?? err.message : err), this.policy.maxErrorBytes),
+            }))
+                .finally(() => {
+                this.leafTasks.delete(task);
+                this.writeLeafTasks.delete(task);
+                this.signal();
+            });
+            this.leafTasks.add(task);
+            if (!spec.readOnly)
+                this.writeLeafTasks.add(task);
+        }
+    }
+    onLeafDone(leaf, outcome) {
+        if (this.stopping || !this.inflight.has(leaf.seq))
+            return; // already aborted+settled
+        this.inflight.delete(leaf.seq);
+        // Supervisor retry table: a failed READ-ONLY leaf gets a bounded number of
+        // fresh attempts before it (and its parallel group) is failed. Write leaves
+        // are never auto-retried (a retry would double-apply mutations); a cancelled
+        // leaf is not retried either.
+        if (outcome.status === "error" &&
+            leaf.spec.readOnly &&
+            !leaf.abort.signal.aborted &&
+            isRetryable(outcome.error) &&
+            leaf.attempt <= this.policy.readOnlyRetries) {
+            this.traceEvent({
+                kind: "log",
+                message: `leaf ${leaf.seq} failed (read-only) → retry ${leaf.attempt}/${this.policy.readOnlyRetries}`,
+            });
+            this.dispatchQueue.push(leaf.seq); // re-dispatch with a fresh attempt
+            this.signal();
+            return;
+        }
+        // A failed leaf no longer cancels its parallel() siblings: every lane runs
+        // to completion independently and parallel() returns a per-lane outcome, so
+        // one malformed lane never wastes the rest of a fan-out. (Fail-fast is still
+        // available: a raw Promise.all over agent() rejects on the first failure.)
+        this.completed.push({ seq: leaf.seq, attempt: leaf.attempt, outcome });
+        this.signal();
+    }
+    deliverLive(done) {
+        let outcome = done.outcome;
+        if (outcome.status === "ok") {
+            const size = Buffer.byteLength(canonicalJson(outcome.value));
+            if (size > this.policy.maxResultBytes) {
+                outcome = { status: "error", error: `result exceeds cap (${size} > ${this.policy.maxResultBytes} bytes)` };
+            }
+        }
+        let rec;
+        if (outcome.status === "ok") {
+            const { sha, sizeBytes } = writeResult(this.paths, outcome.value);
+            rec = { t: "done", seq: done.seq, status: "ok", resultSha: sha, sizeBytes, attempt: done.attempt };
+        }
+        else {
+            rec = {
+                t: "done",
+                seq: done.seq,
+                status: "error",
+                // LeafOutcome errors are bounded exactly once at their source. Reusing
+                // the same string here keeps live and replay identity identical.
+                error: outcome.error,
+                attempt: done.attempt,
+            };
+        }
+        // WAL invariant: fsync the completion BEFORE delivering it into the sandbox.
+        this.journalOut.append(rec, { fsync: true });
+        this.deliver(done.seq, outcome);
+        this.hooks.onUpdate?.(this.manifest.runId);
+    }
+    /** Deliver identically during replay and live execution, including causality. */
+    deliver(seq, outcome) {
+        const before = this.vm.state();
+        this.vm.deliver(seq, outcome);
+        if (before.state === "pending" && outcome.status === "error") {
+            const after = this.vm.state();
+            if (after.state === "error" &&
+                (after.error === outcome.error || after.error.startsWith(`${outcome.error}\n`))) {
+                this.terminalErrorSeq = seq;
+            }
+        }
+    }
+    finish(state) {
+        if (state.state === "ok") {
+            let result;
+            try {
+                result = this.validateResult(state.result ?? null);
+            }
+            catch (err) {
+                throw new PolicyError(String(err instanceof Error ? err.message : err));
+            }
+            const { sha } = writeResult(this.paths, result);
+            this.journalOut.append({
+                t: "finish",
+                status: "completed",
+                resultSha: sha,
+                controlOffset: this.controlConsumed,
+            });
+        }
+        else if (state.state === "error") {
+            this.journalOut.append({
+                t: "finish",
+                status: "failed",
+                error: boundString(state.error ?? "program failed", this.policy.maxErrorBytes),
+                errorSeq: this.terminalErrorSeq,
+                controlOffset: this.controlConsumed,
+            });
+        }
+        this.hooks.onUpdate?.(this.manifest.runId);
+    }
+    finishFailed(error) {
+        this.abortAll(error);
+        this.journalOut.append({
+            t: "finish",
+            status: "failed",
+            error: boundString(error, this.policy.maxErrorBytes),
+            controlOffset: this.controlConsumed,
+        });
+        this.hooks.onUpdate?.(this.manifest.runId);
+    }
+    abortAll(reason) {
+        for (const leaf of this.inflight.values())
+            leaf.abort.abort(new Error(reason));
+    }
+    async stopAll(reason) {
+        this.stopping = true;
+        this.abortAll(reason);
+        const tasks = [...this.leafTasks];
+        if (tasks.length === 0)
+            return;
+        const writeTasks = new Set(this.writeLeafTasks);
+        const readTasks = tasks.filter((task) => !writeTasks.has(task));
+        // Write tasks retain exclusive ownership until each one actually stops;
+        // a stuck read-only sibling must not extend that requirement.
+        await Promise.allSettled(writeTasks);
+        if (readTasks.length === 0)
+            return;
+        let timer;
+        try {
+            await Promise.race([
+                Promise.allSettled(readTasks),
+                new Promise((resolve) => {
+                    timer = setTimeout(resolve, READ_ONLY_SHUTDOWN_GRACE_MS);
+                }),
+            ]);
+        }
+        finally {
+            if (timer)
+                clearTimeout(timer);
+        }
+    }
+    // -------------------------------------------------------------------------
+    async executeLeaf(leaf) {
+        const { spec, seq, attempt } = leaf;
+        const host = spec.host ?? this.manifest.host;
+        const cwd = spec.cwd ?? this.manifest.cwd;
+        const executor = this.registry.executorFor(host);
+        let rev = 0;
+        const startMs = Date.now();
+        const toolCalls = new Map();
+        let tokensIn;
+        let tokensOut;
+        let costUsd;
+        let costEstimated;
+        let sessionId;
+        let resolvedModel = spec.model;
+        let resolvedEffort = spec.reasoningEffort;
+        const base = {
+            t: "leaf",
+            seq,
+            attempt,
+            id: spec.id,
+            phase: spec.phase,
+            kind: spec.kind,
+            harness: spec.kind === "agent" ? (spec.harness ?? this.manifest.defaultHarness) : undefined,
+            host,
+            cwd,
+            readOnly: spec.readOnly,
+            startMs,
+            prompt: spec.prompt ? boundString(spec.prompt, 16 * 1024) : undefined,
+            brief: boundString(this.manifest.brief, 4 * 1024),
+            reoriented: this.reorientSeqs.has(seq) || undefined,
+        };
+        const emitTrace = (status, extra = {}, fsync = false) => {
+            if (this.stopping)
+                return;
+            this.traceOut.append({
+                ...base,
+                rev: rev++,
+                status,
+                toolCalls: [...toolCalls.values()],
+                model: resolvedModel,
+                reasoningEffort: resolvedEffort,
+                tokensIn,
+                tokensOut,
+                costUsd,
+                costEstimated,
+                sessionId,
+                ...extra,
+            }, { fsync });
+            this.hooks.onUpdate?.(this.manifest.runId);
+        };
+        emitTrace("running");
+        const ctx = {
+            executor,
+            signal: leaf.abort.signal,
+            // Harness stderr/tracing is per-leaf detail, not run narrative: record it
+            // as an hlog trace (drawer's collapsed "Harness log"), never as a feed event.
+            log: (m) => this.harnessLog(seq, m),
+            requestApproval: (req) => this.bridgeApproval(req, leaf),
+        };
+        try {
+            if (spec.kind.startsWith("ext:")) {
+                const name = spec.kind.slice(4);
+                const ext = this.registry.extensions.get(name);
+                if (!ext)
+                    throw new Error(`unknown extension leaf: ext.${name} (register it in orc.config)`);
+                const value = this.validateResult((await ext.execute(spec.payload ?? null, ctx)) ?? null);
+                emitTrace("ok", { endMs: Date.now(), output: value }, true);
+                return { status: "ok", value };
+            }
+            const harnessName = spec.harness ?? this.manifest.defaultHarness;
+            const harness = this.registry.harnesses.get(harnessName);
+            if (!harness) {
+                throw new Error(`unknown harness "${harnessName}" (available: ${[...this.registry.harnesses.keys()].join(", ") || "none"})`);
+            }
+            let prompt = spec.prompt ?? "";
+            if (this.reorientSeqs.has(seq)) {
+                prompt = (await this.reorientPreamble(executor, cwd)) + prompt;
+            }
+            const req = {
+                runId: this.manifest.runId,
+                seq,
+                id: spec.id,
+                prompt,
+                system: leafSystemPrompt(spec.readOnly, cwd, this.manifest.brief),
+                brief: this.manifest.brief,
+                schema: spec.schema,
+                model: spec.model,
+                reasoningEffort: spec.reasoningEffort,
+                readOnly: spec.readOnly,
+                cwd,
+                host,
+                approvalMode: this.manifest.approvalMode,
+                idleTimeoutMs: leaf.idleTimeoutMs,
+                sandbox: this.manifest.sandbox,
+                sandboxDirs: this.manifest.sandboxDirs,
+            };
+            let result;
+            let errorMsg;
+            for await (const ev of harness.invoke(req, ctx)) {
+                if (this.stopping)
+                    throw new Error(`aborted: ${String(leaf.abort.signal.reason ?? "run ended")}`);
+                assertHarnessEvent(ev);
+                leaf.lastEventAtMs = Date.now();
+                this.applyEvent(ev, toolCalls, (u) => {
+                    tokensIn = u.tokensIn ?? tokensIn;
+                    tokensOut = u.tokensOut ?? tokensOut;
+                    if (u.costUsd !== undefined) {
+                        if (!Number.isFinite(u.costUsd) || u.costUsd < 0) {
+                            throw new Error(`harness reported invalid costUsd: ${String(u.costUsd)}`);
+                        }
+                        const changed = u.costUsd !== costUsd;
+                        costUsd = u.costUsd;
+                        costEstimated = u.costEstimated ?? false;
+                        this.costByAttempt.set(attemptKey(seq, attempt), u.costUsd);
+                        if (changed) {
+                            this.journalOut.append({
+                                t: "cost",
+                                seq,
+                                attempt,
+                                costUsd: u.costUsd,
+                                costEstimated,
+                                atMs: Date.now(),
+                            });
+                        }
+                        this.checkBudget();
+                    }
+                });
+                if (ev.kind === "session")
+                    sessionId = ev.sessionId;
+                if (ev.kind === "model") {
+                    if (ev.model)
+                        resolvedModel = ev.model;
+                    if (ev.reasoningEffort)
+                        resolvedEffort = ev.reasoningEffort;
+                }
+                if (ev.kind === "result")
+                    result = ev.output;
+                if (ev.kind === "error")
+                    errorMsg = ev.message;
+                if (ev.kind === "tool-call-open" || ev.kind === "tool-call-close")
+                    emitTrace("running");
+                if (ev.kind === "denied") {
+                    this.traceEvent({ kind: "denied", seq, toolName: ev.toolName, reason: ev.reason });
+                }
+            }
+            if (leaf.abort.signal.aborted) {
+                throw new Error(`aborted: ${String(leaf.abort.signal.reason ?? "cancelled")}`);
+            }
+            if (errorMsg !== undefined && result === undefined)
+                throw new Error(errorMsg);
+            if (result === undefined)
+                throw new Error("harness produced no result event");
+            result = this.validateResult(result, spec.schema);
+            emitTrace("ok", { endMs: Date.now(), output: result }, true);
+            return { status: "ok", value: result };
+        }
+        catch (err) {
+            const msg = boundString(String(err instanceof Error ? (err.message ?? err) : err), this.policy.maxErrorBytes);
+            emitTrace("error", { endMs: Date.now(), error: msg }, true);
+            return { status: "error", error: msg };
+        }
+    }
+    validateResult(value, schema) {
+        let canonical;
+        try {
+            canonical = canonicalJson(value);
+        }
+        catch (err) {
+            throw new Error(`result is not valid JSON: ${String(err instanceof Error ? err.message : err)}`);
+        }
+        const size = Buffer.byteLength(canonical);
+        if (size > this.policy.maxResultBytes) {
+            throw new Error(`result exceeds cap (${size} > ${this.policy.maxResultBytes} bytes)`);
+        }
+        const snapshot = JSON.parse(canonical);
+        if (schema !== undefined) {
+            const problem = validateAgainstSchema(snapshot, schema);
+            if (problem)
+                throw new Error(`result fails output schema: ${problem}`);
+        }
+        return snapshot;
+    }
+    applyEvent(ev, toolCalls, usage) {
+        if (ev.kind === "tool-call-open") {
+            toolCalls.set(ev.id, { id: ev.id, name: ev.name, input: ev.input, startMs: ev.atMs, status: "running" });
+        }
+        else if (ev.kind === "tool-call-close") {
+            const tc = toolCalls.get(ev.id);
+            if (tc) {
+                tc.status = ev.status;
+                tc.endMs = ev.atMs;
+                if (ev.result !== undefined)
+                    tc.result = ev.result;
+            }
+        }
+        else if (ev.kind === "usage") {
+            usage(ev);
+        }
+    }
+    async reorientPreamble(executor, cwd) {
+        let snapshot = "";
+        try {
+            const status = await executor.run(["git", "status", "--porcelain"], { cwd, timeoutMs: 5_000 });
+            const diff = await executor.run(["git", "diff", "--stat"], { cwd, timeoutMs: 5_000 });
+            if (status.code === 0) {
+                snapshot = `Observed working-tree state:\n${status.stdout.trim() || "(clean)"}\n${diff.stdout.trim()}`;
+            }
+        }
+        catch {
+            snapshot = "(state snapshot unavailable)";
+        }
+        return (`RE-ORIENT NOTE: A previous attempt at this task may have partially completed before being interrupted. ` +
+            `${snapshot}\nInspect the current state before acting, do not blindly redo work that is already done, ` +
+            `and complete the task idempotently.\n\n---\n\n`);
+    }
+    // -------------------------------------------------------------------------
+    bridgeApproval(req, leaf) {
+        if (this.stopping || leaf.abort.signal.aborted) {
+            return Promise.resolve({ behavior: "deny", message: "leaf is no longer running" });
+        }
+        const id = `a_${leaf.seq}_${randomUUID()}`;
+        const approval = { ...req, id, requestedAtMs: Date.now() };
+        this.traceEvent({ kind: "approval-requested", approval });
+        return new Promise((resolve) => {
+            const settle = (d) => {
+                this.pendingApprovals.delete(id);
+                resolve(d);
+            };
+            this.pendingApprovals.set(id, settle);
+            leaf.abort.signal.addEventListener("abort", () => {
+                if (this.pendingApprovals.get(id) !== settle)
+                    return;
+                const decision = { behavior: "deny", message: "leaf aborted while approval was pending" };
+                this.traceEvent({ kind: "approval-resolved", approvalId: id, decision, by: "supervisor" });
+                settle(decision);
+            });
+        });
+    }
+    /** Called by the control-file watcher: drain control immediately and wake the loop. */
+    wakeForControl() {
+        try {
+            this.pollControl();
+        }
+        catch {
+            /* the loop will re-poll */
+        }
+        this.signal();
+    }
+    pollControl() {
+        const messages = readControl(this.manifest.runId);
+        for (let i = this.controlConsumed; i < messages.length; i++) {
+            const msg = messages[i];
+            if (msg.t === "cancel")
+                this.cancelled = true;
+            if (msg.t === "approval") {
+                const settle = this.pendingApprovals.get(msg.approvalId);
+                if (settle) {
+                    this.traceEvent({ kind: "approval-resolved", approvalId: msg.approvalId, decision: msg.decision, by: msg.by });
+                    settle(msg.decision);
+                }
+            }
+        }
+        this.controlConsumed = messages.length;
+    }
+    checkIdleTimeouts() {
+        const now = Date.now();
+        for (const leaf of this.inflight.values()) {
+            if (typeof leaf.idleTimeoutMs === "number" && now - leaf.lastEventAtMs > leaf.idleTimeoutMs) {
+                const error = `idle timeout: no harness events for ${leaf.idleTimeoutMs}ms`;
+                leaf.abort.abort(new Error(error));
+                // AbortSignal is cooperative. Settle the deterministic leaf outcome
+                // now; a late task result is ignored because the inflight entry is gone.
+                this.onLeafDone(leaf, { status: "error", error });
+            }
+        }
+    }
+    traceEvent(event) {
+        this.traceOut.append({ t: "event", atMs: Date.now(), event }, { fsync: false });
+        this.hooks.onUpdate?.(this.manifest.runId);
+    }
+    harnessLog(seq, message) {
+        if (this.stopping)
+            return;
+        this.traceOut.append({ t: "hlog", seq, atMs: Date.now(), message }, { fsync: false });
+        this.hooks.onUpdate?.(this.manifest.runId);
+    }
+    /**
+     * Budget cap: once the run's summed estimated cost passes manifest.budgetUsd,
+     * fail the run terminally (all inflight leaves are aborted by finishFailed).
+     * Checked on every cost update and once at start (a resume can already be over).
+     */
+    checkBudget() {
+        const budget = this.manifest.budgetUsd;
+        if (budget === undefined || this.terminalError)
+            return;
+        let total = 0;
+        for (const v of this.costByAttempt.values())
+            total += v;
+        if (total > budget) {
+            const msg = `budget exceeded: estimated cost ~$${total.toFixed(2)} passed the $${budget.toFixed(2)} budget`;
+            this.traceEvent({ kind: "log", message: `${msg} — failing run` });
+            this.terminalError = msg;
+            this.signal();
+        }
+    }
+    signal() {
+        this.completionSignal?.();
+        this.completionSignal = null;
+    }
+    waitForSignal(maxMs) {
+        return new Promise((resolve) => {
+            const timer = setTimeout(() => {
+                this.completionSignal = null;
+                resolve();
+            }, maxMs);
+            this.completionSignal = () => {
+                clearTimeout(timer);
+                resolve();
+            };
+        });
+    }
+}
+// ---------------------------------------------------------------------------
+function lastFinish(journal) {
+    let out;
+    let outIdx = -1;
+    journal.forEach((r, i) => {
+        if (r.t === "finish") {
+            out = r;
+            outIdx = i;
+        }
+    });
+    // A retry record after the finish re-arms the run: treat as not finished.
+    const retried = journal.some((r, i) => r.t === "retry" && i > outIdx);
+    return retried ? undefined : out;
+}
+/**
+ * Effective completion per seq: completion records are void if a later retry
+ * record names their seq. Returns map seq -> {record, index-in-journal}.
+ */
+function effectiveCompletions(journal) {
+    const voidedBefore = new Map(); // seq -> journal index of last retry naming it
+    journal.forEach((r, i) => {
+        if (r.t === "retry")
+            for (const seq of r.seqs)
+                voidedBefore.set(seq, i);
+    });
+    const out = new Map();
+    journal.forEach((r, i) => {
+        if (r.t !== "done")
+            return;
+        const voidIdx = voidedBefore.get(r.seq);
+        if (voidIdx !== undefined && i < voidIdx)
+            return; // voided by a later retry
+        out.set(r.seq, { record: r, index: i });
+    });
+    return out;
+}
+/** Deterministic setup errors are not worth retrying. */
+function isRetryable(error) {
+    // Deterministic errors won't change on a re-run, so retrying only wastes a
+    // model call and delays the real failure. Config/routing errors plus the
+    // structured-output schema rejections (OpenAI/codex strict mode) are all
+    // author-fixable, not transient.
+    return !/unknown harness|unknown extension|unregistered extension|fails inputSchema|not supported for remote|allow_writes|invalid_json_schema|invalid schema|unsupported schema|additionalProperties|output[\s_-]?schema|result exceeds cap|result is not valid JSON/i.test(error);
+}
+function attemptKey(seq, attempt) {
+    return `${seq}:${attempt}`;
+}
+function assertHarnessEvent(value) {
+    if (value === null || typeof value !== "object" || Array.isArray(value)) {
+        throw new Error("harness emitted a non-object event");
+    }
+    const event = value;
+    const kind = event.kind;
+    const stringField = (name) => {
+        if (typeof event[name] !== "string")
+            throw new Error(`harness ${String(kind)} event has invalid ${name}`);
+    };
+    const optionalString = (name) => {
+        if (event[name] !== undefined && typeof event[name] !== "string") {
+            throw new Error(`harness ${String(kind)} event has invalid ${name}`);
+        }
+    };
+    const finiteNumber = (name, optional = false) => {
+        const v = event[name];
+        if (optional && v === undefined)
+            return;
+        if (typeof v !== "number" || !Number.isFinite(v) || v < 0) {
+            throw new Error(`harness ${String(kind)} event has invalid ${name}`);
+        }
+    };
+    const jsonField = (name) => {
+        if (event[name] !== undefined)
+            canonicalJson(event[name]);
+    };
+    switch (kind) {
+        case "tool-call-open":
+            stringField("id");
+            stringField("name");
+            finiteNumber("atMs");
+            jsonField("input");
+            break;
+        case "tool-call-close":
+            stringField("id");
+            if (event.status !== "ok" && event.status !== "error") {
+                throw new Error("harness tool-call-close event has invalid status");
+            }
+            finiteNumber("atMs");
+            jsonField("result");
+            break;
+        case "text":
+            stringField("delta");
+            finiteNumber("atMs");
+            break;
+        case "usage":
+            finiteNumber("tokensIn", true);
+            finiteNumber("tokensOut", true);
+            finiteNumber("costUsd", true);
+            if (event.costEstimated !== undefined && typeof event.costEstimated !== "boolean") {
+                throw new Error("harness usage event has invalid costEstimated");
+            }
+            break;
+        case "model":
+            optionalString("model");
+            optionalString("reasoningEffort");
+            break;
+        case "session":
+            stringField("sessionId");
+            break;
+        case "denied":
+            stringField("toolName");
+            stringField("reason");
+            finiteNumber("atMs");
+            break;
+        case "result":
+            if (!Object.prototype.hasOwnProperty.call(event, "output")) {
+                throw new Error("harness result event has no output");
+            }
+            break;
+        case "error":
+            stringField("message");
+            break;
+        default:
+            throw new Error(`harness emitted unknown event kind: ${String(kind)}`);
+    }
+}
+export function leafSystemPrompt(readOnly, cwd, brief) {
+    const mutation = readOnly
+        ? "This task is READ-ONLY: inspect freely but do not modify files, run mutating commands, or change system state."
+        : "This task may modify files under the working directory. Make only the changes the task requires.";
+    return (`You are an orc leaf agent. You start fresh with no memory of other leaves; everything you need is in this prompt.\n` +
+        `Working directory: ${cwd}\n${mutation}\n` +
+        `SHARED CONTEXT (brief):\n${brief}`);
+}
+export { newRunId };