open-multi-agent-kit 0.80.2 → 0.80.4

package/dist/orchestration/executor.js

@@ -1,1075 +0,0 @@
-import { createScheduler } from "./scheduler.js";
-import { createStatePersister } from "./state-persister.js";
-import { createEnsembleTaskRunner } from "./ensemble.js";
-import { estimateRunProgress } from "./eta.js";
-import { dagNodeRoutingEnv } from "./routing.js";
-import { getOmkResourceSettings } from "../util/resource-profile.js";
-import { checkEvidenceGates } from "./evidence-gate.js";
-import { checkEvidenceGate } from "../runtime/contracts/evidence.js";
-import { invalidateTaskDagGraph } from "./task-graph.js";
-import { resolveTimeoutMs } from "../util/timeout-config.js";
-import { createNodeMonitorEngine } from "./node-monitor.js";
-import { appendEvent } from "../util/events-logger.js";
-import { buildTaskRunContext, envFromWorkerManifest } from "../runtime/worker-manifest.js";
-export function createExecutor(executorOptions = {}) {
-    const scheduler = createScheduler();
-    const persister = executorOptions.persister ?? createStatePersister();
-    const stateChangeHandlers = [];
-    const nodeStartHandlers = [];
-    const nodeCompleteHandlers = [];
-    let commitQueue = Promise.resolve();
-    let commitQueueSize = 0;
-    const MAX_COMMIT_QUEUE = 10;
-    const COMMIT_SAVE_TIMEOUT_MS = 10_000;
-    const RUNNER_ABORT_DRAIN_MS = 2_000;
-    const activeTimers = new Map();
-    const nodeAbortRegistry = new Map();
-    let aborting = false;
-    let isShuttingDown = false;
-    let activeDag;
-    let activeTick;
-    let activeRunOptions;
-    let lastTerminalChangeAt = Date.now();
-    let telemetryQueue = Promise.resolve();
-    function emitTelemetry(event) {
-        if (!executorOptions.eventRunDir)
-            return;
-        telemetryQueue = telemetryQueue
-            .then(() => appendEvent(executorOptions.eventRunDir, event))
-            .catch(() => { });
-    }
-    async function flushTelemetry() {
-        await telemetryQueue.catch(() => { });
-    }
-    function bumpTerminalActivity() {
-        lastTerminalChangeAt = Date.now();
-    }
-    function clearActiveTimers() {
-        for (const [, timers] of activeTimers) {
-            clearInterval(timers.progress);
-            clearInterval(timers.persist);
-            clearInterval(timers.heartbeat);
-        }
-        activeTimers.clear();
-    }
-    function markOpenNodesBlocked(dag, reason) {
-        for (const node of dag.nodes) {
-            if (node.status === "running" || node.status === "pending") {
-                node.status = "blocked";
-                node.blockedReason = reason;
-            }
-        }
-    }
-    function rewriteNodeInputFrom(node, fromId, toId) {
-        if (!node.inputs)
-            return;
-        node.inputs = node.inputs.map((input) => input.from === fromId ? { ...input, from: toId } : input);
-    }
-    const monitor = createNodeMonitorEngine({
-        heartbeatIntervalMs: 30_000,
-        stallThresholdMultiplier: 3,
-        onStall: (m) => {
-            process.stderr.write(`[omk] node ${m.nodeId} stalled (no heartbeat for ${m.stallThresholdMs}ms)\n`);
-            emitTelemetry({
-                type: "lane.stalled",
-                runId: m.runId,
-                nodeId: m.nodeId,
-                laneId: m.nodeId,
-                data: { stallThresholdMs: m.stallThresholdMs },
-            });
-        },
-        onKill: (m) => {
-            process.stderr.write(`[omk] node ${m.nodeId} killed (no heartbeat for ${m.stallThresholdMs}ms)\n`);
-            // Forcibly abort the underlying child process so it does not hang.
-            const abortFn = nodeAbortRegistry.get(m.nodeId);
-            if (abortFn) {
-                try {
-                    abortFn();
-                }
-                catch {
-                    // ignore abort errors
-                }
-            }
-            if (activeDag && activeRunOptions) {
-                try {
-                    const node = activeDag.nodes.find((n) => n.id === m.nodeId);
-                    if (node && node.status === "running") {
-                        markNodeFinished(node, "failed");
-                        scheduler.updateNodeStatus(activeDag, m.nodeId, "failed", activeRunOptions.runId);
-                        activeTick?.().catch(() => { });
-                    }
-                }
-                catch {
-                    // ignore kill handler errors
-                }
-            }
-        },
-    });
-    function buildState(dag, options) {
-        const startedAt = new Date().toISOString();
-        return {
-            schemaVersion: 1,
-            runId: options.runId,
-            nodes: dag.nodes.map((n) => ({
-                ...n,
-                dependsOn: [...n.dependsOn],
-                outputs: n.outputs ? n.outputs.map((o) => ({ ...o })) : undefined,
-                routing: n.routing ? { ...n.routing } : undefined,
-                failurePolicy: n.failurePolicy ? { ...n.failurePolicy } : undefined,
-            })),
-            startedAt,
-            updatedAt: startedAt,
-            lastActivityAt: startedAt,
-            lastHeartbeatAt: startedAt,
-            activitySeq: 0,
-            estimate: estimateRunProgress({
-                nodes: dag.nodes,
-                startedAt,
-                workerCount: options.workers,
-            }),
-        };
-    }
-    function isLocalSystemNode(node) {
-        return node.id === "bootstrap" || node.role === "omk" || node.routing?.actionAtom?.verb === "bootstrap";
-    }
-    function markLocalSystemNodesDone(dag, state, options) {
-        let changed = false;
-        for (const node of dag.nodes) {
-            if (!isLocalSystemNode(node))
-                continue;
-            if (node.status === "done")
-                continue;
-            node.status = "done";
-            node.startedAt ??= state.startedAt;
-            node.completedAt ??= state.startedAt;
-            node.durationMs ??= 0;
-            changed = true;
-        }
-        if (changed) {
-            refreshState(state, dag, options);
-        }
-    }
-    function refreshState(state, dag, options) {
-        state.nodes = dag.nodes.map((n) => ({
-            ...n,
-            dependsOn: [...n.dependsOn],
-            outputs: n.outputs ? n.outputs.map((o) => ({ ...o })) : undefined,
-            routing: n.routing ? { ...n.routing } : undefined,
-            failurePolicy: n.failurePolicy ? { ...n.failurePolicy } : undefined,
-            attempts: n.attempts?.map((attempt) => ({ ...attempt })),
-        }));
-        state.estimate = estimateRunProgress({
-            nodes: dag.nodes,
-            startedAt: state.startedAt,
-            workerCount: options.workers,
-        });
-        state.updatedAt = new Date().toISOString();
-    }
-    function bumpActivity(state) {
-        const now = new Date().toISOString();
-        state.lastActivityAt = now;
-        state.lastHeartbeatAt = now;
-        state.activitySeq = (state.activitySeq ?? 0) + 1;
-    }
-    function cloneState(state) {
-        return {
-            ...state,
-            nodes: state.nodes.map((n) => ({
-                ...n,
-                dependsOn: [...n.dependsOn],
-                outputs: n.outputs ? n.outputs.map((o) => ({ ...o })) : undefined,
-                routing: n.routing ? { ...n.routing } : undefined,
-                failurePolicy: n.failurePolicy ? { ...n.failurePolicy } : undefined,
-                attempts: n.attempts?.map((attempt) => ({ ...attempt })),
-            })),
-            estimate: state.estimate ? { ...state.estimate } : undefined,
-        };
-    }
-    let latestSnapshot;
-    async function commitState(state, opts) {
-        latestSnapshot = cloneState(state);
-        // Coalesce: skip intermediate snapshots when queue is full,
-        // but always persist final/must-persist snapshots.
-        if (!opts?.mustPersist && commitQueueSize >= MAX_COMMIT_QUEUE) {
-            return;
-        }
-        commitQueueSize++;
-        const snap = latestSnapshot;
-        commitQueue = commitQueue
-            .then(async () => {
-            let timeoutHandle;
-            try {
-                await Promise.race([
-                    persister.save(snap),
-                    new Promise((_, reject) => {
-                        timeoutHandle = setTimeout(() => reject(new Error(`state persist timed out after ${COMMIT_SAVE_TIMEOUT_MS}ms`)), COMMIT_SAVE_TIMEOUT_MS);
-                    }),
-                ]);
-            }
-            finally {
-                if (timeoutHandle)
-                    clearTimeout(timeoutHandle);
-                commitQueueSize = Math.max(0, commitQueueSize - 1);
-            }
-        })
-            .catch((err) => {
-            const message = err instanceof Error ? err.message : String(err);
-            process.stderr.write(`[omk] state persist warning: ${message}
-`);
-        });
-        // Only await when mustPersist (caller needs durability guarantee)
-        if (opts?.mustPersist)
-            await commitQueue;
-    }
-    function emit(state) {
-        for (const h of stateChangeHandlers) {
-            try {
-                h(state);
-            }
-            catch {
-                // ignore handler errors
-            }
-        }
-    }
-    function markNodeStarted(node) {
-        const startedAt = new Date().toISOString();
-        const attemptNumber = node.retries + 1;
-        node.startedAt = startedAt;
-        node.completedAt = undefined;
-        node.durationMs = undefined;
-        const attempts = node.attempts ?? [];
-        attempts.push({ attempt: attemptNumber, startedAt });
-        node.attempts = attempts;
-    }
-    function markNodeFinished(node, status) {
-        const completedAt = new Date().toISOString();
-        const startedAtMs = node.startedAt ? Date.parse(node.startedAt) : Date.parse(completedAt);
-        const completedAtMs = Date.parse(completedAt);
-        const durationMs = Math.max(0, completedAtMs - startedAtMs);
-        const latestAttempt = node.attempts?.[node.attempts.length - 1];
-        if (latestAttempt) {
-            latestAttempt.completedAt = completedAt;
-            latestAttempt.durationMs = durationMs;
-            latestAttempt.status = status;
-        }
-        if (status === "done") {
-            node.completedAt = completedAt;
-            node.durationMs = durationMs;
-        }
-    }
-    function recordProviderAttempt(node, result) {
-        const latestAttempt = node.attempts?.[node.attempts.length - 1];
-        if (!latestAttempt)
-            return;
-        const metadata = result.metadata ?? {};
-        const provider = metadata.provider;
-        const requestedProvider = metadata.requestedProvider;
-        if (isProviderId(provider))
-            latestAttempt.provider = provider;
-        if (isProviderId(requestedProvider))
-            latestAttempt.requestedProvider = requestedProvider;
-        if (typeof metadata.providerModel === "string")
-            latestAttempt.providerModel = metadata.providerModel;
-        if (isDeepSeekModelTier(metadata.providerModelTier))
-            latestAttempt.providerModelTier = metadata.providerModelTier;
-        if (isDeepSeekParticipation(metadata.providerParticipation)) {
-            latestAttempt.providerParticipation = metadata.providerParticipation;
-        }
-        const fallback = metadata.providerFallback;
-        if (isProviderFallback(fallback)) {
-            latestAttempt.fallbackFrom = fallback.from;
-            latestAttempt.fallbackReason = fallback.reason;
-        }
-        // Capture error from failed attempts for debugging
-        if (!result.success) {
-            const errorMsg = (result.stderr?.trim() || result.stdout?.trim() || "").slice(0, 1000);
-            if (errorMsg) {
-                latestAttempt.error = errorMsg;
-            }
-            else if (typeof metadata.error === "string") {
-                latestAttempt.error = metadata.error.slice(0, 1000);
-            }
-        }
-    }
-    function etaEnv(estimate) {
-        if (!estimate)
-            return {};
-        return {
-            OMK_ETA_REMAINING_MS: String(estimate.estimatedRemainingMs ?? 0),
-            OMK_ETA_COMPLETED_AT: estimate.estimatedCompletedAt ?? "",
-            OMK_ETA_CONFIDENCE: estimate.confidence,
-            OMK_PROGRESS_PERCENT: String(estimate.percentComplete),
-            OMK_PROGRESS_NODES: `${estimate.completedNodes}/${estimate.totalNodes}`,
-        };
-    }
-    function emitNodeStart(node) {
-        for (const h of nodeStartHandlers) {
-            try {
-                h(node);
-            }
-            catch { /* ignore */ }
-        }
-    }
-    function emitNodeComplete(node, result) {
-        for (const h of nodeCompleteHandlers) {
-            try {
-                h(node, result);
-            }
-            catch { /* ignore */ }
-        }
-    }
-    async function checkNodeEvidence(node, result, options) {
-        const cwd = options.worktreeRoot ?? process.cwd();
-        const latestAttempt = node.attempts?.[node.attempts.length - 1];
-        const attemptId = latestAttempt ? `${node.id}__${latestAttempt.attempt}` : `${node.id}__1`;
-        const required = nodeRequiresEvidence(node);
-        const bridge = await legacyEvidenceBridge(node, result, { cwd, runId: options.runId, attemptId, required });
-        const metadata = {
-            ...(result.metadata ?? {}),
-            ...(bridge.evidenceGates.length > 0 && { evidenceGates: bridge.evidenceGates }),
-            ...(bridge.diffObserved && { diff: true }),
-        };
-        const check = checkEvidenceGate(required, node.outputs, metadata, result.stdout, bridge.artifactPaths);
-        return {
-            passed: check.satisfied,
-            evidence: evidenceGateCheckToDagEvidence(check, bridge.evidence),
-        };
-    }
-    async function legacyEvidenceBridge(node, result, options) {
-        const legacyGates = [];
-        for (const output of node.outputs ?? []) {
-            switch (output.gate) {
-                case "file-exists":
-                case "artifact":
-                    if (output.ref)
-                        legacyGates.push({ type: "file-exists", path: output.ref });
-                    break;
-                case "test-pass":
-                    legacyGates.push({ type: "command-pass", command: output.ref ?? "npm test" });
-                    break;
-                case "command-pass":
-                    legacyGates.push({ type: "command-pass", command: output.ref ?? "" });
-                    break;
-                case "diff":
-                    legacyGates.push({ type: "diff-nonempty" });
-                    break;
-                case "review-pass":
-                case "summary":
-                    legacyGates.push({ type: "summary-present", summaryMarker: output.ref ?? "## Summary" });
-                    break;
-                case "none":
-                default:
-                    break;
-            }
-        }
-        if (options.required && node.routing?.evidenceRequired && !legacyGates.some((gate) => gate.type === "command-pass")) {
-            legacyGates.push({ type: "summary-present", summaryMarker: "## Evidence" });
-        }
-        if (legacyGates.length === 0) {
-            return { evidenceGates: [], artifactPaths: [], diffObserved: false, evidence: [] };
-        }
-        const legacy = await checkEvidenceGates(legacyGates, {
-            cwd: options.cwd,
-            stdout: result.stdout,
-            nodeId: node.id,
-            runId: options.runId,
-            attemptId: options.attemptId,
-        });
-        const evidenceGates = [];
-        const artifactPaths = [];
-        let diffObserved = false;
-        for (const item of legacy.evidence) {
-            if (!item.passed)
-                continue;
-            if (item.gate === "command-pass")
-                evidenceGates.push("command-pass");
-            else if (item.gate === "summary-present")
-                evidenceGates.push("summary");
-            else if (item.gate === "file-exists") {
-                evidenceGates.push("artifact");
-                if (item.ref)
-                    artifactPaths.push(item.ref);
-            }
-            else if (item.gate === "diff-nonempty") {
-                evidenceGates.push("diff");
-                diffObserved = true;
-            }
-        }
-        return { evidenceGates, artifactPaths, diffObserved, evidence: legacy.evidence };
-    }
-    function evidenceGateCheckToDagEvidence(check, bridgeEvidence) {
-        const evidence = [...bridgeEvidence];
-        for (const observation of check.observations ?? []) {
-            evidence.push({
-                gate: observation.kind,
-                passed: check.satisfied,
-                ref: observation.ref ?? observation.artifactPath,
-                message: `Evidence observation ${observation.kind} from ${observation.source} confidence=${observation.confidence}`,
-            });
-        }
-        for (const missing of check.missing) {
-            evidence.push({
-                gate: missing,
-                passed: false,
-                failureKind: "missing-evidence-observation",
-                message: check.reason,
-            });
-        }
-        return evidence;
-    }
-    function hasRequiredEvidenceOutput(node) {
-        return (node.outputs ?? []).some((output) => output.required !== false && output.gate !== undefined && output.gate !== "none");
-    }
-    function nodeRequiresEvidence(node) {
-        const highRisk = nodeHasHighRiskCapabilities(node);
-        const outputs = node.outputs ?? [];
-        const explicitlyNoEvidence = outputs.length > 0 && outputs.every((output) => output.required === false || output.gate === undefined || output.gate === "none");
-        if (explicitlyNoEvidence && !highRisk)
-            return false;
-        return hasRequiredEvidenceOutput(node) || highRisk;
-    }
-    function nodeHasHighRiskCapabilities(node) {
-        return node.routing?.assignedProviderCapabilities?.some((cap) => ["write", "patch", "shell", "merge"].includes(cap)) ?? false;
-    }
-    async function runNode(node, dag, runner, options, state, signal, outAbort) {
-        if (nodeRequiresEvidence(node)) {
-            const hasGate = (node.outputs ?? []).some((output) => ["file-exists", "test-pass", "review-pass", "command-pass", "summary", "artifact", "diff"].includes(output.gate ?? ""));
-            if (!hasGate) {
-                scheduler.updateNodeStatus(dag, node.id, "failed", options.runId);
-                markNodeStarted(node);
-                const completedAt = new Date().toISOString();
-                const startedAtMs = Date.parse(node.startedAt ?? completedAt);
-                const durationMs = Math.max(0, Date.parse(completedAt) - startedAtMs);
-                const latestAttempt = { attempt: node.retries + 1, startedAt: node.startedAt ?? completedAt, completedAt, durationMs, status: "failed", error: "[omk] Evidence gate required but missing: high-risk node has no command-pass, test-pass, review-pass, file-exists, or summary output gate" };
-                node.attempts = [...(node.attempts ?? []), latestAttempt];
-                node.evidence = [
-                    ...(node.evidence ?? []),
-                    { gate: "evidence-required", passed: false, message: latestAttempt.error, failureKind: "missing-evidence-gate" },
-                ];
-                markNodeFinished(node, "failed");
-                refreshState(state, dag, options);
-                await commitState(state);
-                emit(cloneState(state));
-                emitTelemetry({ type: "lane.failed", runId: options.runId, nodeId: node.id, laneId: node.id, status: "failed", data: { success: false, exitCode: 78 } });
-                return;
-            }
-        }
-        scheduler.updateNodeStatus(dag, node.id, "running", options.runId);
-        markNodeStarted(node);
-        bumpActivity(state);
-        refreshState(state, dag, options);
-        await commitState(state);
-        emit(cloneState(state));
-        emitTelemetry({ type: "lane.started", runId: options.runId, nodeId: node.id, laneId: node.id, data: { role: node.role, name: node.name } });
-        emitNodeStart(node);
-        const resources = await getOmkResourceSettings();
-        const env = {
-            OMK_NODE_ID: node.id,
-            OMK_RUN_ID: options.runId,
-            OMK_ROLE: node.role,
-            OMK_NODE_ROLE: node.role,
-            OMK_PROVIDER_MODEL: node.routing?.providerModel ?? "",
-            OMK_MCP_ENABLED: resources.mcpScope === "none" ? "false" : "true",
-            OMK_SKILLS_ENABLED: resources.skillsScope === "none" ? "false" : "true",
-            OMK_HOOKS_ENABLED: resources.hooksScope === "none" ? "false" : "true",
-            ...dagNodeRoutingEnv(node, dag),
-            ...etaEnv(state.estimate),
-        };
-        const runContext = buildTaskRunContext({
-            runId: options.runId,
-            root: options.worktreeRoot ?? process.cwd(),
-            goalId: state.goalId,
-            objective: state.goalSnapshot?.objective ?? state.goal?.objective ?? node.name,
-            node,
-            toolPlane: {
-                mcpServers: node.routing?.mcpServers ?? node.routing?.assignedCapabilities?.mcpServers ?? [],
-                skills: node.routing?.skills ?? node.routing?.assignedCapabilities?.skills ?? [],
-                hooks: node.routing?.hooks ?? node.routing?.assignedCapabilities?.hooks ?? [],
-                tools: node.routing?.tools ?? node.routing?.assignedCapabilities?.tools ?? [],
-                requiresRuntimeMcp: node.routing?.requiresMcp === true,
-            },
-            selectedRuntimeId: node.routing?.providerModel,
-            model: node.routing?.providerModel,
-        });
-        Object.assign(env, envFromWorkerManifest(runContext.worker));
-        let result;
-        let ignoredAbortEvidence;
-        node.thinking = undefined;
-        // Use fork() if available for parallel-safe isolated thinking callbacks.
-        const nodeRunner = runner.fork
-            ? runner.fork((thinking) => {
-                node.thinking = thinking;
-            })
-            : {
-                ...runner,
-                onThinking: (thinking) => {
-                    node.thinking = thinking;
-                },
-                run: runner.run.bind(runner),
-            };
-        // Sync thinking back to state.nodes periodically so the live UI
-        // can show ensemble / runner progress while the node is running.
-        let lastEmittedThinking;
-        const progressTimer = setInterval(() => {
-            if (node.status !== "running")
-                return;
-            const stateNode = state.nodes.find((sn) => sn.id === node.id);
-            if (stateNode)
-                stateNode.thinking = node.thinking;
-            if (node.thinking && node.thinking !== lastEmittedThinking) {
-                lastEmittedThinking = node.thinking;
-                bumpActivity(state);
-                emitTelemetry({
-                    type: "lane.activity",
-                    runId: options.runId,
-                    nodeId: node.id,
-                    laneId: node.id,
-                    data: { phase: node.thinking },
-                });
-                emit(cloneState(state));
-            }
-        }, 500);
-        // Persist live activity to disk less frequently to avoid I/O thrash.
-        const persistTimer = setInterval(() => {
-            void commitState(state);
-        }, 2000);
-        monitor.register(node.id, options.runId);
-        const heartbeatTimer = setInterval(() => {
-            if (node.status === "running") {
-                monitor.heartbeat(node.id, options.runId);
-                state.lastHeartbeatAt = new Date().toISOString();
-                emitTelemetry({ type: "lane.heartbeat", runId: options.runId, nodeId: node.id, laneId: node.id });
-            }
-        }, 30_000);
-        activeTimers.set(node.id, { progress: progressTimer, persist: persistTimer, heartbeat: heartbeatTimer });
-        try {
-            const timeoutPreset = node.timeoutPreset ?? options.timeoutPreset;
-            const nodeTimeoutMs = await resolveTimeoutMs({
-                timeoutMs: node.timeoutMs ?? (timeoutPreset ? undefined : options.nodeTimeoutMs),
-                timeoutPreset,
-            });
-            const HARD_MAX_TIMEOUT_MS = 1_800_000; // 30 minutes
-            const nodeAbortController = new AbortController();
-            let abortReason;
-            const abortNode = (reason) => {
-                if (reason)
-                    abortReason = reason instanceof Error ? reason.message : reason;
-                if (!nodeAbortController.signal.aborted) {
-                    nodeAbortController.abort(reason);
-                }
-            };
-            nodeAbortRegistry.set(node.id, abortNode);
-            if (outAbort) {
-                outAbort.abort = abortNode;
-            }
-            let runPromiseSettled = false;
-            const runPromise = nodeRunner.run(node, env, nodeAbortController.signal, runContext).finally(() => {
-                runPromiseSettled = true;
-            });
-            let hardMaxHandle;
-            const hardMaxPromise = new Promise((_, reject) => {
-                hardMaxHandle = setTimeout(() => {
-                    const error = new Error(`Node ${node.id} exceeded hard maximum timeout`);
-                    abortNode(error);
-                    reject(error);
-                }, HARD_MAX_TIMEOUT_MS);
-            });
-            let timeoutHandle;
-            let abortHandler;
-            const racePromises = [runPromise, hardMaxPromise];
-            if (nodeTimeoutMs > 0) {
-                const timeoutPromise = new Promise((_, reject) => {
-                    timeoutHandle = setTimeout(() => {
-                        const error = new Error(`Node ${node.id} timed out after ${nodeTimeoutMs}ms`);
-                        abortNode(error);
-                        reject(error);
-                    }, nodeTimeoutMs);
-                });
-                racePromises.push(timeoutPromise);
-            }
-            if (signal) {
-                const abortPromise = new Promise((_, reject) => {
-                    abortHandler = () => {
-                        const reason = signal.reason instanceof Error ? signal.reason : new Error(`Node ${node.id} aborted`);
-                        abortNode(reason);
-                        reject(reason);
-                    };
-                    signal.addEventListener('abort', abortHandler, { once: true });
-                });
-                racePromises.push(abortPromise);
-            }
-            try {
-                result = await Promise.race(racePromises);
-            }
-            finally {
-                clearTimeout(hardMaxHandle);
-                clearTimeout(timeoutHandle);
-                if (signal && abortHandler)
-                    signal.removeEventListener("abort", abortHandler);
-                // If the race rejected (timeout/abort), runPromise may still be alive
-                // with a dangling child process. Wait for it to settle so we don't
-                // leak subprocesses or keep the event loop alive indefinitely.
-                const drained = await Promise.race([
-                    runPromise.then(() => true, () => true),
-                    new Promise((resolve) => {
-                        const drainTimer = setTimeout(() => resolve(false), RUNNER_ABORT_DRAIN_MS);
-                        drainTimer.unref?.();
-                    }),
-                ]);
-                if (!drained && abortReason && !runPromiseSettled) {
-                    ignoredAbortEvidence = {
-                        gate: "runner-abort",
-                        passed: false,
-                        failureKind: "runner-abort-ignored",
-                        message: `Runner did not settle within ${RUNNER_ABORT_DRAIN_MS}ms after abort: ${abortReason}`,
-                    };
-                    process.stderr.write(`[omk] run ${options.runId} node ${node.id} runner did not settle within ${RUNNER_ABORT_DRAIN_MS}ms after abort\n`);
-                }
-            }
-            recordProviderAttempt(node, result);
-            if (aborting && (node.status === "blocked" || node.status === "skipped")) {
-                return;
-            }
-            if (result.success) {
-                const evidenceCheck = await checkNodeEvidence(node, result, options);
-                node.evidence = evidenceCheck.evidence;
-                if (evidenceCheck.passed) {
-                    markNodeFinished(node, "done");
-                    scheduler.updateNodeStatus(dag, node.id, "done", options.runId);
-                }
-                else {
-                    markNodeFinished(node, "failed");
-                    scheduler.updateNodeStatus(dag, node.id, "failed", options.runId);
-                }
-            }
-            else {
-                markNodeFinished(node, "failed");
-                scheduler.updateNodeStatus(dag, node.id, "failed", options.runId);
-            }
-        }
-        catch (error) {
-            const errorMessage = error instanceof Error ? error.message : String(error);
-            result = {
-                success: false,
-                exitCode: 1,
-                stdout: `[ERROR] ${errorMessage}`,
-                stderr: errorMessage,
-            };
-            if (ignoredAbortEvidence) {
-                node.evidence = [...(node.evidence ?? []), ignoredAbortEvidence];
-            }
-            if (aborting && (node.status === "blocked" || node.status === "skipped")) {
-                return;
-            }
-            recordProviderAttempt(node, result);
-            markNodeFinished(node, "failed");
-            scheduler.updateNodeStatus(dag, node.id, "failed", options.runId);
-        }
-        finally {
-            clearInterval(progressTimer);
-            clearInterval(persistTimer);
-            clearInterval(heartbeatTimer);
-            monitor.unregister(node.id, options.runId);
-            activeTimers.delete(node.id);
-            nodeAbortRegistry.delete(node.id);
-            const stateNode = state.nodes.find((sn) => sn.id === node.id);
-            if (stateNode)
-                stateNode.thinking = node.thinking;
-            bumpActivity(state);
-        }
-        bumpTerminalActivity();
-        if (aborting) {
-            // Skip final emit/persist so the abort state is not overwritten
-            // by a late-finishing node after execute() has already returned.
-            return;
-        }
-        emitNodeComplete(node, result);
-        emitTelemetry({
-            type: result.success ? "lane.completed" : "lane.failed",
-            runId: options.runId,
-            nodeId: node.id,
-            laneId: node.id,
-            status: node.status,
-            data: { success: result.success, exitCode: result.exitCode },
-        });
-        for (const evidence of node.evidence ?? []) {
-            emitTelemetry({
-                type: "evidence.result",
-                runId: options.runId,
-                nodeId: node.id,
-                laneId: node.id,
-                status: evidence.passed ? "passed" : "failed",
-                data: { ...evidence },
-            });
-        }
-        refreshState(state, dag, options);
-        emit(cloneState(state));
-        await commitState(state);
-    }
-    return {
-        onStateChange(handler) {
-            stateChangeHandlers.push(handler);
-            return () => {
-                const idx = stateChangeHandlers.indexOf(handler);
-                if (idx !== -1)
-                    stateChangeHandlers.splice(idx, 1);
-            };
-        },
-        onNodeStart(handler) {
-            nodeStartHandlers.push(handler);
-            return () => {
-                const idx = nodeStartHandlers.indexOf(handler);
-                if (idx !== -1)
-                    nodeStartHandlers.splice(idx, 1);
-            };
-        },
-        onNodeComplete(handler) {
-            nodeCompleteHandlers.push(handler);
-            return () => {
-                const idx = nodeCompleteHandlers.indexOf(handler);
-                if (idx !== -1)
-                    nodeCompleteHandlers.splice(idx, 1);
-            };
-        },
-        async execute(dag, runner, options) {
-            if (!Number.isInteger(options.workers) || options.workers < 1) {
-                throw new TypeError(`options.workers must be a positive integer, got ${options.workers}`);
-            }
-            aborting = false;
-            isShuttingDown = false;
-            activeTick = undefined;
-            const effectiveRunner = executorOptions.ensemble === false
-                ? runner
-                : createEnsembleTaskRunner(runner, executorOptions.ensemble ?? {});
-            activeDag = dag;
-            activeRunOptions = options;
-            lastTerminalChangeAt = Date.now();
-            let state;
-            if (executorOptions.resumeFromState) {
-                state = cloneState(executorOptions.resumeFromState);
-                const stateNodeById = new Map(state.nodes.map((n) => [n.id, n]));
-                for (const node of dag.nodes) {
-                    const saved = stateNodeById.get(node.id);
-                    if (saved) {
-                        // Do not restore non-terminal running state; reset to pending so
-                        // the scheduler can re-dispatch it. Only preserve terminal states.
-                        node.status = saved.status === "running" ? "pending" : saved.status;
-                        node.retries = saved.retries;
-                        node.startedAt = saved.startedAt;
-                        node.completedAt = saved.completedAt;
-                        node.durationMs = saved.durationMs;
-                        node.attempts = saved.attempts?.map((a) => ({ ...a }));
-                        node.evidence = saved.evidence?.map((e) => ({ ...e }));
-                        node.blockedReason = saved.blockedReason;
-                    }
-                }
-            }
-            else {
-                state = buildState(dag, options);
-            }
-            markLocalSystemNodesDone(dag, state, options);
-            await commitState(state);
-            const runningMap = new Map();
-            const runAbortController = new AbortController();
-            const runningCoreCount = () => [...runningMap.keys()]
-                .filter((nodeId) => {
-                const node = dag.nodes.find((entry) => entry.id === nodeId);
-                return node ? !isOptionalExecutionLane(dag, node) : true;
-            })
-                .length;
-            const hasOpenRequiredNodes = () => dag.nodes.some((node) => (!isOptionalExecutionLane(dag, node) && (node.status === "pending" || node.status === "running")));
-            const skipOpenOptionalLanes = (reason) => {
-                for (const node of dag.nodes) {
-                    if (!isOptionalExecutionLane(dag, node))
-                        continue;
-                    if (node.status !== "pending" && node.status !== "running")
-                        continue;
-                    node.status = "skipped";
-                    node.blockedReason = reason;
-                    try {
-                        runningMap.get(node.id)?.abort();
-                    }
-                    catch {
-                        // ignore optional-lane abort errors
-                    }
-                }
-            };
-            let resolveDone;
-            const donePromise = new Promise((resolve) => {
-                resolveDone = resolve;
-            });
-            let settled = false;
-            function resolveOnce(result) {
-                if (settled)
-                    return;
-                settled = true;
-                if (runTimeoutHandle) {
-                    clearInterval(runTimeoutHandle);
-                    runTimeoutHandle = undefined;
-                }
-                if (executorOptions.signal) {
-                    executorOptions.signal.removeEventListener("abort", externalAbortHandler);
-                }
-                void flushTelemetry().finally(() => resolveDone(result));
-            }
-            async function finalizeRunFailure(reason, logMessage) {
-                if (settled)
-                    return;
-                if (logMessage)
-                    process.stderr.write(logMessage);
-                aborting = true;
-                isShuttingDown = true;
-                runAbortController.abort();
-                for (const [, running] of runningMap) {
-                    try {
-                        running.abort();
-                    }
-                    catch {
-                        // ignore abort errors
-                    }
-                }
-                clearActiveTimers();
-                markOpenNodesBlocked(dag, reason);
-                bumpTerminalActivity();
-                state.completedAt = new Date().toISOString();
-                refreshState(state, dag, options);
-                await commitState(state, { mustPersist: true });
-                monitor.dispose();
-                resolveOnce({ state, success: false });
-            }
-            function externalAbortHandler() {
-                void finalizeRunFailure("cancelled");
-            }
-            if (executorOptions.signal) {
-                if (executorOptions.signal.aborted) {
-                    void finalizeRunFailure("cancelled");
-                }
-                else {
-                    executorOptions.signal.addEventListener("abort", externalAbortHandler, { once: true });
-                }
-            }
-            const fallbackNodes = new Map();
-            async function tick() {
-                try {
-                    activeTick = tick;
-                    if (settled)
-                        return;
-                    // Handle fallback nodes FIRST, before isFailed check, so that a
-                    // terminal failure with a fallbackRole does not immediately fail the run.
-                    let fallbackCreated = false;
-                    for (const node of dag.nodes) {
-                        if (node.status === "failed" && node.retries >= node.maxRetries && node.failurePolicy?.fallbackRole && !fallbackNodes.has(node.id)) {
-                            let fallbackId = `${node.id}--fallback`;
-                            let fallbackCounter = 1;
-                            while (dag.nodes.some((n) => n.id === fallbackId)) {
-                                fallbackId = `${node.id}--fallback-${fallbackCounter++}`;
-                            }
-                            const fallbackNode = {
-                                id: fallbackId,
-                                name: `${node.name} (fallback)`,
-                                role: node.failurePolicy.fallbackRole,
-                                dependsOn: [...node.dependsOn],
-                                status: "pending",
-                                retries: 0,
-                                maxRetries: 1,
-                                timeoutMs: node.timeoutMs,
-                                timeoutPreset: node.timeoutPreset,
-                                priority: node.priority,
-                                cost: node.cost,
-                                inputs: node.inputs?.map((input) => ({ ...input })),
-                                outputs: node.outputs?.map((output) => ({ ...output })),
-                                routing: node.routing ? { ...node.routing } : undefined,
-                                failurePolicy: { retryable: false, blockDependents: true },
-                            };
-                            dag.nodes.push(fallbackNode);
-                            fallbackNodes.set(node.id, fallbackNode);
-                            state.nodes.push({ ...fallbackNode });
-                            // Mark original node as skipped so dependents can proceed via fallback
-                            node.status = "skipped";
-                            node.blockedReason = `fallback created: ${fallbackId}`;
-                            invalidateTaskDagGraph(dag);
-                            // Redirect dependents from original node to fallback node
-                            for (const dep of dag.nodes) {
-                                if (dep.dependsOn.includes(node.id)) {
-                                    if (dep.status === "blocked" || dep.status === "skipped") {
-                                        dep.status = "pending";
-                                        dep.blockedReason = undefined;
-                                    }
-                                    dep.dependsOn = dep.dependsOn.filter((id) => id !== node.id);
-                                    if (!dep.dependsOn.includes(fallbackId)) {
-                                        dep.dependsOn.push(fallbackId);
-                                    }
-                                    rewriteNodeInputFrom(dep, node.id, fallbackId);
-                                }
-                            }
-                            fallbackCreated = true;
-                        }
-                    }
-                    if (fallbackCreated) {
-                        bumpTerminalActivity();
-                        refreshState(state, dag, options);
-                        await commitState(state);
-                        tick().catch(() => { });
-                        return;
-                    }
-                    if (executorOptions.signal?.aborted) {
-                        await finalizeRunFailure("cancelled");
-                        return;
-                    }
-                    if (runningMap.size === 0 && scheduler.isComplete(dag)) {
-                        state.completedAt = new Date().toISOString();
-                        refreshState(state, dag, options);
-                        await commitState(state, { mustPersist: true });
-                        monitor.dispose();
-                        resolveOnce({ state, success: true });
-                        return;
-                    }
-                    if (runningMap.size === 0 && scheduler.isFailed(dag)) {
-                        state.completedAt = new Date().toISOString();
-                        refreshState(state, dag, options);
-                        await commitState(state, { mustPersist: true });
-                        monitor.dispose();
-                        resolveOnce({ state, success: false });
-                        return;
-                    }
-                    if (scheduler.isFailed(dag))
-                        return;
-                    if (runningCoreCount() === 0 && isRequiredDagComplete(dag)) {
-                        aborting = true;
-                        isShuttingDown = true;
-                        skipOpenOptionalLanes("optional lane skipped after required DAG completed");
-                        clearActiveTimers();
-                        state.completedAt = new Date().toISOString();
-                        refreshState(state, dag, options);
-                        await commitState(state, { mustPersist: true });
-                        monitor.dispose();
-                        resolveOnce({ state, success: true });
-                        return;
-                    }
-                    const runnable = scheduler.getRunnableNodes(dag);
-                    const eligible = runnable
-                        .filter((node) => !runningMap.has(node.id) && !fallbackNodes.has(node.id));
-                    const coreRunnable = eligible.filter((node) => !isOptionalExecutionLane(dag, node));
-                    const optionalRunnable = eligible.filter((node) => isOptionalExecutionLane(dag, node));
-                    const availableCoreSlots = Math.max(0, options.workers - runningCoreCount());
-                    const coreToRun = coreRunnable.slice(0, availableCoreSlots);
-                    const optionalSlots = options.workers > 1
-                        ? Math.max(0, options.workers - runningMap.size - coreToRun.length)
-                        : (coreToRun.length === 0 && !hasOpenRequiredNodes() && runningMap.size === 0 ? 1 : 0);
-                    const toRun = [
-                        ...coreToRun,
-                        ...optionalRunnable.slice(0, optionalSlots),
-                    ];
-                    for (const node of toRun) {
-                        if (isShuttingDown)
-                            break;
-                        const nodeAbortRef = { abort: () => { } };
-                        const runnerForNode = isOptionalExecutionLane(dag, node) ? runner : effectiveRunner;
-                        const promise = runNode(node, dag, runnerForNode, options, state, runAbortController.signal, nodeAbortRef)
-                            .catch(() => {
-                            // runNode already marks node as failed on runner errors;
-                            // swallow persist/emit errors to allow tick() to continue
-                        })
-                            .finally(() => {
-                            runningMap.delete(node.id);
-                            tick().catch(() => { });
-                        });
-                        runningMap.set(node.id, { promise, abort: () => nodeAbortRef.abort() });
-                    }
-                    if (runningMap.size === 0 && toRun.length === 0 && runnable.length === 0) {
-                        // Deadlock or nothing to do — treat as failure
-                        state.completedAt = new Date().toISOString();
-                        refreshState(state, dag, options);
-                        await commitState(state, { mustPersist: true });
-                        monitor.dispose();
-                        resolveOnce({ state, success: false });
-                    }
-                }
-                catch (error) {
-                    if (settled)
-                        return;
-                    const message = error instanceof Error ? error.message : String(error);
-                    process.stderr.write(`[omk] executor tick crashed: ${message}\n`);
-                    aborting = true;
-                    isShuttingDown = true;
-                    runAbortController.abort();
-                    clearActiveTimers();
-                    markOpenNodesBlocked(dag, "executor crash");
-                    state.completedAt = new Date().toISOString();
-                    try {
-                        refreshState(state, dag, options);
-                        await commitState(state, { mustPersist: true });
-                    }
-                    catch {
-                        // ignore
-                    }
-                    monitor.dispose();
-                    resolveOnce({ state, success: false });
-                }
-            }
-            let runTimeoutHandle;
-            if (options.runTimeoutMs && options.runTimeoutMs > 0) {
-                runTimeoutHandle = setInterval(() => {
-                    if (settled)
-                        return;
-                    if (Date.now() - lastTerminalChangeAt > options.runTimeoutMs) {
-                        void finalizeRunFailure("run timeout", `[omk] run ${options.runId} timed out after ${options.runTimeoutMs}ms with no terminal state change\n`);
-                    }
-                }, Math.min(options.runTimeoutMs, 30_000));
-            }
-            tick().catch(() => { });
-            return donePromise;
-        },
-    };
-}
-function isRequiredDagComplete(dag) {
-    const requiredNodes = dag.nodes.filter((node) => !isOptionalExecutionLane(dag, node));
-    return requiredNodes.length > 0 && requiredNodes.every(isTerminalForRequiredCompletion);
-}
-function isTerminalForRequiredCompletion(node) {
-    return node.status === "done" ||
-        node.status === "skipped" ||
-        (node.status === "failed" && Boolean(node.failurePolicy?.fallbackRole));
-}
-function isOptionalExecutionLane(dag, node) {
-    if (hasRequiredDependent(dag, node))
-        return false;
-    const outputs = node.outputs ?? [];
-    if (outputs.length > 0 && outputs.every((output) => output.required === false))
-        return true;
-    if (hasOptionalDependent(dag, node))
-        return true;
-    return node.routing?.autoSpawned === true || node.routing?.assignedProviderAuthority === "advisory";
-}
-function hasRequiredDependent(dag, node) {
-    return dag.nodes.some((dependent) => (dependent.dependsOn.includes(node.id) && isRequiredReadinessDependency(dependent, node)));
-}
-function hasOptionalDependent(dag, node) {
-    return dag.nodes.some((dependent) => (dependent.dependsOn.includes(node.id) && !isRequiredReadinessDependency(dependent, node)));
-}
-function isRequiredReadinessDependency(dependent, predecessor) {
-    const inputsFromPredecessor = dependent.inputs?.filter((input) => input.from === predecessor.id) ?? [];
-    if (inputsFromPredecessor.length > 0) {
-        return inputsFromPredecessor.some((input) => input.required !== false);
-    }
-    const outputs = predecessor.outputs ?? [];
-    if (outputs.length > 0 && outputs.every((output) => output.required === false)) {
-        return false;
-    }
-    return dependent.dependsOn.includes(predecessor.id);
-}
-function isProviderId(value) {
-    return typeof value === "string" && value.trim().length > 0;
-}
-function isDeepSeekModelTier(value) {
-    return value === "flash" || value === "pro";
-}
-function isDeepSeekParticipation(value) {
-    return value === "direct" || value === "advisory";
-}
-function isProviderFallback(value) {
-    if (typeof value !== "object" || value === null)
-        return false;
-    const record = value;
-    return isProviderId(record.from) && typeof record.reason === "string";
-}