@tt-a1i/hive 2.1.0 → 2.1.3

package/dist/src/server/workflow-runner.js

@@ -1,177 +1,17 @@
-import { randomUUID } from 'node:crypto';
-import { cpus } from 'node:os';
 import { dirname, join } from 'node:path';
-import { Worker } from 'node:worker_threads';
 import { assertWindowsSafeFilename } from './windows-filename.js';
-import { resolveWorkflowCli } from './workflow-cli-policy.js';
-import { buildSchemaInstruction, extractJsonBlock } from './workflow-output-schema.js';
+import { createWorkflowAgentCallExecutor, } from './workflow-agent-call-executor.js';
+import { createWorkflowDagLayerTracker } from './workflow-dag-layer-tracker.js';
 import { loadWorkflowScriptFile, loadWorkflowScriptSource } from './workflow-script-loader.js';
+import { runWorkflowScriptWorker } from './workflow-script-worker.js';
+import { WORKFLOW_VM_WORKER_URL } from './workflow-vm-worker-source.js';
 import { getWorkflowAgentId } from './workspace-store-support.js';
-// TIER 2 #2 + #11 — runtime caps. Match Claude Code's documented caps
-// (1000 lifetime calls, min(16, cores-2) concurrent) so scripts copied
-// from CC docs behave the same way. The duration default is Hive's own
-// choice; CC docs don't specify one but workflows that run >60min are
-// almost always a runaway loop in practice.
-const DEFAULT_MAX_AGENTS_PER_RUN = 1000;
+// TIER 2 #11 — workflow wall-clock budget. Agent call count and concurrency
+// caps live in workflow-agent-call-executor; the duration default is Hive's
+// own choice because Claude Code docs don't specify one.
 const DEFAULT_MAX_DURATION_MS = 60 * 60 * 1000;
-const DEFAULT_MAX_CONCURRENT_AGENTS = Math.min(16, Math.max(2, cpus().length - 2));
-const WORKFLOW_VM_WORKER_SOURCE = `
-const { parentPort, workerData } = require('node:worker_threads');
-const { Script, createContext } = require('node:vm');
-
-const BRIDGE_FACTORY = new Script(\`"use strict";
-((hostCall) => {
-  const stringify = JSON.stringify;
-  const parse = JSON.parse;
-  const promiseResolve = Promise.resolve.bind(Promise);
-  const promiseThen = Promise.prototype.then;
-  return (...args) =>
-    promiseThen.call(promiseResolve(hostCall(stringify(args))), (payloadJson) => {
-      const payload = parse(payloadJson);
-      if (!payload.ok) throw new Error(payload.error || 'Hive workflow host call failed');
-      return payload.hasValue ? payload.value : undefined;
-    });
-})\`);
-const FLOW_FACTORY = new Script(\`"use strict";
-((catchPerItem) => {
-  const promiseAll = Promise.all.bind(Promise);
-  const promiseResolve = Promise.resolve.bind(Promise);
-  const promiseThen = Promise.prototype.then;
-  const promiseCatch = Promise.prototype.catch;
-  const parallel = (thunks) =>
-    promiseAll(Array.from(thunks).map((thunk) =>
-      promiseCatch.call(promiseThen.call(promiseResolve(), () => thunk()), (err) => catchPerItem(err))
-    ));
-  const pipeline = (items, ...stages) =>
-    promiseAll(Array.from(items).map((item, index) => {
-      let chain = promiseResolve(item);
-      for (const stage of stages) {
-        chain = promiseThen.call(chain, (prev) => stage(prev, item, index));
-      }
-      return promiseCatch.call(chain, (err) => catchPerItem(err));
-    }));
-  return { parallel, pipeline };
-})\`);
-const JSON_PARSE = new Script('JSON.parse');
-
-const encodeSuccess = (value) => {
-  const payload = { ok: true, hasValue: value !== undefined };
-  if (value !== undefined) payload.value = value;
-  return JSON.stringify(payload);
-};
-const encodeFailure = (error) =>
-  JSON.stringify({ ok: false, error: error instanceof Error ? error.message : String(error) });
-
-const createSafeHostCall = (fn) =>
-  new Proxy(
-    (serializedArgs) => {
-      let args;
-      try {
-        const parsed = JSON.parse(serializedArgs);
-        args = Array.isArray(parsed) ? parsed : [];
-      } catch (error) {
-        return encodeFailure(error);
-      }
-      try {
-        return Promise.resolve(fn(args)).then(encodeSuccess, encodeFailure);
-      } catch (error) {
-        return encodeFailure(error);
-      }
-    },
-    {
-      get: () => undefined,
-      getOwnPropertyDescriptor: () => undefined,
-      getPrototypeOf: () => null,
-      has: () => false,
-      ownKeys: () => [],
-      set: () => false,
-    }
-  );
-
-let nextCallId = 0;
-const pending = new Map();
-
-parentPort.on('message', (message) => {
-  if (!message || message.type !== 'hostResponse') return;
-  const entry = pending.get(message.id);
-  if (!entry) return;
-  pending.delete(message.id);
-  if (message.ok) entry.resolve(message.value);
-  else entry.reject(new Error(message.error || 'Hive workflow host call failed'));
-});
-
-const callHost = (name, args) =>
-  new Promise((resolve, reject) => {
-    const id = String(++nextCallId);
-    pending.set(id, { resolve, reject });
-    parentPort.postMessage({ type: 'hostCall', id, name, args });
-  });
-
-const cloneIntoVm = (context, value) => {
-  if (value === undefined) return undefined;
-  const serialized = JSON.stringify(value);
-  if (serialized === undefined) return undefined;
-  return JSON_PARSE.runInContext(context)(serialized);
-};
-
-const cloneOutOfVm = (value) => {
-  if (value === undefined) return undefined;
-  const serialized = JSON.stringify(value);
-  return serialized === undefined ? undefined : JSON.parse(serialized);
-};
-
-(async () => {
-  try {
-    const context = createContext(Object.create(null), {
-      codeGeneration: { strings: false, wasm: false },
-    });
-    const bridgeFactory = BRIDGE_FACTORY.runInContext(context, { timeout: 1000 });
-    const bridge = (name) =>
-      bridgeFactory(createSafeHostCall((args) => callHost(name, args)));
-    const vmAgent = bridge('agent');
-    const vmPhase = bridge('phase');
-    const vmLog = bridge('log');
-    const vmWorkflow = bridge('workflow');
-    const vmCatchPerItem = bridge('catchPerItem');
-    const { parallel, pipeline } = FLOW_FACTORY.runInContext(context, { timeout: 1000 })(
-      vmCatchPerItem
-    );
-    const fn = new Script(\`"use strict";\\n\${workerData.compiledFunctionSource}\\n; __wf\`, {
-      filename: workerData.scriptPath,
-    }).runInContext(context, { timeout: 1000 });
-    const value = cloneOutOfVm(
-      await fn(
-        vmAgent,
-        parallel,
-        pipeline,
-        vmPhase,
-        vmLog,
-        vmWorkflow,
-        cloneIntoVm(context, workerData.args)
-      )
-    );
-    parentPort.postMessage({ type: 'done', ok: true, value });
-  } catch (error) {
-    parentPort.postMessage({
-      type: 'done',
-      ok: false,
-      error: error instanceof Error ? error.message : String(error),
-    });
-  }
-})();
-`;
+const ACTIVE_AGENT_CALL_CLEANUP_TIMEOUT_MS = 5000;
 const errorToMessage = (error) => error instanceof Error ? error.message : String(error);
-const BUILT_IN_WORKER_ROLES = new Set(['coder', 'reviewer', 'tester', 'custom']);
-const isBuiltInWorkerRole = (value) => BUILT_IN_WORKER_ROLES.has(value);
-const buildModelArgs = (_cli, model) => {
-    if (!model?.trim())
-        return [];
-    /* All supported CLIs accept
-       `--model <id>` as a positional flag. Keeping the mapping centralised
-       here so future CLI quirks (e.g. opencode wanting `-m` instead) can
-       be patched without touching the runner body. */
-    return ['--model', model];
-};
 const toNestedWorkflowFilename = (scriptName) => {
     const trimmed = scriptName.trim();
     if (!trimmed)
@@ -183,6 +23,7 @@ const toNestedWorkflowFilename = (scriptName) => {
 export const createWorkflowRunner = (deps) => {
     const { store, workflowRunStore, awaiter, dispatchPort, resolveWorkspacePath, roleTemplateResolver, logStore, resolveCliLaunchConfig, getWorkflowCliPolicy, } = deps;
     const stoppedRuns = new Set();
+    const abortedRuns = new Map();
     const activeScriptWorkers = new Map();
     // In-memory map: runId → triggering agent. Lost on restart; the spec already
     // doesn't auto-resume interrupted runs, so this is consistent.
@@ -191,50 +32,60 @@ export const createWorkflowRunner = (deps) => {
     const assertRunActive = (runId) => {
         if (isRunStopped(runId))
             throw new Error('Stopped by user');
+        const abortReason = abortedRuns.get(runId);
+        if (abortReason)
+            throw new Error(abortReason);
+        const current = workflowRunStore.getRun(runId);
+        if (current && current.status !== 'running') {
+            throw new Error(current.error || `Workflow run is ${current.status}`);
+        }
+    };
+    const cancelOpenDispatch = (workspaceId, dispatchId, reason) => {
+        const cancelled = dispatchPort.cancelOpenDispatchForRun(workspaceId, dispatchId, reason);
+        if (cancelled)
+            awaiter.notifyCancel(dispatchId, reason);
+    };
+    const cancelOpenDispatchesForRun = (runId, workspaceId, reason) => {
+        const errors = [];
+        for (const dispatchId of dispatchPort.listOpenDispatchIdsForRun(runId)) {
+            try {
+                cancelOpenDispatch(workspaceId, dispatchId, reason);
+            }
+            catch (error) {
+                errors.push(errorToMessage(error));
+                console.error('[hive] swallowed:workflow.cancelOpenDispatch', error);
+            }
+        }
+        return errors;
+    };
+    const appendTerminalRunError = (runId, message) => {
+        const current = workflowRunStore.getRun(runId);
+        if (!current || current.status === 'completed')
+            return;
+        const nextError = current.error ? `${current.error}; ${message}` : message;
+        workflowRunStore.updateRun(runId, { error: nextError });
     };
     const executeWorkflow = async (run, loaded, args, hivePort) => {
         const workspaceId = run.workspaceId;
         const workflowAgentId = getWorkflowAgentId(workspaceId);
-        let stepCounter = 0;
-        const spawnedWorkers = [];
-        const activeAgentCalls = new Set();
         // Read the CLI policy once per run so a 1000-way fan-out doesn't hit the
         // app_state table per agent() call.
         const cliPolicy = getWorkflowCliPolicy();
+        let agentExecutor;
+        const dagLayerTracker = createWorkflowDagLayerTracker({
+            awaiter,
+            cancelQueuedAgentCallsForDagLayer: (layerId, reason) => agentExecutor.cancelQueuedAgentCallsForDagLayer(layerId, reason),
+            dispatchPort,
+            workspaceId,
+        });
         // TIER 2 #2 + #11 — runtime caps. Per-run agent ceiling, per-call
         // concurrency throttle, and a hard wall-clock budget. meta can
         // override the defaults per script. The budget timer arms here and
         // gets cleared in the finally branch so a fast-completing run
         // doesn't leave a dangling timer in the event loop.
-        const maxAgents = typeof loaded.meta.maxAgentCalls === 'number' && loaded.meta.maxAgentCalls > 0
-            ? loaded.meta.maxAgentCalls
-            : DEFAULT_MAX_AGENTS_PER_RUN;
         const maxDurationMs = typeof loaded.meta.maxDurationMs === 'number' && loaded.meta.maxDurationMs > 0
             ? loaded.meta.maxDurationMs
             : DEFAULT_MAX_DURATION_MS;
-        // Simple semaphore — a FIFO queue of resolvers. Each acquire() returns
-        // a release() callback that pumps the next waiter. Default
-        // concurrency is min(16, cores-2), matching CC's documented cap;
-        // intentionally shared across all parallel/pipeline calls in this
-        // run so nested fan-outs don't blow through the cap.
-        const concurrencyLimit = DEFAULT_MAX_CONCURRENT_AGENTS;
-        let inFlight = 0;
-        const waitQueue = [];
-        const acquireSlot = async () => {
-            if (inFlight < concurrencyLimit) {
-                inFlight++;
-                return () => {
-                    inFlight--;
-                    waitQueue.shift()?.();
-                };
-            }
-            await new Promise((resolve) => waitQueue.push(resolve));
-            inFlight++;
-            return () => {
-                inFlight--;
-                waitQueue.shift()?.();
-            };
-        };
         // R1/R2 auto-rounding: a phase('Find') called twice in the same run gets
         // rendered as "Find" then "Find R2" etc. Mirrors Claude Code's /workflows
         // view — keeps the UI readable when a workflow loops phases.
@@ -247,139 +98,27 @@ export const createWorkflowRunner = (deps) => {
             currentPhaseTitle = next > 1 ? `${trimmed} R${next}` : trimmed;
             workflowRunStore.updateRun(run.id, { phase: currentPhaseTitle });
         };
-        const agent = async (prompt, opts = {}) => {
-            let markAgentCallDone;
-            const agentCallDone = new Promise((resolve) => {
-                markAgentCallDone = resolve;
-            });
-            activeAgentCalls.add(agentCallDone);
-            try {
-                assertRunActive(run.id);
-                // TIER 2 #11 — hard ceiling so a runaway `while (true) await agent()`
-                // can't spawn unbounded PTY subprocesses. The check is BEFORE the
-                // step counter increments so the error message names the cap, not
-                // the over-cap step.
-                if (stepCounter >= maxAgents) {
-                    throw new Error(`Workflow agent cap exceeded: ${maxAgents} calls (set meta.maxAgentCalls to raise)`);
-                }
-                const myStep = ++stepCounter;
-                // TIER 2 #4 — resolve agentType. If it's a built-in WorkerRole
-                // (coder/reviewer/tester/custom) we honour it directly with the
-                // CLI default. Otherwise look up a workspace custom role template
-                // by name: on hit we clone its command + args; on miss we throw a
-                // clear error (silent fallback to 'coder' would mask typos and
-                // make the Hive-distinctive custom-role library invisible).
-                const requestedType = opts.agentType ?? 'coder';
-                let role;
-                let command;
-                let templateArgs = [];
-                if (typeof requestedType === 'string' && !isBuiltInWorkerRole(requestedType)) {
-                    const template = roleTemplateResolver.findByName(requestedType);
-                    if (!template) {
-                        throw new Error(`Workflow agentType '${requestedType}' is not a built-in role (coder/reviewer/tester/custom) and no matching role template exists in this workspace. ` +
-                            `Use a built-in role or an existing dispatchable role template.`);
-                    }
-                    if (template.roleType === 'sentinel') {
-                        throw new Error(`Workflow agentType '${requestedType}' resolves to a sentinel role, but sentinels are read-only observers and cannot receive workflow dispatches. Use coder, reviewer, tester, custom, or a dispatchable role template.`);
-                    }
-                    /* Templates can carry roleType='orchestrator' for the system-level
-                     Orchestrator template; that's not a valid workflow worker role,
-                     so we collapse it to 'custom'. Anything else maps through as-is. */
-                    role = template.roleType === 'orchestrator' ? 'custom' : template.roleType;
-                    command = resolveWorkflowCli({
-                        ...(opts.cli !== undefined ? { requestedCli: opts.cli } : {}),
-                        isCustomTemplate: true,
-                        templateDefaultCommand: template.defaultCommand,
-                        policy: cliPolicy,
-                    });
-                    templateArgs = template.defaultArgs;
-                }
-                else {
-                    role = requestedType;
-                    command = resolveWorkflowCli({
-                        ...(opts.cli !== undefined ? { requestedCli: opts.cli } : {}),
-                        isCustomTemplate: false,
-                        policy: cliPolicy,
-                    });
-                }
-                const name = opts.label ?? `${requestedType}-${myStep}-${randomUUID()}`;
-                const baseLaunchConfig = resolveCliLaunchConfig(command) ?? { command, args: [] };
-                /* Model flag goes AFTER the template's own args so an explicit
-                 opts.model overrides any --model the template baked in. */
-                const launchArgs = [
-                    ...(baseLaunchConfig.args ?? []),
-                    ...templateArgs,
-                    ...buildModelArgs(baseLaunchConfig.command, opts.model),
-                ];
-                // TIER 2 #2 — semaphore. Holding the slot across the full
-                // dispatch+await means a parallel(100) fan-out gets paced at
-                // min(16, cores-2) concurrent PTYs instead of 100 simultaneous
-                // process spawns.
-                const releaseSlot = await acquireSlot();
-                let worker;
-                try {
-                    assertRunActive(run.id);
-                    worker = store.addWorkerWithLaunch(workspaceId, { name, role, ephemeral: true, spawnedBy: 'workflow' }, { ...baseLaunchConfig, args: launchArgs });
-                    spawnedWorkers.push(worker.id);
-                    assertRunActive(run.id);
-                    const liveRun = await store.startAgent(workspaceId, worker.id, { hivePort });
-                    await liveRun.postStartInputReady;
-                    assertRunActive(run.id);
-                    const dispatchPrompt = opts.outputSchema
-                        ? prompt + buildSchemaInstruction(opts.outputSchema)
-                        : prompt;
-                    const dispatch = await store.dispatchTaskByWorkerName(workspaceId, name, dispatchPrompt, {
-                        fromAgentId: workflowAgentId,
-                        hivePort,
-                        workflowRunId: run.id,
-                        stepIndex: myStep,
-                        ...(currentPhaseTitle ? { phase: currentPhaseTitle } : {}),
-                        label: opts.label ?? name,
-                    });
-                    assertRunActive(run.id);
-                    const report = await awaiter.awaitReport(dispatch.id, opts.timeoutMs);
-                    assertRunActive(run.id);
-                    // Structured output: hand back the parsed object, or { text } on a
-                    // parse miss so the script can still branch (treating an absent field
-                    // as the safe default).
-                    if (opts.outputSchema) {
-                        return extractJsonBlock(report.text) ?? { text: report.text };
-                    }
-                    return report.text;
-                }
-                finally {
-                    if (worker) {
-                        try {
-                            store.deleteWorker(workspaceId, worker.id);
-                        }
-                        catch {
-                            /* idempotent — worker may already be gone via cascade or boot cleanup */
-                        }
-                        const idx = spawnedWorkers.indexOf(worker.id);
-                        if (idx !== -1)
-                            spawnedWorkers.splice(idx, 1);
-                    }
-                    releaseSlot();
-                }
-            }
-            finally {
-                markAgentCallDone();
-                activeAgentCalls.delete(agentCallDone);
-            }
-        };
-        // TIER 1 #3 — per-item rejections become null (preserves the
-        // documented contract: callers use `.filter(Boolean)` patterns), BUT
-        // if the run is being stopped we re-throw so the outer catch handles
-        // it as `status='stopped'` instead of silently completing with a
-        // result array full of nulls. Per-item failures still get logged so
-        // they're not entirely invisible (TIER 2 #3 will pipe these into the
-        // run timeline via log()).
-        const catchPerItem = (value) => {
-            if (isRunStopped(run.id))
-                throw value;
-            console.warn(`[workflow ${loaded.meta.name}] item failed:`, value);
-            return null;
-        };
+        agentExecutor = createWorkflowAgentCallExecutor({
+            assertRunActive: () => assertRunActive(run.id),
+            awaiter,
+            cancelOpenDispatch: (dispatchId, reason) => cancelOpenDispatch(workspaceId, dispatchId, reason),
+            cliPolicy,
+            getCurrentPhaseTitle: () => currentPhaseTitle,
+            hivePort,
+            isRunStopped: () => isRunStopped(run.id),
+            ...(typeof loaded.meta.maxAgentCalls === 'number'
+                ? { maxAgentCalls: loaded.meta.maxAgentCalls }
+                : {}),
+            registerDagDispatch: dagLayerTracker.registerDispatch,
+            resolveCliLaunchConfig,
+            roleTemplateResolver,
+            runId: run.id,
+            store,
+            workflowAgentId,
+            workflowName: loaded.meta.name,
+            workspaceId,
+        });
+        const cancelDagLayerAgents = dagLayerTracker.cancelLayerAgents;
         const log = (message) => {
             assertRunActive(run.id);
             // TIER 2 #3 — persist + still echo to stdout for server-log
@@ -429,108 +168,22 @@ export const createWorkflowRunner = (deps) => {
             assertRunActive(run.id);
             return child;
         };
-        const runScriptWorker = () => new Promise((resolve, reject) => {
-            const worker = new Worker(WORKFLOW_VM_WORKER_SOURCE, {
-                eval: true,
-                workerData: {
-                    args,
-                    compiledFunctionSource: loaded.compiledFunctionSource,
-                    scriptPath: loaded.scriptPath,
-                },
-            });
-            activeScriptWorkers.set(run.id, worker);
-            let settled = false;
-            const activeHostCalls = new Set();
-            const settle = (fn) => {
-                if (settled)
-                    return;
-                settled = true;
-                activeScriptWorkers.delete(run.id);
-                void (async () => {
-                    await Promise.allSettled(activeHostCalls);
-                    fn();
-                    void worker.terminate().catch(() => { });
-                })();
-            };
-            const respond = (id, response) => {
-                try {
-                    worker.postMessage({ type: 'hostResponse', id, ...response });
-                }
-                catch {
-                    /* worker already terminated */
-                }
-            };
-            worker.on('message', (message) => {
-                const record = message;
-                if (!record)
-                    return;
-                if (record.type === 'done') {
-                    settle(() => {
-                        if (record.ok)
-                            resolve(record.value);
-                        else
-                            reject(new Error(record.error || 'Hive workflow script failed'));
-                    });
-                    return;
-                }
-                if (record.type !== 'hostCall' || typeof record.id !== 'string')
-                    return;
-                const callId = record.id;
-                if (settled) {
-                    respond(callId, { ok: false, error: 'Stopped by user' });
-                    return;
-                }
-                const hostCall = (async () => {
-                    try {
-                        assertRunActive(run.id);
-                        const callArgs = Array.isArray(record.args) ? record.args : [];
-                        let value;
-                        switch (record.name) {
-                            case 'agent': {
-                                const [prompt, opts] = callArgs;
-                                value = await agent(typeof prompt === 'string' ? prompt : String(prompt), (opts ?? {}));
-                                break;
-                            }
-                            case 'phase': {
-                                const [title] = callArgs;
-                                phase(typeof title === 'string' ? title : String(title ?? ''));
-                                break;
-                            }
-                            case 'log': {
-                                const [message] = callArgs;
-                                log(typeof message === 'string' ? message : String(message));
-                                break;
-                            }
-                            case 'workflow': {
-                                const [scriptName, childArgs] = callArgs;
-                                value = await workflow(typeof scriptName === 'string' ? scriptName : String(scriptName), childArgs);
-                                break;
-                            }
-                            case 'catchPerItem': {
-                                const [item] = callArgs;
-                                value = catchPerItem(item);
-                                break;
-                            }
-                            default:
-                                throw new Error(`Unknown workflow host call: ${record.name}`);
-                        }
-                        assertRunActive(run.id);
-                        respond(callId, { ok: true, value });
-                    }
-                    catch (error) {
-                        respond(callId, { ok: false, error: errorToMessage(error) });
-                    }
-                })();
-                activeHostCalls.add(hostCall);
-                void hostCall.finally(() => activeHostCalls.delete(hostCall));
-            });
-            worker.on('error', (error) => settle(() => reject(error)));
-            worker.on('exit', (code) => {
-                if (settled)
-                    return;
-                activeScriptWorkers.delete(run.id);
-                reject(new Error(`Hive workflow VM worker exited before completion (code ${code})`));
-            });
+        const runScriptWorker = () => runWorkflowScriptWorker({
+            args,
+            compiledFunctionSource: loaded.compiledFunctionSource,
+            scriptPath: loaded.scriptPath,
+            workerSource: WORKFLOW_VM_WORKER_URL,
+            onWorkerCreated: (worker) => activeScriptWorkers.set(run.id, worker),
+            onWorkerSettled: () => activeScriptWorkers.delete(run.id),
+            handlers: {
+                agent: agentExecutor.agent,
+                phase,
+                log,
+                workflow,
+                catchPerItem: agentExecutor.catchPerItem,
+                cancelDagLayerAgents,
+                assertRunActive: () => assertRunActive(run.id),
+            },
         });
         // TIER 2 #11 — wall-clock budget timer. Triggers stopRun on
         // expiry, which routes through the same path as a user-initiated
@@ -547,6 +200,10 @@ export const createWorkflowRunner = (deps) => {
         try {
             assertRunActive(run.id);
             const returnValue = await runScriptWorker();
+            const abortReason = abortedRuns.get(run.id);
+            if (abortReason) {
+                throw new Error(abortReason);
+            }
             // TIER 1 #2 — if stop was called DURING the run, parallel/pipeline may
             // have caught the cancel rejections (one per in-flight thunk) before
             // the per-item catch could re-throw, e.g. when the user stops AFTER
@@ -580,7 +237,8 @@ export const createWorkflowRunner = (deps) => {
             const wasStopped = workflowRunStore.getRun(run.id)?.status === 'stopped';
             if (wasStopped)
                 return;
-            const message = error instanceof Error ? error.message : String(error);
+            const message = errorToMessage(error);
+            abortedRuns.set(run.id, message);
             workflowRunStore.updateRun(run.id, {
                 status: 'failed',
                 finishedAt: Date.now(),
@@ -589,18 +247,29 @@ export const createWorkflowRunner = (deps) => {
         }
         finally {
             clearTimeout(budgetTimer);
-            await Promise.allSettled(activeAgentCalls);
+            const finalRecord = workflowRunStore.getRun(run.id);
+            let terminalCleanup = false;
+            if (finalRecord && finalRecord.status !== 'completed') {
+                terminalCleanup = true;
+                const cleanupReason = finalRecord.error || abortedRuns.get(run.id) || 'Workflow run stopped';
+                agentExecutor.forceCancelActiveDispatchWaiters(cleanupReason);
+                const cancelErrors = cancelOpenDispatchesForRun(run.id, finalRecord.workspaceId, cleanupReason);
+                if (cancelErrors.length > 0) {
+                    appendTerminalRunError(run.id, `cleanup failed to cancel ${cancelErrors.length} workflow dispatch${cancelErrors.length === 1 ? '' : 'es'}: ${cancelErrors.join('; ')}`);
+                }
+            }
+            const activeCalls = await agentExecutor.waitForActiveCalls(terminalCleanup ? ACTIVE_AGENT_CALL_CLEANUP_TIMEOUT_MS : undefined);
+            if (!activeCalls.settled) {
+                const message = `cleanup timed out after ${ACTIVE_AGENT_CALL_CLEANUP_TIMEOUT_MS}ms waiting for ${activeCalls.activeCount} active workflow agent call${activeCalls.activeCount === 1 ? '' : 's'} to settle`;
+                console.error(`[hive] workflow.${message}`);
+                appendTerminalRunError(run.id, message);
+                agentExecutor.forceCancelActiveDispatchWaiters(message);
+                await agentExecutor.waitForActiveCalls(1000);
+            }
             // Belt-and-suspenders: dismiss any ephemeral worker still alive. The
             // per-call try/finally should already have cleaned each one up; this is
             // an idempotent safety net for unexpected paths.
-            for (const workerId of spawnedWorkers.splice(0)) {
-                try {
-                    store.deleteWorker(workspaceId, workerId);
-                }
-                catch {
-                    /* swallow */
-                }
-            }
+            agentExecutor.deleteSpawnedWorkers();
             // Notify the triggering agent (orchestrator) that the run reached a
             // terminal state. Mirrors Claude Code's <task-notification> envelope.
             const triggeredByAgentId = triggeringAgentByRun.get(run.id);
@@ -617,6 +286,7 @@ export const createWorkflowRunner = (deps) => {
                 }
             }
             stoppedRuns.delete(run.id);
+            abortedRuns.delete(run.id);
         }
     };
     const buildCreateInput = (input, loaded) => {
@@ -694,9 +364,7 @@ export const createWorkflowRunner = (deps) => {
         // Cancel every open workflow dispatch tied to this run; this rejects the
         // runner's pending `awaitReport` promises, which propagates up the
         // executeWorkflow try → its catch sets status='stopped'.
-        for (const dispatchId of dispatchPort.listOpenDispatchIdsForRun(runId)) {
-            awaiter.notifyCancel(dispatchId, 'Stopped by user');
-        }
+        cancelOpenDispatchesForRun(runId, current.workspaceId, 'Stopped by user');
         for (const child of workflowRunStore.listChildRuns(runId)) {
             if (child.status === 'running')
                 stopRunAndChildren(child.id, visited);

package/dist/src/server/workflow-script-host-dispatcher.d.ts

@@ -0,0 +1,14 @@
+export interface WorkflowScriptHostHandlers<AgentOptions> {
+    agent: (prompt: string, opts: AgentOptions) => Promise<unknown>;
+    assertRunActive: () => void;
+    cancelDagLayerAgents: (layerId: unknown, reason: unknown) => void;
+    catchPerItem: (value: unknown) => unknown;
+    log: (message: string) => void;
+    phase: (title: string) => void;
+    workflow: (scriptName: string, childArgs?: unknown) => Promise<unknown>;
+}
+export interface WorkflowScriptHostDispatchResult {
+    skipPostActiveAssert?: boolean;
+    value?: unknown;
+}
+export declare const dispatchWorkflowScriptHostCall: <AgentOptions>(name: string | undefined, callArgs: unknown[], handlers: WorkflowScriptHostHandlers<AgentOptions>) => Promise<WorkflowScriptHostDispatchResult>;