cool-workflow 0.2.4 → 0.2.5

package/dist/shell/commit.js

@@ -199,7 +199,13 @@ function commitState(run, input) {
     // NOT prematurely move the run off "interpret".
     if (gate.verifierGated)
         run.loopStage = "checkpoint";
-    (0, fs_atomic_1.writeJson)(snapshotPath, { commit, run });
+    // Only store the commit's own (small, bounded) record. The whole run
+    // (all tasks, nodes, workers, feedback, and run.commits itself) used to
+    // be embedded here too, but no reader anywhere ever parses a `run` key
+    // back out of a commits/<id>.json file — see docs/run-retention-reclamation.7.md.
+    // Embedding the growing run made total commit-dir bytes grow like N^2
+    // in the number of commits.
+    (0, fs_atomic_1.writeJson)(snapshotPath, { commit });
     run.commits.push(commit);
     return commit;
 }
@@ -214,7 +220,7 @@ function recordCommitNode(run, commit, options, gate) {
             artifacts: [{ id: "snapshot", kind: "json", path: commit.snapshotPath }],
             evidence: commit.evidence || verifierNode.evidence,
             metadata: { ...(options.metadata || {}), reason: options.reason, commitId: commit.id, verifierGated: true, checkpoint: false, verifierNodeId: verifierNode.id, candidateId: gate.candidateId, selectionId: gate.selectionId, selectionNodeId: gate.selectionNodeId },
-        }, { persist: false, persistNode: node_store_1.writeRunNode });
+        }, { persist: false, persistNode: node_store_1.writeRunNode, pathExists: fs.existsSync });
         if (gate.selectionNodeId && commitResult.outputNodeId)
             linkAdditionalParent(run, gate.selectionNodeId, commitResult.outputNodeId);
         return commitResult.outputNodeId;

package/dist/shell/coordinator-io.js

@@ -158,6 +158,75 @@ function dirtySetsFor(state) {
 function markBlackboardDirty(state, kind, id) {
     dirtySetsFor(state)[kind].add(id);
 }
+// Dirty-id tracking for state.messages, PARALLEL to the 5-kind tracking
+// above (not folded in: messages share ONE ordered log file, not a
+// recordPath-per-id file). The only push site is postBlackboardMessage
+// (`state.messages.push(message)`), so the dirty ids at persist time are
+// always exactly the trailing ids of state.messages since the last
+// persist. Kept off the serialized state for the same reason as
+// blackboardDirtySets: a WeakMap, never a field on BlackboardState.
+const blackboardMessagesDirty = new WeakMap();
+function dirtyMessageIdsFor(state) {
+    let ids = blackboardMessagesDirty.get(state);
+    if (!ids) {
+        ids = new Set();
+        blackboardMessagesDirty.set(state, ids);
+    }
+    return ids;
+}
+function markBlackboardMessageDirty(state, id) {
+    dirtyMessageIdsFor(state).add(id);
+}
+// Writes messages.jsonl for a persist call, or skips it entirely when
+// nothing changed (the common case: 7 of the 8 persistBlackboardState call
+// sites never touch state.messages).
+//
+// Fast path (the common case for the ONE call site that does change
+// messages, postBlackboardMessage): the dirty ids are exactly the trailing
+// `k` entries of state.messages (the sole push site always appends). Sort
+// only that small batch, check it does not need to interleave with the
+// already-on-disk tail, and APPEND it — no read, no resort, no rewrite of
+// the earlier messages. This turns posting M messages from O(M^2) written
+// bytes into O(M) total.
+//
+// Fallback (a caller-supplied custom --id ties createdAt with an earlier
+// message and sorts BEFORE it, so a plain append would leave the file out
+// of order): resort the whole array and rewrite the whole file, byte-
+// identical to the code path this replaces.
+//
+// Either way state.messages is left fully sorted after this call, exactly
+// as the old unconditional `.sort()` left it — buildBlackboardGraph (which
+// iterates state.messages in raw array order) sees the same order as
+// before.
+function persistBlackboardMessages(run, state) {
+    const dirty = dirtyMessageIdsFor(state);
+    if (dirty.size === 0)
+        return;
+    const file = messagesPath(run);
+    const total = state.messages.length;
+    const k = dirty.size;
+    const priorCount = total - k;
+    const batch = priorCount >= 0 ? state.messages.slice(priorCount).sort(cb.compareRecords) : [];
+    const batchMatchesDirty = batch.length === dirty.size && batch.every((message) => dirty.has(message.id));
+    const priorTail = priorCount > 0 ? state.messages[priorCount - 1] : undefined;
+    const canAppend = batchMatchesDirty && (!priorTail || cb.compareRecords(priorTail, batch[0]) <= 0);
+    if (canAppend) {
+        state.messages.splice(priorCount, k, ...batch);
+        const lines = batch.map((message) => `${JSON.stringify(message)}\n`).join("");
+        const currentBytes = fs.existsSync(file) ? fs.statSync(file).size : 0;
+        // Same torn-tail guard as trust-audit's events.jsonl (shared helper):
+        // a crash mid-append can leave the log without its final "\n"; put the
+        // new lines on their own clean line rather than merging with a torn
+        // tail.
+        const leadingNewline = currentBytes > 0 && !(0, fs_atomic_1.logEndsWithNewline)(file, currentBytes) ? "\n" : "";
+        (0, fs_atomic_1.durableAppendFileSync)(file, leadingNewline + lines);
+    }
+    else {
+        // Same bytes the old unconditional code path always wrote.
+        (0, fs_atomic_1.writeTextDurable)(file, state.messages.sort(cb.compareRecords).map((message) => JSON.stringify(message)).join("\n") + (state.messages.length ? "\n" : ""));
+    }
+    dirty.clear();
+}
 function linkMultiAgent(run, blackboardId, topicIds, input) {
     const groupId = input.agentGroupId ?? input.groupId;
     const roleId = input.agentRoleId ?? input.roleId;
@@ -218,7 +287,9 @@ function persistBlackboardState(run) {
         decisions: state.decisions.map(cb.indexRow),
         messages: state.messages.map((message) => ({ id: message.id, blackboardId: message.blackboardId, topicId: message.topicId, createdAt: message.createdAt, status: message.status, author: message.author, evidenceRefs: message.linkedEvidenceRefs, artifactRefIds: message.linkedArtifactRefIds })),
     });
-    fs.writeFileSync(messagesPath(run), state.messages.sort(cb.compareRecords).map((message) => JSON.stringify(message)).join("\n") + (state.messages.length ? "\n" : ""), "utf8");
+    // messages.jsonl: see persistBlackboardMessages above for the
+    // skip-if-unchanged / append-if-safe / full-rewrite-fallback split.
+    persistBlackboardMessages(run, state);
     const dirty = dirtySetsFor(state);
     for (const id of dirty.topics) {
         const record = state.topics.find((entry) => entry.id === id);
@@ -308,6 +379,7 @@ function postBlackboardMessage(run, input) {
     requireArtifactRefs(run, input.artifactRefIds || []);
     const message = cb.buildMessage(run.id, board, topic, input, id, now(), trust_policy_io_1.hashText, sourceForActorLocal);
     state.messages.push(message);
+    markBlackboardMessageDirty(state, message.id);
     topic.messageIds = cb.unique([...topic.messageIds, message.id]);
     board.messageCount = state.messages.filter((entry) => entry.blackboardId === board.id).length;
     cb.touch(topic, now());

package/dist/shell/drive.js

@@ -630,7 +630,7 @@ function maybeExpandLoop(run) {
                     requiresEvidence: t.requiresEvidence,
                     sandboxProfileId: t.sandboxProfileId,
                 },
-            }, { persist: false, persistNode: (r, node) => void (0, node_store_1.appendRunNode)(r, node) });
+            }, { persist: false, persistNode: (r, node) => void (0, node_store_1.appendRunNode)(r, node), pathExists: fs.existsSync });
             t.stateNodeId = result.outputNodeId;
         }
         (0, dispatch_1.updatePhaseStatuses)(run);
@@ -656,17 +656,32 @@ function driveStep(ctx) {
     return processSelectedTask(ctx, selected.id);
 }
 /** Dispatch every batch task (sequential — dispatch mutates state), then
- *  collect ALL spawn-style agent child outcomes in one concurrent window
- *  (one batch delegate child process, per-job timeout kill). Returns
- *  outcomes keyed by task id; a cache-hit or endpoint-configured agent
- *  gets no prepared outcome and settles through the serial accept path
- *  inside processSelectedTask. Dispatch failures become recorded fail
- *  steps up front, exactly what the serial path would emit. Byte-exact
- *  to the old build's src/drive.ts's prepareConcurrentOutcomes. */
+ *  collect ALL agent child outcomes in one concurrent window: spawn-style
+ *  (CLI-binary) agents through the batch delegate child, and endpoint-mode
+ *  agents through the HTTP batch delegate child — so `--concurrency N` runs N
+ *  endpoint delegations in parallel instead of one-at-a-time on the serial
+ *  path. Returns outcomes keyed by task id; a cache-hit, endpoint-configured
+ *  agent, or sub-workflow task gets no prepared outcome and settles through
+ *  the serial accept path inside processSelectedTask (a sub-workflow task is
+ *  never spawn-eligible in the first place — processSelectedTask always
+ *  takes its own runSubWorkflow branch for it, so building a spawn job here
+ *  would only be spawned and thrown away unused). Dispatch failures become
+ *  recorded fail steps up front, exactly what the serial path would emit. */
 function prepareConcurrentOutcomes(ctx, batch) {
     const failSteps = new Map();
     const jobs = [];
     const jobTaskIds = [];
+    const endpointJobs = [];
+    const endpointTaskIds = [];
+    // Set whenever a task's status flips pending -> running (workerId
+    // assigned) this round, regardless of whether it later becomes a real
+    // spawn job, an endpoint job, a cache hit, or a sub-workflow task. That
+    // dispatch is a real state mutation on the round-cached run object that
+    // nothing has written to disk yet — the pre-spawn checkpoint below must
+    // flush it even when the batch produces zero spawn/endpoint jobs (e.g. an
+    // all-sub-workflow batch), or a crash before the round-end flush loses
+    // the dispatch.
+    let dispatchedThisRound = false;
     for (const taskId of batch) {
         const run = loadRun(ctx);
         const task = run.tasks.find((candidate) => candidate.id === taskId);
@@ -681,6 +696,7 @@ function prepareConcurrentOutcomes(ctx, batch) {
                 continue;
             }
             workerId = dispatchedTask.workerId;
+            dispatchedThisRound = true;
         }
         if (!workerId) {
             failSteps.set(taskId, (0, drive_decide_1.makeStep)("dispatch", "failed", { runId: ctx.runId, taskId, phase: task.phase, reason: "no worker scope for task" }));
@@ -694,7 +710,15 @@ function prepareConcurrentOutcomes(ctx, batch) {
         const cachePath = resultCachePath(freshRun, task, (0, hash_1.sha256)(task.prompt || ""), ctx.incremental, delegationDigest);
         if (cachePath && fs.existsSync(cachePath))
             continue;
-        const job = (0, agent_1.prepareAgentSpawn)({
+        // A sub-workflow task never goes through the spawn path — processSelectedTask
+        // checks task.subWorkflow before it ever looks at a prepared spawn outcome
+        // and always takes its own runSubWorkflow branch instead (mirrors that
+        // check byte-for-byte). Building a spawn OR endpoint job for it here would
+        // just be a real agent child spawned/delegated and thrown away unused.
+        const subWorkflow = task.subWorkflow;
+        if (subWorkflow)
+            continue;
+        const request = {
             schemaVersion: 1,
             runId: ctx.runId,
             taskId: task.id,
@@ -705,7 +729,8 @@ function prepareConcurrentOutcomes(ctx, batch) {
             label: task.id,
             timeoutMs: ctx.config.timeoutMs,
             delegation: { command: ctx.config.command, args: ctx.config.args, endpoint: ctx.config.endpoint, model: task.model || ctx.config.model },
-        });
+        };
+        const job = (0, agent_1.prepareAgentSpawn)(request);
         if (job) {
             const sandboxPolicy = manifest.sandboxPolicy;
             if (sandboxPolicy) {
@@ -713,32 +738,53 @@ function prepareConcurrentOutcomes(ctx, batch) {
             }
             jobs.push(job);
             jobTaskIds.push(taskId);
+            continue;
         }
+        // Endpoint-configured agent: no CLI binary to batch-spawn, but its HTTP
+        // delegation joins the concurrent window through the endpoint batch child
+        // (runEndpointBatchOutcomes) rather than settling one-at-a-time on the
+        // serial path. An unconfigured agent (neither command nor endpoint) returns
+        // undefined from both prepares here and falls to the serial refusal.
+        const endpointJob = (0, agent_1.prepareEndpointJob)(request);
+        if (endpointJob) {
+            endpointJobs.push(endpointJob);
+            endpointTaskIds.push(taskId);
+        }
+    }
+    const totalJobs = jobs.length + endpointJobs.length;
+    if (totalJobs) {
+        emitProgress(`⇉ concurrent round: ${totalJobs} agent${totalJobs > 1 ? "s" : ""} spawning in parallel, may take minutes…`);
     }
-    if (jobs.length) {
-        emitProgress(`⇉ concurrent round: ${jobs.length} agent${jobs.length > 1 ? "s" : ""} spawning in parallel, may take minutes…`);
+    if (totalJobs || dispatchedThisRound) {
         // Every task above that reached "pending" got dispatched (workerId
         // assigned) in the round-cached run object, but nothing durable was
         // written for it — unlike the serial path's own dispatch branch, which
         // always checkpoints immediately. Flush now, BEFORE the batch's long
-        // spawn window opens: a crash mid-spawn (which can run for minutes)
-        // then leaves state.json correctly showing these tasks as dispatched,
-        // instead of losing the dispatch entirely.
+        // settle window opens: a crash mid-spawn OR mid-sub-workflow-recursion
+        // (both can run for minutes) then leaves state.json correctly showing
+        // these tasks as dispatched, instead of losing the dispatch entirely.
+        // Gated on dispatchedThisRound (not just totalJobs) so a batch made
+        // entirely of sub-workflow and/or cache-hit tasks — which builds zero
+        // spawn/endpoint jobs — still gets this flush before its long-running
+        // settle loop starts.
         (0, run_store_1.saveCheckpoint)(loadRun(ctx));
     }
-    const settled = (0, agent_1.runAgentBatchOutcomes)(jobs);
     const outcomes = new Map();
+    const settled = (0, agent_1.runAgentBatchOutcomes)(jobs);
     jobTaskIds.forEach((taskId, index) => outcomes.set(taskId, settled[index]));
+    const endpointSettled = (0, agent_1.runEndpointBatchOutcomes)(endpointJobs);
+    endpointTaskIds.forEach((taskId, index) => outcomes.set(taskId, endpointSettled[index]));
     return { outcomes, failSteps };
 }
 /** One concurrent round inside one cached in-memory run: dispatches every
  *  batch task, spawns all spawn-style agent children in one concurrent
- *  window, then settles + accepts in DETERMINISTIC batch (task-id) order
- *  regardless of wall-clock finish order. At round end it flushes once:
- *  commitState(run, "concurrent-round:<n>-tasks") + writeReport +
- *  saveCheckpoint. Cache-hit tasks and endpoint-only agents get no
- *  prepared outcome and settle through the serial path (still inside
- *  this one deferred-persist round). If no step was produced (nothing
+ *  window (CLI-binary agents through the batch child, endpoint agents
+ *  through the HTTP batch child), then settles + accepts in DETERMINISTIC
+ *  batch (task-id) order regardless of wall-clock finish order. At round end
+ *  it flushes once: commitState(run, "concurrent-round:<n>-tasks") +
+ *  writeReport + saveCheckpoint. Cache-hit tasks get no prepared outcome and
+ *  settle through the serial path (still inside this one deferred-persist
+ *  round). If no step was produced (nothing
  *  runnable at round entry — terminal/blocked/token-budget gate) the
  *  round degrades to one plain driveStep. Byte-exact to the old build's
  *  src/drive.ts's driveConcurrentRound. */
@@ -1004,28 +1050,35 @@ function finalizeDriveResult(ctx, options, steps, plannedWorkers, maxIter, exhau
  *  exact synchronous shape) and fixed only in driveAsync(). */
 function drive(runId, cwd, options = {}) {
     const ctx = buildDriveContext(runId, cwd, options);
-    const steps = [];
-    const run0 = loadRun(ctx);
-    const plannedWorkers = run0.tasks.length;
-    const maxIter = (0, drive_decide_1.maxIterations)(plannedWorkers, (0, loop_expansion_1.maxLoopExpansion)(run0), ctx.policy);
-    const emitPhaseProgress = createPhaseProgressEmitter();
-    let exhaustedMaxIterations = !options.once;
-    const stopSignal = createStopSignalController(runId);
-    stopSignal.install();
-    try {
-        for (let i = 0; i < maxIter; i++) {
-            if (stopSignal.getInterruptedBy())
-                break;
-            if (!driveOneRound(ctx, options, steps, emitPhaseProgress)) {
-                exhaustedMaxIterations = false;
-                break;
+    // Resolve the run dir before the mutex: a missing run throws "Run not found"
+    // here (never a drive.lock side effect on a nonexistent run), via a stat only
+    // so the per-round read accounting is unchanged. Mirrors withRunStateLock's
+    // own probe-before-lock.
+    const runDir = (0, run_store_1.resolveRunDir)(runId, cwd);
+    return (0, run_store_1.withDriveLock)(runDir, runId, () => {
+        const steps = [];
+        const run0 = loadRun(ctx);
+        const plannedWorkers = run0.tasks.length;
+        const maxIter = (0, drive_decide_1.maxIterations)(plannedWorkers, (0, loop_expansion_1.maxLoopExpansion)(run0), ctx.policy);
+        const emitPhaseProgress = createPhaseProgressEmitter();
+        let exhaustedMaxIterations = !options.once;
+        const stopSignal = createStopSignalController(runId);
+        stopSignal.install();
+        try {
+            for (let i = 0; i < maxIter; i++) {
+                if (stopSignal.getInterruptedBy())
+                    break;
+                if (!driveOneRound(ctx, options, steps, emitPhaseProgress)) {
+                    exhaustedMaxIterations = false;
+                    break;
+                }
             }
         }
-    }
-    finally {
-        stopSignal.remove();
-    }
-    return finalizeDriveResult(ctx, options, steps, plannedWorkers, maxIter, exhaustedMaxIterations, stopSignal.getInterruptedBy());
+        finally {
+            stopSignal.remove();
+        }
+        return finalizeDriveResult(ctx, options, steps, plannedWorkers, maxIter, exhaustedMaxIterations, stopSignal.getInterruptedBy());
+    });
 }
 /** Same contract and same DriveResult as drive() -- but actually
  *  interruptible. After every round it awaits yieldToEventLoop(), a real
@@ -1053,29 +1106,33 @@ function drive(runId, cwd, options = {}) {
  *  future cycle if it proves needed in practice. */
 async function driveAsync(runId, cwd, options = {}) {
     const ctx = buildDriveContext(runId, cwd, options);
-    const steps = [];
-    const run0 = loadRun(ctx);
-    const plannedWorkers = run0.tasks.length;
-    const maxIter = (0, drive_decide_1.maxIterations)(plannedWorkers, (0, loop_expansion_1.maxLoopExpansion)(run0), ctx.policy);
-    const emitPhaseProgress = createPhaseProgressEmitter();
-    let exhaustedMaxIterations = !options.once;
-    const stopSignal = createStopSignalController(runId);
-    stopSignal.install();
-    try {
-        for (let i = 0; i < maxIter; i++) {
-            if (stopSignal.getInterruptedBy())
-                break;
-            if (!driveOneRound(ctx, options, steps, emitPhaseProgress)) {
-                exhaustedMaxIterations = false;
-                break;
+    // Resolve the run dir before the mutex — see drive() above.
+    const runDir = (0, run_store_1.resolveRunDir)(runId, cwd);
+    return (0, run_store_1.withDriveLockAsync)(runDir, runId, async () => {
+        const steps = [];
+        const run0 = loadRun(ctx);
+        const plannedWorkers = run0.tasks.length;
+        const maxIter = (0, drive_decide_1.maxIterations)(plannedWorkers, (0, loop_expansion_1.maxLoopExpansion)(run0), ctx.policy);
+        const emitPhaseProgress = createPhaseProgressEmitter();
+        let exhaustedMaxIterations = !options.once;
+        const stopSignal = createStopSignalController(runId);
+        stopSignal.install();
+        try {
+            for (let i = 0; i < maxIter; i++) {
+                if (stopSignal.getInterruptedBy())
+                    break;
+                if (!driveOneRound(ctx, options, steps, emitPhaseProgress)) {
+                    exhaustedMaxIterations = false;
+                    break;
+                }
+                await yieldToEventLoop();
             }
-            await yieldToEventLoop();
         }
-    }
-    finally {
-        stopSignal.remove();
-    }
-    return finalizeDriveResult(ctx, options, steps, plannedWorkers, maxIter, exhaustedMaxIterations, stopSignal.getInterruptedBy());
+        finally {
+            stopSignal.remove();
+        }
+        return finalizeDriveResult(ctx, options, steps, plannedWorkers, maxIter, exhaustedMaxIterations, stopSignal.getInterruptedBy());
+    });
 }
 function drivePreview(runId, cwd, args = {}) {
     const run = (0, run_store_1.loadRunFromCwd)(runId, cwd);

package/dist/shell/error-feedback-io.js

@@ -290,6 +290,12 @@ function resolveFeedback(run, feedbackId, result) {
 function runPipelineStage(run, stageId, inputNodeId, options = {}, runnerOptions = {}) {
     return (0, runner_1.runPipelineStage)(run, stageId, inputNodeId, options, {
         ...runnerOptions,
+        // The core gate's `pathExists` defaults to `() => true` (a pure core/
+        // module never reads the filesystem). This is the shell seam, so default
+        // it to the real `fs.existsSync` — that is what makes the contract's
+        // `requireReadablePaths` / `missing-artifact-path` gate live. A caller may
+        // still override it (e.g. a replay against a captured path set).
+        pathExists: runnerOptions.pathExists || fs.existsSync,
         // Keep the caller-named failure node id (the old build honored outputNodeId
         // for the preserved failure node); the raw core auto-mints when unset.
         failureNodeId: runnerOptions.failureNodeId || options.outputNodeId,