cowork-harness 0.6.0 → 0.7.1

package/.claude/skills/cowork-harness/scripts/assertion-keys.json

@@ -21,6 +21,8 @@
     "subagent_tool_used",
     "tool_called",
     "tool_not_called",
+    "tool_result_contains",
+    "tool_result_not_contains",
     "transcript_contains",
     "transcript_matches",
     "transcript_no_host_path",

package/.claude/skills/cowork-harness/scripts/scenario.py

@@ -45,6 +45,8 @@ CONTENT_KEYS = {
     "transcript_not_contains",
     "transcript_matches",
     "transcript_not_matches",
+    "tool_result_contains",
+    "tool_result_not_contains",
     "tool_called",
     "tool_not_called",
     "subagent_tool_used",

package/CHANGELOG.md

@@ -6,6 +6,71 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [0.7.1] — 2026-06-20
+
+### Fixed
+
+- **`file_exists` and `user_visible_artifact` now pass for truncated (large) cassette artifacts.**
+  A truncated manifest entry (`truncated: true`) carries `path`, `bytes`, and `sha256` — positive
+  proof the file existed at record time. 0.7.0 incorrectly failed these existence/promotion
+  assertions with `"was truncated in the cassette — content was not committed; assertion cannot
+  pass"`, producing false-REDs for any cassette whose artifacts exceeded the 64 KiB inline cap.
+  Only **content** assertions (`artifact_json`) require the inlined body; existence assertions
+  now correctly pass from the manifest. `artifact_json` on a truncated artifact continues to fail
+  (the 0-byte placeholder is not valid JSON). Regression test added.
+
+## [0.7.0] — 2026-06-19
+
+### Added
+
+- **`chat --plugin <dir>` (repeatable)** — load additional local plugins into a `chat` session alongside
+  the primary skill folder. Each `--plugin <dir>` is appended to `plugins.local_plugins` so multi-plugin
+  interactive debugging no longer requires a custom session YAML. In `--raw` mode (native Docker), `--plugin`
+  flags are silently ignored with a warning: `chat --raw: --plugin flags are ignored in --raw mode`.
+- **`/help` in the `chat` REPL** — typing `/help` in an interactive session now prints
+  `Commands: /exit  /quit  /help` and continues rather than forwarding the text to the model. The startup
+  prompt was updated from "type your message, /exit to quit" to "type your message (/help for commands)".
+- **`scrubField(value, secrets)` exported from `src/secrets.ts`** — a new multi-pass field-level scrubber
+  that covers token appearances beyond direct substring matches:
+  - Pass 1: direct `scrub()` — catches literal, base64-encoded, and `encodeURIComponent`-encoded tokens.
+  - Pass 2: whole-field base64 decode (≥20-char pure base64 strings) — if the decoded form contains a
+    secret hit, returns `[REDACTED:base64]`.
+  - Pass 3: whole-field URI decode (values containing `%`) — if the decoded form contains a secret hit,
+    returns `[REDACTED:uri]`.
+  Applied in `cassette.ts` artifact scrubbing: base64 artifacts (`encoding === "base64"`) are replaced
+  wholesale with `[REDACTED:base64]`, the encoding marker is cleared (so replay decodes as UTF-8), and the
+  sha256 is recomputed over the marker bytes (with a `::warning::` that artifact assertions will fail at
+  replay). UTF-8 artifacts pass through `scrubField` safely.
+- **`prompt_asset_missing` VerdictSignal** — `computeVerdict()` now pushes a `{ code: "prompt_asset_missing",
+  severity: "warn" }` signal when `result.fidelityWarnings` contains a "referenced asset not found" entry,
+  making a missing prompt asset visible in the verdict output rather than buried in the run log.
+- **`onInfraError` callback in `makeWorkspaceHandler`** — an optional sixth parameter
+  `onInfraError?: (message: string) => void` lets callers intercept infrastructure errors
+  (ETIMEDOUT / killed / no code+stdout+stderr) separately from model-visible error text.
+  `spawnHostLoop()` wires this to append `{ type: "infra_error", ts, message }` to `events.jsonl` so
+  infrastructure failures are structured and queryable rather than only appearing in the model-visible
+  response string.
+
+### Fixed
+
+- **`ExternalDecider` no longer coerces `"first"` to option 1.** `coerceLabel` gained an
+  `enableFirstShorthand` parameter (default `true`). External deciders (`--decider-cmd`,
+  `--decider-dir` helpers) now call `coerceLabel(raw, labels, false)`, so a helper script that
+  accidentally returns the string `"first"` must match an actual label named `"first"` — it is no
+  longer silently promoted to the first option. The shorthand remains active for internal (scripted)
+  use.
+- **`flagValue` and `chat --model` reject empty strings.** `flagValue()` in `src/cli.ts` now exits `2`
+  with a clear message when the supplied value is blank or whitespace-only. `chat` additionally guards
+  the `--model` value inline after parsing, so `--model ""` and `--model $UNSET_VAR` both fail loudly
+  instead of passing an empty model ID to the runtime.
+- **`redactCassette` skips `[REDACTED*]` marker bodies.** The per-line JSON redaction pass
+  (`redactJsonLine`) is now bypassed for artifact bodies that already start with `[REDACTED` — preventing
+  the sha256 from being corrupted by a second redaction pass over the marker string.
+- **TLD list extended from 22 to 51 entries.** The domain scanner in `src/scan.ts` now recognises
+  major European, Asian, and Latin American ccTLDs:
+  `ch|nl|se|no|it|jp|br|nz|in|sg|kr|mx|es|pt|pl|be|at|dk|fi|ie|ru|cn|tw|hu|cz|ro|il|za|ar|cl|pe|tr`.
+  Domain findings that were previously missed on these TLDs now fire correctly.
+
 ## [0.6.0] — 2026-06-19
 
 ### Breaking changes

package/README.md

@@ -24,6 +24,15 @@ Scriptable, CI-friendly test harness that reproduces **Claude Cowork's observabl
 
 > **What this is and isn't.** This is an *emulator of the contract*, not the Desktop runtime. Cowork's real session control plane lives behind the Desktop renderer's IPC (per-build UUID + `senderFrame` origin checks) and the app ships with remote debugging disabled (verified: `--remote-debugging-port` opens no listener; Electron `EnableNodeCliInspectArguments` fuse is OFF). So you **cannot** drive the real Apple Virtualization.framework microVM from a script. What you *can* faithfully reproduce is everything that actually changes how a **skill** behaves: the same agent binary in **cowork mode** (`CLAUDE_CODE_IS_COWORK=1` — there is no `--cowork` flag), the same mount layout, the same egress allowlist, and the same permission/question protocol. That's what this project does.
 
+**Zero-friction preview — no token, no Docker.** A committed cassette replays on a fresh clone:
+
+```bash
+npm install -g cowork-harness
+cowork-harness replay examples/replays/example-pdf-skill.cassette.json
+```
+
+Full setup → [Quick start](#quick-start).
+
 ---
 
 ## Why this works for skill testing
@@ -226,6 +235,9 @@ cowork-harness replay --cassette examples/replays/example-pdf-skill.cassette.jso
 
 # Cassettes are COMMITTED fixtures — record against synthetic data, and gate them in CI:
 cowork-harness verify-cassettes examples/replays/   # privacy scan (email/currency/domain) + staleness; exit 1 on a finding
+
+# 5. Lint scenarios before committing (catches silent false-greens in assertion placement)
+cowork-harness lint scenarios/*.yaml
 ```
 
 > **Privacy:** a cassette snapshots the transcript and the `outputs/` JSON bodies, so it's committed PII

package/dist/agent/session.js

@@ -162,6 +162,10 @@ export class LiveAgentSession {
     rejectError;
     /** Bounded tail of the child's stderr, for the nonzero-exit error message. */
     stderrTail = "";
+    writeQueue = [];
+    pumping = false;
+    queueIdle;
+    closing = false;
     constructor(proc, outDir) {
         this.proc = proc;
         this.outDir = outDir;
@@ -180,7 +184,7 @@ export class LiveAgentSession {
             if (this.rejectError)
                 this.rejectError(e);
             // else: the error fired before/after the generator — log it but don't throw
-            else
+            else if (!this.closing)
                 this.events.write(JSON.stringify({ _emu: "stdin_error", message: String(e) }) + "\n");
         });
     }
@@ -214,7 +218,8 @@ export class LiveAgentSession {
         // below rejects and the generator yields a typed {type:"error"} event instead of hanging on
         // stdout that will never arrive.
         this.proc.on("error", (e) => {
-            this.events.write(JSON.stringify({ _emu: "spawn_error", message: String(e) }) + "\n");
+            if (!this.closing)
+                this.events.write(JSON.stringify({ _emu: "spawn_error", message: String(e) }) + "\n");
             if (this.rejectError)
                 this.rejectError(e instanceof Error ? e : new Error(String(e)));
         });
@@ -238,7 +243,8 @@ export class LiveAgentSession {
                 const line = next.value;
                 if (!line.trim())
                     continue;
-                this.events.write(line + "\n");
+                if (!this.closing)
+                    this.events.write(line + "\n");
                 let msg;
                 try {
                     msg = JSON.parse(line);
@@ -266,6 +272,8 @@ export class LiveAgentSession {
         }
         finally {
             this.rejectError = undefined; // generator is done; stop routing errors here
+            this.closing = true; // discard any late write() from translate()'s async hook/mcp paths
+            await this.drainAll(); // queue empty + all callbacks confirmed before ending either stream
             // #46: AWAIT the stream flush before the generator resolves. executeScenario reads/scans/scrubs
             // events.jsonl + control-out.jsonl immediately after `drive()` returns; a fire-and-forget end()
             // races the final buffered writes. end(cb) fires the callback on 'finish' (fully flushed).
@@ -344,6 +352,9 @@ export class LiveAgentSession {
         if (req.kind !== r.kind)
             warn(`::warning:: decider returned kind "${r.kind}" for a "${req.kind}" request (id ${decisionId}) → sending a safe deny/cancel; the agent did NOT receive an answer\n`);
         this.write(serializeDecision(req, r));
+        // Invariant: each decision id is answered at most once. Delete after a successful write so
+        // stale entries don't accumulate (live sessions may process thousands of decisions per run).
+        this.reqById.delete(decisionId);
     }
     close() {
         try {
@@ -355,26 +366,99 @@ export class LiveAgentSession {
     }
     write(obj) {
         const line = JSON.stringify(obj);
-        // The control protocol writes small single-line JSON frames, so stdin backpressure effectively never
-        // engages; we ignore the write() return / drain here. A frame past the safe threshold is anomalous —
-        // hard-FAIL rather than risk a partially-buffered write that silently corrupts the protocol stream.
-        // (If large control frames ever become legitimate, switch to a drain-aware queue, making writes async.)
-        if (line.length > 256 * 1024)
-            throw new Error(`control frame is ${line.length} bytes (> 256 KiB safe limit) — refusing to write to avoid partial stdin buffering; this indicates an unexpectedly large control payload`);
+        if (this.closing)
+            return; // session draining — discard silently
         // Bug 34: check stream writability before writing — a closed/destroyed stdin after a child crash
         // loses decision frames silently. Throw immediately so callers surface the failure rather than
         // hanging or silently dropping frames.
         if (this.proc.stdin.destroyed || !this.proc.stdin.writable)
             throw new Error("control-protocol write failed: child stdin is no longer writable (process may have crashed)");
-        this.controlOut.write(line + "\n");
-        // Attach a one-shot error handler on the write so stream errors surface via rejectError rather
-        // than as unhandled 'error' events (the permanent 'error' handler on constructor covers stdin
-        // errors that fire outside a write, e.g. EPIPE after a crash).
-        this.proc.stdin.write(line + "\n", (err) => {
-            if (err && this.rejectError)
+        this.writeQueue.push(line + "\n");
+        void this.pump().catch((err) => {
+            // Route unexpected pump errors through rejectError so they surface as a typed error + clean
+            // teardown instead of a silent hang. Known failure modes call rejectError themselves before
+            // the pump returns, so this only fires on unanticipated throws.
+            if (this.rejectError)
                 this.rejectError(err instanceof Error ? err : new Error(String(err)));
         });
     }
+    async pump() {
+        if (this.pumping)
+            return;
+        this.pumping = true;
+        try {
+            while (this.writeQueue.length) {
+                const frame = this.writeQueue.shift();
+                if (this.proc.stdin.destroyed || !this.proc.stdin.writable) {
+                    this.events.write(JSON.stringify({ _emu: "control_undelivered", frame }) + "\n");
+                    if (this.rejectError)
+                        this.rejectError(new Error("stdin no longer writable while draining queue"));
+                    while (this.writeQueue.length) {
+                        const remaining = this.writeQueue.shift();
+                        this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
+                    }
+                    break;
+                }
+                // Await the write callback directly — the ONLY reliable signal for child-process stdin
+                // pipes (EPIPE goes to the callback, not an 'error' event; 'drain'/'close' are unreliable).
+                // Awaiting inline also bounds in-flight depth to 1, giving deterministic replay ordering.
+                const writeErr = await new Promise((resolve) => {
+                    try {
+                        this.proc.stdin.write(frame, (err) => {
+                            resolve(err ?? null);
+                        });
+                    }
+                    catch (e) {
+                        resolve(e instanceof Error ? e : new Error(String(e)));
+                    }
+                });
+                if (writeErr) {
+                    this.events.write(JSON.stringify({ _emu: "control_undelivered", frame }) + "\n");
+                    if (this.rejectError)
+                        this.rejectError(writeErr);
+                    while (this.writeQueue.length) {
+                        const remaining = this.writeQueue.shift();
+                        this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
+                    }
+                    break;
+                }
+                else {
+                    if (frame.length > 256 * 1024) {
+                        // Frame too large to mirror verbatim — record a truncation marker instead.
+                        this.controlOut.write(JSON.stringify({ _emu: "control_out_truncated", bytes: frame.length, head: frame.slice(0, 4096) }) + "\n");
+                    }
+                    else {
+                        this.controlOut.write(frame);
+                    }
+                }
+            }
+        }
+        finally {
+            this.pumping = false;
+            // Unconditionally drain any remaining frames — handles a synchronous throw from
+            // controlOut.write/events.write inside the try body, which would otherwise leave
+            // writeQueue non-empty and queueIdle unfired, hanging drainAll().
+            while (this.writeQueue.length) {
+                const remaining = this.writeQueue.shift();
+                try {
+                    this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
+                }
+                catch {
+                    /* events may already be ended; discard silently */
+                }
+            }
+            const idle = this.queueIdle;
+            this.queueIdle = undefined;
+            idle?.();
+        }
+    }
+    drainAll() {
+        if (!this.pumping && this.writeQueue.length === 0)
+            return Promise.resolve();
+        return new Promise((resolve) => {
+            this.queueIdle = resolve;
+        });
+    }
 }
 /** Pure translation of one parsed stream-json message → AgentEvents (no side-effects). Shared by
  *  LiveAgentSession and CassetteAgentSession. mcp_message is handled by Live before this is called. */
@@ -396,8 +480,13 @@ export function parseMessage(msg) {
             break;
         }
         case "assistant": {
-            const parentToolUseId = msg.parent_tool_use_id ? String(msg.parent_tool_use_id) : undefined;
+            // Protocol v1: parentToolUseId is message-level. Block-level parent_tool_use_id (if present)
+            // is canonical — prefer it when both exist (block-level is more precise for nested dispatches).
+            const msgParentToolUseId = msg.parent_tool_use_id ? String(msg.parent_tool_use_id) : undefined;
+            let blockIndex = 0;
             for (const block of msg.message?.content ?? []) {
+                // Block-level parent wins over message-level when both are present (see comment above).
+                const parentToolUseId = block.parent_tool_use_id ? String(block.parent_tool_use_id) : msgParentToolUseId;
                 if (block.type === "text")
                     ev.push({ type: "assistant_text", text: block.text, parentToolUseId });
                 else if (block.type === "thinking")
@@ -422,9 +511,12 @@ export function parseMessage(msg) {
                             : Array.isArray(inp.allowedTools)
                                 ? inp.allowedTools.map(String)
                                 : []; // the `Agent` tool declares no tools list → []; declared-but-unused is legacy-`Task`-only
+                        // When block.id is absent, synthesize a fallback to avoid collapsing all anonymous
+                        // dispatches into the empty-string identity, which breaks dispatch tracking.
+                        const toolUseId = block.id ? String(block.id) : `unpaired-${blockIndex}`;
                         ev.push({
                             type: "subagent_dispatch",
-                            toolUseId: String(block.id ?? ""),
+                            toolUseId,
                             parentToolUseId,
                             // Skills often dispatch with only {description, prompt} (no subagent_type) → agentType is
                             // "unknown" but the description still identifies the dispatch (e.g. "TOP_DOWN market sizing").
@@ -434,6 +526,7 @@ export function parseMessage(msg) {
                         });
                     }
                 }
+                blockIndex++;
             }
             break;
         }
@@ -449,6 +542,7 @@ export function parseMessage(msg) {
                         isError: !!block.is_error,
                         text: toolResultText(block.content),
                         provenanceText: toolResultRaw(block.content),
+                        assertText: toolResultAssertText(block.content),
                     });
             }
             break;
@@ -472,7 +566,11 @@ function flattenToolResult(content, max) {
             if (b && typeof b === "object" && "text" in b)
                 return String(b.text);
             // Non-text blocks (json, resource, link, unknown): JSON-stringify to preserve all content.
-            return JSON.stringify(b);
+            // JSON.stringify returns `undefined` for blocks that contain functions or other
+            // non-serializable values; wrap those in an explicit marker so downstream code can identify
+            // and skip them rather than treating the literal string "undefined" as block content.
+            const serialized = JSON.stringify(b);
+            return serialized !== undefined ? serialized : JSON.stringify({ type: "unsupported", raw: "[unserializable]" });
         })
             .join(" ")
             .slice(0, max);
@@ -486,8 +584,19 @@ function toolResultText(content) {
 function toolResultRaw(content) {
     return flattenToolResult(content, 200_000);
 }
+/** Assertion-fidelity cap — enough to cover realistic tool outputs for content assertions without
+ *  blowing up result.json; deliberately larger than the 500-char display cap so text past the display
+ *  truncation is still assertable. */
+function toolResultAssertText(content) {
+    return flattenToolResult(content, 10_240);
+}
 export function toDecisionRequest(msg) {
     const sub = msg.request?.subtype;
+    // Validate request_id early — every DecisionRequest.id is a non-empty string. A missing or
+    // non-string request_id means the agent sent a malformed control_request; reject loudly rather
+    // than silently building a DecisionRequest with an unusable id (respond() would always miss it).
+    if (typeof msg.request_id !== "string" || msg.request_id === "")
+        throw new Error(`control-in: malformed request_id: ${JSON.stringify(msg.request_id)}`);
     const id = msg.request_id;
     if (sub === "can_use_tool") {
         const tool = msg.request.tool_name ?? "";