cowork-harness 0.5.0 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (42) hide show
  1. package/.claude/skills/cowork-harness/scripts/assertion-keys.json +3 -0
  2. package/.claude/skills/cowork-harness/scripts/scenario.py +3 -1
  3. package/CHANGELOG.md +133 -0
  4. package/README.md +28 -15
  5. package/dist/agent/session.js +143 -16
  6. package/dist/assert.js +198 -81
  7. package/dist/baseline.js +10 -3
  8. package/dist/cli.js +423 -159
  9. package/dist/decide/decider.js +134 -23
  10. package/dist/decide/external-channel.js +2 -2
  11. package/dist/egress/proxy.js +10 -15
  12. package/dist/egress/sidecar.js +5 -1
  13. package/dist/hostloop/workspace-handler.js +43 -5
  14. package/dist/prompt.js +12 -4
  15. package/dist/redact.js +18 -1
  16. package/dist/run/cassette.js +627 -62
  17. package/dist/run/chat.js +132 -38
  18. package/dist/run/doctor.js +5 -0
  19. package/dist/run/execute.js +87 -13
  20. package/dist/run/renderer.js +5 -0
  21. package/dist/run/run.js +20 -5
  22. package/dist/run/runs-gc.js +99 -0
  23. package/dist/run/skill-hash.js +158 -12
  24. package/dist/run/verdict.js +17 -0
  25. package/dist/runtime/hostloop.js +16 -6
  26. package/dist/runtime/microvm.js +25 -2
  27. package/dist/runtime/protocol.js +13 -4
  28. package/dist/runtime/stage.js +6 -3
  29. package/dist/scan.js +4 -1
  30. package/dist/secrets.js +43 -0
  31. package/dist/session.js +31 -3
  32. package/dist/types.js +27 -0
  33. package/docs/cassette.md +94 -0
  34. package/docs/chat.md +148 -0
  35. package/docs/cowork-spawn-contract-1.12603.1.md +1 -1
  36. package/docs/fidelity-gaps.md +81 -0
  37. package/docs/session.md +8 -1
  38. package/package.json +1 -1
  39. package/schema/cassette.v2.json +19 -0
  40. package/schema/cassette.v3.json +30 -0
  41. package/schema/run-result.json +57 -0
  42. package/schema/scenario.schema.json +12 -0
@@ -1,6 +1,7 @@
1
1
  {
2
2
  "$comment": "GENERATED from the Zod Assertion schema (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
3
3
  "keys": [
4
+ "allow_l0_plugin_divergence",
4
5
  "allow_permissive_auto_allow",
5
6
  "artifact_json",
6
7
  "dispatch_count_max",
@@ -20,6 +21,8 @@
20
21
  "subagent_tool_used",
21
22
  "tool_called",
22
23
  "tool_not_called",
24
+ "tool_result_contains",
25
+ "tool_result_not_contains",
23
26
  "transcript_contains",
24
27
  "transcript_matches",
25
28
  "transcript_no_host_path",
@@ -45,6 +45,8 @@ CONTENT_KEYS = {
45
45
  "transcript_not_contains",
46
46
  "transcript_matches",
47
47
  "transcript_not_matches",
48
+ "tool_result_contains",
49
+ "tool_result_not_contains",
48
50
  "tool_called",
49
51
  "tool_not_called",
50
52
  "subagent_tool_used",
@@ -68,7 +70,7 @@ LIVE_ONLY_KEYS = {
68
70
  }
69
71
  EGRESS_KEYS = {"egress_denied", "egress_allowed"}
70
72
  # verdict modifiers — don't verify anything themselves (e.g. suppress a default-fail)
71
- VERDICT_MODIFIER_KEYS = {"allow_permissive_auto_allow"}
73
+ VERDICT_MODIFIER_KEYS = {"allow_permissive_auto_allow", "allow_l0_plugin_divergence"}
72
74
 
73
75
  # Every key the replay-class logic knows how to handle. `replay_protocol_fidelity` is valid-but-not-authorable
74
76
  # (errored separately below). This is also the embedded fallback for ASSERT_KEYS — kept EQUAL to the generated
package/CHANGELOG.md CHANGED
@@ -6,6 +6,139 @@ All notable changes to this project are documented here. The format is based on
6
6
 
7
7
  ## [Unreleased]
8
8
 
9
+ ## [0.7.0] — 2026-06-19
10
+
11
+ ### Added
12
+
13
+ - **`chat --plugin <dir>` (repeatable)** — load additional local plugins into a `chat` session alongside
14
+ the primary skill folder. Each `--plugin <dir>` is appended to `plugins.local_plugins` so multi-plugin
15
+ interactive debugging no longer requires a custom session YAML. In `--raw` mode (native Docker), `--plugin`
16
+ flags are silently ignored with a warning: `chat --raw: --plugin flags are ignored in --raw mode`.
17
+ - **`/help` in the `chat` REPL** — typing `/help` in an interactive session now prints
18
+ `Commands: /exit /quit /help` and continues rather than forwarding the text to the model. The startup
19
+ prompt was updated from "type your message, /exit to quit" to "type your message (/help for commands)".
20
+ - **`scrubField(value, secrets)` exported from `src/secrets.ts`** — a new multi-pass field-level scrubber
21
+ that covers token appearances beyond direct substring matches:
22
+ - Pass 1: direct `scrub()` — catches literal, base64-encoded, and `encodeURIComponent`-encoded tokens.
23
+ - Pass 2: whole-field base64 decode (≥20-char pure base64 strings) — if the decoded form contains a
24
+ secret hit, returns `[REDACTED:base64]`.
25
+ - Pass 3: whole-field URI decode (values containing `%`) — if the decoded form contains a secret hit,
26
+ returns `[REDACTED:uri]`.
27
+ Applied in `cassette.ts` artifact scrubbing: base64 artifacts (`encoding === "base64"`) are replaced
28
+ wholesale with `[REDACTED:base64]`, the encoding marker is cleared (so replay decodes as UTF-8), and the
29
+ sha256 is recomputed over the marker bytes (with a `::warning::` that artifact assertions will fail at
30
+ replay). UTF-8 artifacts pass through `scrubField` safely.
31
+ - **`prompt_asset_missing` VerdictSignal** — `computeVerdict()` now pushes a `{ code: "prompt_asset_missing",
32
+ severity: "warn" }` signal when `result.fidelityWarnings` contains a "referenced asset not found" entry,
33
+ making a missing prompt asset visible in the verdict output rather than buried in the run log.
34
+ - **`onInfraError` callback in `makeWorkspaceHandler`** — an optional sixth parameter
35
+ `onInfraError?: (message: string) => void` lets callers intercept infrastructure errors
36
+ (ETIMEDOUT / killed / no code+stdout+stderr) separately from model-visible error text.
37
+ `spawnHostLoop()` wires this to append `{ type: "infra_error", ts, message }` to `events.jsonl` so
38
+ infrastructure failures are structured and queryable rather than only appearing in the model-visible
39
+ response string.
40
+
41
+ ### Fixed
42
+
43
+ - **`ExternalDecider` no longer coerces `"first"` to option 1.** `coerceLabel` gained an
44
+ `enableFirstShorthand` parameter (default `true`). External deciders (`--decider-cmd`,
45
+ `--decider-dir` helpers) now call `coerceLabel(raw, labels, false)`, so a helper script that
46
+ accidentally returns the string `"first"` must match an actual label named `"first"` — it is no
47
+ longer silently promoted to the first option. The shorthand remains active for internal (scripted)
48
+ use.
49
+ - **`flagValue` and `chat --model` reject empty strings.** `flagValue()` in `src/cli.ts` now exits `2`
50
+ with a clear message when the supplied value is blank or whitespace-only. `chat` additionally guards
51
+ the `--model` value inline after parsing, so `--model ""` and `--model $UNSET_VAR` both fail loudly
52
+ instead of passing an empty model ID to the runtime.
53
+ - **`redactCassette` skips `[REDACTED*]` marker bodies.** The per-line JSON redaction pass
54
+ (`redactJsonLine`) is now bypassed for artifact bodies that already start with `[REDACTED` — preventing
55
+ the sha256 from being corrupted by a second redaction pass over the marker string.
56
+ - **TLD list extended from 22 to 51 entries.** The domain scanner in `src/scan.ts` now recognises
57
+ major European, Asian, and Latin American ccTLDs:
58
+ `ch|nl|se|no|it|jp|br|nz|in|sg|kr|mx|es|pt|pl|be|at|dk|fi|ie|ru|cn|tw|hu|cz|ro|il|za|ar|cl|pe|tr`.
59
+ Domain findings that were previously missed on these TLDs now fire correctly.
60
+
61
+ ## [0.6.0] — 2026-06-19
62
+
63
+ ### Breaking changes
64
+
65
+ - **Exit code 3 for boundary/integrity violations.** Commands that previously exited `2` for a
66
+ boundary constraint (e.g. `skill` hitting the egress sandbox, `run` with a `boundary`-category
67
+ failure) now exit `3`. Exit `2` is narrowed to usage errors, unknown flags, and runtime errors.
68
+ Scripts that check `$? -eq 2` to detect boundary failures must be updated to `$? -eq 3`.
69
+ - **`verify-cassettes --staleness-only` / `--privacy-only` removed.** Replaced by
70
+ `--skip-staleness` (run privacy scan only) and `--skip-privacy` (run staleness scan only). The
71
+ old flags are not aliased — they now exit `2` as unknown flags.
72
+ - **`decide` with no configuration exits `2` instead of `1`.** Previously, calling `decide` with no
73
+ `--decider-*`, `--answer`, or `--answer-policy` would fall through to a `ScriptedDecider([])` and
74
+ exit `1` ("no rule matched"). It now fails early with exit `2` ("no decider configured") and a
75
+ clear message.
76
+
77
+ ### Added
78
+
79
+ - **`chat` — full flag parity for interactive debugging:**
80
+ - `[prompt]` — optional seed prompt sent as the first turn before the REPL opens.
81
+ - `--upload <file>` (repeatable) — attach a file at `mnt/uploads/<basename>`; live at session start.
82
+ - `--folder <dir>` (repeatable) — connect a project folder at `mnt/.projects/<basename>` as a live bind mount.
83
+ - `--verbose` / `-V` — show thinking blocks, tool inputs, and the sub-agent tree (previously hardcoded off).
84
+ - `--fidelity protocol` — no-Docker fastest tier; accepted alongside `container` and `hostloop`.
85
+ - `--model` in `--raw` mode — previously silently dropped; now passed as `--model <id>` to the docker argv.
86
+ - Idle heartbeat wired in all three fidelity branches (protocol / hostloop / container).
87
+ - Run ID printed at session start (before first turn) so a mid-session crash still tells you where the transcript is.
88
+ - **`assertions` command** — canonical rename of `assert`; `assert` is kept as a deprecated alias
89
+ that prints a migration notice. `assertions --list` is the new canonical form.
90
+ - **`scaffold <run-id>` positional** — canonical form; `--from-run <id>` is kept as a deprecated
91
+ alias that prints a migration notice.
92
+ - **`trace --view tools|questions|dispatches`** — replaces the three separate `--tools` / `--gates`
93
+ / `--dispatches` flags with a single `--view` enum. Legacy flags are kept as backward-compat
94
+ aliases (`--gates` maps to `--view questions`).
95
+ - **Env-var defaults for all live commands:**
96
+ - `COWORK_HARNESS_FIDELITY` — default fidelity tier for `skill` and `chat` (validated; exits 2 on an invalid value).
97
+ - `COWORK_HARNESS_MODEL` — default model override for `skill` and `chat`.
98
+ - `COWORK_HARNESS_OUTPUT_FORMAT` — default `--output-format` for all commands (`text` or `json`).
99
+ - **`decide` no-decider guard** — calling `decide` with no configuration fails immediately with a
100
+ clear message and exit `2` instead of falling through to a vacuous "no rule matched" exit `1`.
101
+ - **`vm` per-subcommand `--help`** — `vm <sub> --help` prints the subcommand usage and exits `0`.
102
+ - **`--quiet` / `-q` accepted in `decide`** — no-op flag for flag-surface consistency with `skill` / `run`.
103
+
104
+ ### Fixed
105
+
106
+ - **`--rerecord-stale` now prefers on-disk scenario over embedded snapshot (G-1).** When a
107
+ `scenarios/<name>.yaml` exists alongside the cassette dir, `--rerecord-stale` re-records from it
108
+ instead of the embedded copy. Edits to the scenario (e.g. adding `skills:` for staleness scoping)
109
+ now take effect. Falls back to the embedded snapshot when no on-disk file is found, with a clear
110
+ warning.
111
+ - **Staleness message distinguishes format-version bump from real content change (G-2).** After a
112
+ harness upgrade that changes the hash algorithm, `verify-cassettes` now reports
113
+ `recorded under an older hash format (vN → vM)` instead of the misleading
114
+ `local skill/plugin dir contents changed`.
115
+ - **`.cowork-hashignore` leading-slash patterns now correctly anchor to the mount root (G-3).**
116
+ `/tests` previously compiled to a regex that never matched (no leading slash in relative paths);
117
+ it now matches only the top-level `tests/` dir, as expected.
118
+ - **Scoped staleness findings now name the changed bucket (G-4).** When a cassette was recorded with
119
+ `skills: [<name>]` scoping, `verify-cassettes` now reports `skills/<name> changed` or
120
+ `shared root changed` rather than the generic `local skill/plugin dir contents changed`.
121
+ - **Unknown flags in `chat` now exit `2`.** Previously ignored silently; any unrecognised flag now
122
+ exits `2` with a clear message.
123
+ - **`chat --model` bounds-checked.** If `--model` is the last argument (no value following), the
124
+ command now exits `2` with a clear message instead of silently using `undefined`.
125
+ - **`cmdSync` platform guard fires before argument parsing.** On non-macOS platforms the guard now
126
+ exits before `parseArgs`, so `sync --help` on Linux no longer crashes on a missing flag.
127
+ - **`record` conflict check.** Passing both a positional and `--cassette` now exits `2` immediately
128
+ instead of silently preferring one.
129
+ - **L0 plugin divergence signal.** `spawnProtocol` now reports whether the skill resolved at L0
130
+ (no container) vs. L1, so callers can surface the divergence accurately.
131
+ - **System-prompt threading.** `systemPromptAppend` is now passed through all runtime paths that
132
+ previously dropped it.
133
+ - **Strict agent binary resolution.** When the exact staged binary path is missing, the harness now
134
+ fails with a clear message pointing to `COWORK_HARNESS_ALLOW_AGENT_FALLBACK=1` instead of
135
+ silently falling back to an arbitrary sibling version.
136
+ - **Path traversal guard on plugin sources.** Plugin source paths are now validated to be
137
+ directories (not files or traversal strings) before mounting.
138
+ - **Cassette correctness (batch):** privacy/staleness coverage tracking, redaction verdict
139
+ preservation across re-records, base64-encoded artifact body scanning, and exhaustiveness
140
+ checks on the replay assertion set.
141
+
9
142
  ## [0.5.0] — 2026-06-18
10
143
 
11
144
  ### Added
package/README.md CHANGED
@@ -24,6 +24,15 @@ Scriptable, CI-friendly test harness that reproduces **Claude Cowork's observabl
24
24
 
25
25
  > **What this is and isn't.** This is an *emulator of the contract*, not the Desktop runtime. Cowork's real session control plane lives behind the Desktop renderer's IPC (per-build UUID + `senderFrame` origin checks) and the app ships with remote debugging disabled (verified: `--remote-debugging-port` opens no listener; Electron `EnableNodeCliInspectArguments` fuse is OFF). So you **cannot** drive the real Apple Virtualization.framework microVM from a script. What you *can* faithfully reproduce is everything that actually changes how a **skill** behaves: the same agent binary in **cowork mode** (`CLAUDE_CODE_IS_COWORK=1` — there is no `--cowork` flag), the same mount layout, the same egress allowlist, and the same permission/question protocol. That's what this project does.
26
26
 
27
+ **Zero-friction preview — no token, no Docker.** A committed cassette replays on a fresh clone:
28
+
29
+ ```bash
30
+ npm install -g cowork-harness
31
+ cowork-harness replay examples/replays/example-pdf-skill.cassette.json
32
+ ```
33
+
34
+ Full setup → [Quick start](#quick-start).
35
+
27
36
  ---
28
37
 
29
38
  ## Why this works for skill testing
@@ -78,14 +87,15 @@ Skill testing is the headline use, but the tool is a general harness over the Co
78
87
  |---|---|---|
79
88
  | `skill <folder> "<prompt>"` | Run a local skill/plugin folder once against the staged agent | ad-hoc "is the skill alive / does it do X?" — the fast inner loop |
80
89
  | `run <scenario.yaml \| dir/>` | Run authored scenarios with `assert:` + a CI-ready exit code | you want a repeatable, **asserted regression test** |
81
- | `chat <folder>` | Interactive multi-turn REPL against a skill (TTY) | debugging a multi-turn flow by hand |
90
+ | `chat <folder> [prompt]` | Interactive multi-turn REPL against a skill (TTY); optional seed prompt is sent as the first turn. `--upload <file>` / `--folder <dir>` (repeatable) attach files/project folders; `--verbose` / `-V` shows thinking blocks + tool inputs; `--fidelity protocol\|container\|hostloop` | debugging a multi-turn flow by hand |
82
91
  | `record` / `replay` | `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--max-artifact-bytes` raises the 64 KiB inline-body cap); `replay` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`<file\|dir>`, `--cassette <file>` for the explicit single form; `--strict` fails on a stale one; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
83
- | `verify-cassettes <file\|dir>` | Token-free CI gate over committed cassettes: a privacy scan (email/currency/domain → exit 1; whole-token allows via `--allow` / class-scoped `--allow-domain` / `--allow-email` / `--allow-file`) + a staleness check (`--staleness-only`). A dir argument scans `*.cassette.json` in that dir only (**non-recursive**) | gating **committed cassettes** against PII leaks + "edited the skill, forgot to re-record" |
92
+ | `verify-cassettes <file\|dir>` | Token-free CI gate over committed cassettes: a privacy scan (email/currency/domain → exit 1; whole-token allows via `--allow` / class-scoped `--allow-domain` / `--allow-email` / `--allow-file`) + a staleness check (`--skip-privacy` or `--skip-staleness` to run only one half). A dir argument scans `*.cassette.json` in that dir only (**non-recursive**) | gating **committed cassettes** against PII leaks + "edited the skill, forgot to re-record" |
84
93
  | `verify-run <run-dir> <scenario.yaml>` | Re-evaluate a scenario's `assert:` against an already-kept run dir — **no live agent, no tokens, no Docker** (~1s) | iterating on a wrong assertion without a full live re-record |
85
- | `trace <run-id>` | Digest a run's `events.jsonl` (`--tools`, `--gates`, `--dispatches` for the sub-agent dispatch tree + total) | "how many sub-agents *actually* dispatched, and which?" |
86
- | `scaffold --from-run <id>` | Turn a kept run into a starter scenario YAML (gates→answers, artifacts→`file_exists`) | authoring a scenario from a real run instead of guessing |
94
+ | `trace <run-id>` | Digest a run's `events.jsonl` (`--view tools\|questions\|dispatches`; legacy `--tools` / `--gates` / `--dispatches` aliases still accepted) | "how many sub-agents *actually* dispatched, and which?" |
95
+ | `scaffold <run-id>` | Turn a kept run into a starter scenario YAML (gates→answers, artifacts→`file_exists`); `--from-run <id>` is a deprecated alias for the positional | authoring a scenario from a real run instead of guessing |
96
+ | `python3 …/scenario.py scaffold --name <name> --skill <dir>` | Generate a starter scenario skeleton from scratch (the `…` is `.claude/skills/cowork-harness/scripts/`; `cowork-harness lint --help` shows the resolved path) | starting a new scenario when you have no prior run |
87
97
  | `lint <scenario.yaml>…` | Check scenarios for silent false-greens — assertions placed on the wrong CI lane, mixed content/live keys, missing `controlOut`-required keys (bundled `scenario.py`; needs python3 + PyYAML) | before committing a new scenario or after changing assertions |
88
- | `assert --list` | List the available scenario assertions (generated from the schema) | "what can I assert?" without grepping the source |
98
+ | `assertions --list` | List the available scenario assertions (generated from the schema); `assert` is a deprecated alias | "what can I assert?" without grepping the source |
89
99
  | `decide` | Validate a decider against a sample question in ~2 s (no run) | sanity-check a `--decider-*` / `--answer` wiring before a long run |
90
100
  | `gates` / `answer` | Stream / answer in-band gates for `--decider-dir` | a **driving agent** answers live questions via a Monitor |
91
101
  | `boundary-check [baseline] [--session <file>]` | Prove the sandbox enforces Cowork's limitations; `--session` folds a session's `egress.extra_allow` into the probe allowlist | verifying the harness's own fidelity |
@@ -119,7 +129,7 @@ cowork-harness skill ~/my-plugin 'review this deck' --upload deck.pdf # att
119
129
  cowork-harness skill ~/my-plugin "..." --session-id s1 # pin a session…
120
130
  cowork-harness skill ~/my-plugin '<next turn>' --session-id s1 --resume # …then resume it (gated/checkpoint skills)
121
131
  cowork-harness skill ~/my-plugin "..." --keep # then: trace the run
122
- cowork-harness trace <run-id> --tools # tool calls + sub-agent dispatches from events.jsonl
132
+ cowork-harness trace <run-id> --view tools # tool calls + sub-agent dispatches from events.jsonl
123
133
  cowork-harness skill --help # full per-command flag reference
124
134
 
125
135
  cowork-harness chat ~/my-plugin # interactive multi-turn REPL (full harness: egress sandbox + control protocol)
@@ -132,8 +142,8 @@ cowork-harness chat ~/my-plugin # interactive multi-turn REPL (
132
142
  `run`/CI), `prompt` (ask at the TTY — the default for `skill` when interactive), or `first` (pick
133
143
  option 1, loudly warn). Pick with `--on-unanswered`; left unset, `skill` is **adaptive** (`prompt` on
134
144
  a TTY, `fail` when piped/CI) and `run` is always `fail`. Exit codes: `0`
135
- pass · `1` assertion/agent failure · `2` usage /
136
- unanswered-under-`fail` / boundary / runtime. After a run, the footer **echoes every auto-answered
145
+ pass · `1` assertion/agent failure · `2` usage / unanswered-under-`fail` / runtime · `3` boundary/integrity.
146
+ After a run, the footer **echoes every auto-answered
137
147
  question as a copy-pasteable `--answer "<q>=<choice>"` line** — run once exploratorily, then paste them
138
148
  back to lock in a deterministic re-run.
139
149
 
@@ -194,9 +204,11 @@ The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it
194
204
 
195
205
  It also follows the open [Agent Skills](https://github.com/vercel-labs/skills) spec, so it installs cross-editor (Cursor, Codex, OpenCode, …) by pointing the `npx skills` CLI at `.claude/skills/cowork-harness` in this repo. (Working *inside* this repo, the skill auto-loads as a project skill — no install needed.)
196
206
 
207
+ > Note: `npm install -g cowork-harness` ships `scenario.py` and assertion keys (enough for `lint` in CI) but not the full SKILL.md teaching document; install via the marketplace command above to get the teaching skill.
208
+
197
209
  **Prerequisites for anything above `protocol` fidelity** (the `protocol` tier skips items 1–2 — no Docker, no staged agent — but still calls a real model via the host `claude`, so it needs item 3, the auth token; only a committed-cassette `replay` needs nothing at all):
198
210
  1. **Claude Desktop, opened once.** The Cowork agent binary is **bind-mounted from your own install** at run time — nothing Anthropic-owned is bundled. Open Cowork once so the agent ELF is staged (`…/claude-code-vm/<ver>/claude`); the harness auto-detects it, or set `COWORK_AGENT_BINARY=<path>` to point at it. Without a staged agent, container/cowork runs fail with "Open Cowork once to stage it…".
199
- 2. **Docker (arm64)** + the agent image: `docker build --platform linux/arm64 -t cowork-agent-base:1 -f docker/Dockerfile.agent .` (override the tag with `COWORK_AGENT_IMAGE`). The `-f docker/Dockerfile.agent .` paths are **repo-relative** — run it from a source checkout, not a global `npm install -g` (where the Dockerfile lives under the package dir).
211
+ 2. **Docker (arm64)** + the agent image: `docker build --platform linux/arm64 -t cowork-agent-base:1 -f docker/Dockerfile.agent .` (override the tag with `COWORK_AGENT_IMAGE`). The `-f docker/Dockerfile.agent .` paths are **repo-relative** — run it from a source checkout, not a global `npm install -g` (where the Dockerfile lives under the package dir). (Global install? Run `cowork-harness doctor --tier container` — it prints the exact `docker build` command with the correct package-local Dockerfile path.)
200
212
  3. **An auth token** — either `export CLAUDE_CODE_OAUTH_TOKEN=$(claude setup-token)` or a **`.env`** file (copy `.env.example` → `.env`; gitignored). The token resolves in priority order: exported env > `--dotenv <path>` > `./.env` (cwd) > `<install>/.env` (the package root), so a `npm link`ed install works from any directory. Keep `.env` at a working-dir or install root, never inside a mounted skill/project folder. (Use `--dotenv`, not `--env-file` — Node reserves the latter.)
201
213
 
202
214
  > `sync` (below) is **optional for a first run** — the repo ships `baselines/desktop-*.json`, so `baseline: latest` already resolves. Run `sync` only to refresh the platform baseline after Claude Desktop updates. (`sync` is **macOS-only** today; on Linux/Windows use the committed baselines — they work cross-platform.)
@@ -221,11 +233,11 @@ cowork-harness run examples/scenarios/ --output-format json
221
233
  cowork-harness record examples/scenarios/example-pdf-skill.yaml --out examples/replays/example-pdf-skill.cassette.json
222
234
  cowork-harness replay --cassette examples/replays/example-pdf-skill.cassette.json
223
235
 
224
- # A committed synthetic fixture is ready to replay on a fresh clone (no record step needed):
225
- cowork-harness replay --cassette examples/replays/example-pdf-skill.cassette.json
226
-
227
236
  # Cassettes are COMMITTED fixtures — record against synthetic data, and gate them in CI:
228
237
  cowork-harness verify-cassettes examples/replays/ # privacy scan (email/currency/domain) + staleness; exit 1 on a finding
238
+
239
+ # 5. Lint scenarios before committing (catches silent false-greens in assertion placement)
240
+ cowork-harness lint scenarios/*.yaml
229
241
  ```
230
242
 
231
243
  > **Privacy:** a cassette snapshots the transcript and the `outputs/` JSON bodies, so it's committed PII
@@ -239,7 +251,8 @@ cowork-harness verify-cassettes examples/replays/ # privacy scan (email/curren
239
251
  > orchestration from both, re-evaluates the **content** assertions, and re-exercises
240
252
  > `serializeDecision` as a token-free O7 guard (the AskUserQuestion `{questions,answers}` answer-shape
241
253
  > invariant). Evaluated on replay: `transcript_*`, `tool_*`, `subagent_*`, `dispatch_count_max`,
242
- > `result`. **`question_asked`, `questions_count_max`, and `gate_answers_delivered` are also evaluated —
254
+ > `result`, `allow_permissive_auto_allow` (always-on Cowork-parity scan). **`question_asked`,
255
+ > `questions_count_max`, and `gate_answers_delivered` are also evaluated —
243
256
  > but only when the cassette carries `controlOut` (full-fidelity)**; old cassettes without it get a
244
257
  > loud warning and those three keys are excluded (not vacuously passed). **Filesystem assertions
245
258
  > (`file_exists`, `user_visible_artifact`, `artifact_json`) are evaluated when the cassette carries an
@@ -296,7 +309,7 @@ assert:
296
309
  ```
297
310
 
298
311
  ```yaml
299
- # sessions/default.yaml (abridged — see the file for every field)
312
+ # examples/sessions/default.yaml (abridged — see the file for every field)
300
313
  # Relative paths below resolve from THIS file's dir (absolute and ~ are used as-is).
301
314
  model: claude-opus-4-8
302
315
  effort: high
@@ -522,7 +535,7 @@ This repo is built to be driven by agents, not just read by humans:
522
535
  - ✅ **Three isolation tiers (L0/L1/L2) + two loop-mode overlays** — `protocol` (L0 control loop), `container` (L1 sandboxed arm64 + per-run default-deny egress sidecar), `microvm` (L2 real Apple-VZ Linux microVM + guest firewall); plus the loop-mode overlays `hostloop` (production split-execution: agent loop on host, shell/web via the workspace SDK-MCP server) and `cowork` (auto-picks host-loop vs container the way Cowork does — gate `1143815894`). Egress enforced at container/microvm/hostloop; `boundary-check` reports **ALL CONSTRAINTS ENFORCED**.
523
536
  - ✅ **Three-seam driver** — `AgentSession` (control protocol) → `Decider` (scripted + `on_unanswered` policy, no silent false-greens) → `Run` (turn loop, sub-agent dispatch tree, `RunRecord`). Multi-turn `chat`, deterministic cassette `record`/`replay` (no token), `run.jsonl`/`trace.json` logging with secret-scrub.
524
537
  - ✅ **Answering live questions, every way you'd need** — `--decider-llm --intent "<one line>"` (a small model picks per question, steered by your test intent — the ergonomic default; non-determinism is flagged so a green isn't mistaken for a scripted pass), `--answer-policy <yaml>`/`--answer "rx=c"` (declarative regex→label, deterministic CI), `--decider-cmd '<helper>'` (custom logic — the Python `serve_decider(fn)` adapter pre-builds the wire loop so the helper writes only the decision), and **`--decider-dir <dir>`** (the *driving agent* answers each gate **in-band** with full context — it arms a Monitor on `cowork-harness gates <dir> --follow` and replies with `cowork-harness answer <dir> --gate <N> --choose <label>`; the session-under-test stays live, no resume/re-worded question — binary-verified). Every channel keeps stdout free, so all compose with `--output-format json`. A question is **never silently answered with option 1** — unhandled fails loud. Validate any decider in ~2s with `cowork-harness decide`.
525
- - ✅ **Sub-agent aggregation + `trace`** — recognizes the real `Agent` dispatch tool (binary-verified; `Task` is its alias) so `subagent_dispatched`/`dispatch_count_max` fire under `--fidelity cowork`, excluding the `TaskCreate` todo list; `cowork-harness trace <id> --tools` digests `events.jsonl` (each tool row now shows its **result status** `ok`/`error`), and `trace <id> --gates` shows the gate lifecycle (**question → injected answer → delivered result**); `result.json`/`--keep` surface the deep `mnt/outputs` deliverable path.
538
+ - ✅ **Sub-agent aggregation + `trace`** — recognizes the real `Agent` dispatch tool (binary-verified; `Task` is its alias) so `subagent_dispatched`/`dispatch_count_max` fire under `--fidelity cowork`, excluding the `TaskCreate` todo list; `cowork-harness trace <id> --view tools` digests `events.jsonl` (each tool row now shows its **result status** `ok`/`error`), and `trace <id> --view questions` shows the gate lifecycle (**question → injected answer → delivered result**); `result.json`/`--keep` surface the deep `mnt/outputs` deliverable path.
526
539
  - ✅ **Answer delivery is verified, not assumed** — an AskUserQuestion answer is injected as the binary's full tool input (`{questions, answers}`, ELF-verified), so the answer actually reaches the model; `RunResult.gateDeliveries[]` + the `gate_answers_delivered` assertion catch any delivery failure, and `RunResult.toolCounts` gives the **truthful** per-tool call count (host-routed `WebSearch` shows here, not the always-0 `usage.server_tool_use`).
527
540
  - ✅ **Binary-grounded fidelity** — cwd `/sessions/<id>`, the three-channel MCP model (`--mcp-config` honored in plain cowork mode; host/API-routed `web_fetch`; SDK-server delivery), host-loop tool partition, auth-env token-only drop, and production GrowthBook gates pinned per release.
528
541
  - ✅ The agent binary is **bind-mounted from your own install** at run time — nothing Anthropic-owned is in any image or distributed.
@@ -162,6 +162,10 @@ export class LiveAgentSession {
162
162
  rejectError;
163
163
  /** Bounded tail of the child's stderr, for the nonzero-exit error message. */
164
164
  stderrTail = "";
165
+ writeQueue = [];
166
+ pumping = false;
167
+ queueIdle;
168
+ closing = false;
165
169
  constructor(proc, outDir) {
166
170
  this.proc = proc;
167
171
  this.outDir = outDir;
@@ -180,7 +184,7 @@ export class LiveAgentSession {
180
184
  if (this.rejectError)
181
185
  this.rejectError(e);
182
186
  // else: the error fired before/after the generator — log it but don't throw
183
- else
187
+ else if (!this.closing)
184
188
  this.events.write(JSON.stringify({ _emu: "stdin_error", message: String(e) }) + "\n");
185
189
  });
186
190
  }
@@ -214,7 +218,8 @@ export class LiveAgentSession {
214
218
  // below rejects and the generator yields a typed {type:"error"} event instead of hanging on
215
219
  // stdout that will never arrive.
216
220
  this.proc.on("error", (e) => {
217
- this.events.write(JSON.stringify({ _emu: "spawn_error", message: String(e) }) + "\n");
221
+ if (!this.closing)
222
+ this.events.write(JSON.stringify({ _emu: "spawn_error", message: String(e) }) + "\n");
218
223
  if (this.rejectError)
219
224
  this.rejectError(e instanceof Error ? e : new Error(String(e)));
220
225
  });
@@ -238,7 +243,8 @@ export class LiveAgentSession {
238
243
  const line = next.value;
239
244
  if (!line.trim())
240
245
  continue;
241
- this.events.write(line + "\n");
246
+ if (!this.closing)
247
+ this.events.write(line + "\n");
242
248
  let msg;
243
249
  try {
244
250
  msg = JSON.parse(line);
@@ -266,6 +272,8 @@ export class LiveAgentSession {
266
272
  }
267
273
  finally {
268
274
  this.rejectError = undefined; // generator is done; stop routing errors here
275
+ this.closing = true; // discard any late write() from translate()'s async hook/mcp paths
276
+ await this.drainAll(); // queue empty + all callbacks confirmed before ending either stream
269
277
  // #46: AWAIT the stream flush before the generator resolves. executeScenario reads/scans/scrubs
270
278
  // events.jsonl + control-out.jsonl immediately after `drive()` returns; a fire-and-forget end()
271
279
  // races the final buffered writes. end(cb) fires the callback on 'finish' (fully flushed).
@@ -344,6 +352,9 @@ export class LiveAgentSession {
344
352
  if (req.kind !== r.kind)
345
353
  warn(`::warning:: decider returned kind "${r.kind}" for a "${req.kind}" request (id ${decisionId}) → sending a safe deny/cancel; the agent did NOT receive an answer\n`);
346
354
  this.write(serializeDecision(req, r));
355
+ // Invariant: each decision id is answered at most once. Delete after a successful write so
356
+ // stale entries don't accumulate (live sessions may process thousands of decisions per run).
357
+ this.reqById.delete(decisionId);
347
358
  }
348
359
  close() {
349
360
  try {
@@ -355,14 +366,98 @@ export class LiveAgentSession {
355
366
  }
356
367
  write(obj) {
357
368
  const line = JSON.stringify(obj);
358
- // The control protocol writes small single-line JSON frames, so stdin backpressure effectively never
359
- // engages; we ignore the write() return / drain here. A frame past the safe threshold is anomalous —
360
- // hard-FAIL rather than risk a partially-buffered write that silently corrupts the protocol stream.
361
- // (If large control frames ever become legitimate, switch to a drain-aware queue, making writes async.)
362
- if (line.length > 256 * 1024)
363
- throw new Error(`control frame is ${line.length} bytes (> 256 KiB safe limit) — refusing to write to avoid partial stdin buffering; this indicates an unexpectedly large control payload`);
364
- this.controlOut.write(line + "\n");
365
- this.proc.stdin.write(line + "\n");
369
+ if (this.closing)
370
+ return; // session draining — discard silently
371
+ // Bug 34: check stream writability before writing — a closed/destroyed stdin after a child crash
372
+ // loses decision frames silently. Throw immediately so callers surface the failure rather than
373
+ // hanging or silently dropping frames.
374
+ if (this.proc.stdin.destroyed || !this.proc.stdin.writable)
375
+ throw new Error("control-protocol write failed: child stdin is no longer writable (process may have crashed)");
376
+ this.writeQueue.push(line + "\n");
377
+ void this.pump().catch((err) => {
378
+ // Route unexpected pump errors through rejectError so they surface as a typed error + clean
379
+ // teardown instead of a silent hang. Known failure modes call rejectError themselves before
380
+ // the pump returns, so this only fires on unanticipated throws.
381
+ if (this.rejectError)
382
+ this.rejectError(err instanceof Error ? err : new Error(String(err)));
383
+ });
384
+ }
385
+ async pump() {
386
+ if (this.pumping)
387
+ return;
388
+ this.pumping = true;
389
+ try {
390
+ while (this.writeQueue.length) {
391
+ const frame = this.writeQueue.shift();
392
+ if (this.proc.stdin.destroyed || !this.proc.stdin.writable) {
393
+ this.events.write(JSON.stringify({ _emu: "control_undelivered", frame }) + "\n");
394
+ if (this.rejectError)
395
+ this.rejectError(new Error("stdin no longer writable while draining queue"));
396
+ while (this.writeQueue.length) {
397
+ const remaining = this.writeQueue.shift();
398
+ this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
399
+ }
400
+ break;
401
+ }
402
+ // Await the write callback directly — the ONLY reliable signal for child-process stdin
403
+ // pipes (EPIPE goes to the callback, not an 'error' event; 'drain'/'close' are unreliable).
404
+ // Awaiting inline also bounds in-flight depth to 1, giving deterministic replay ordering.
405
+ const writeErr = await new Promise((resolve) => {
406
+ try {
407
+ this.proc.stdin.write(frame, (err) => {
408
+ resolve(err ?? null);
409
+ });
410
+ }
411
+ catch (e) {
412
+ resolve(e instanceof Error ? e : new Error(String(e)));
413
+ }
414
+ });
415
+ if (writeErr) {
416
+ this.events.write(JSON.stringify({ _emu: "control_undelivered", frame }) + "\n");
417
+ if (this.rejectError)
418
+ this.rejectError(writeErr);
419
+ while (this.writeQueue.length) {
420
+ const remaining = this.writeQueue.shift();
421
+ this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
422
+ }
423
+ break;
424
+ }
425
+ else {
426
+ if (frame.length > 256 * 1024) {
427
+ // Frame too large to mirror verbatim — record a truncation marker instead.
428
+ this.controlOut.write(JSON.stringify({ _emu: "control_out_truncated", bytes: frame.length, head: frame.slice(0, 4096) }) + "\n");
429
+ }
430
+ else {
431
+ this.controlOut.write(frame);
432
+ }
433
+ }
434
+ }
435
+ }
436
+ finally {
437
+ this.pumping = false;
438
+ // Unconditionally drain any remaining frames — handles a synchronous throw from
439
+ // controlOut.write/events.write inside the try body, which would otherwise leave
440
+ // writeQueue non-empty and queueIdle unfired, hanging drainAll().
441
+ while (this.writeQueue.length) {
442
+ const remaining = this.writeQueue.shift();
443
+ try {
444
+ this.events.write(JSON.stringify({ _emu: "control_undelivered", frame: remaining }) + "\n");
445
+ }
446
+ catch {
447
+ /* events may already be ended; discard silently */
448
+ }
449
+ }
450
+ const idle = this.queueIdle;
451
+ this.queueIdle = undefined;
452
+ idle?.();
453
+ }
454
+ }
455
+ drainAll() {
456
+ if (!this.pumping && this.writeQueue.length === 0)
457
+ return Promise.resolve();
458
+ return new Promise((resolve) => {
459
+ this.queueIdle = resolve;
460
+ });
366
461
  }
367
462
  }
368
463
  /** Pure translation of one parsed stream-json message → AgentEvents (no side-effects). Shared by
@@ -385,8 +480,13 @@ export function parseMessage(msg) {
385
480
  break;
386
481
  }
387
482
  case "assistant": {
388
- const parentToolUseId = msg.parent_tool_use_id ? String(msg.parent_tool_use_id) : undefined;
483
+ // Protocol v1: parentToolUseId is message-level. Block-level parent_tool_use_id (if present)
484
+ // is canonical — prefer it when both exist (block-level is more precise for nested dispatches).
485
+ const msgParentToolUseId = msg.parent_tool_use_id ? String(msg.parent_tool_use_id) : undefined;
486
+ let blockIndex = 0;
389
487
  for (const block of msg.message?.content ?? []) {
488
+ // Block-level parent wins over message-level when both are present (see comment above).
489
+ const parentToolUseId = block.parent_tool_use_id ? String(block.parent_tool_use_id) : msgParentToolUseId;
390
490
  if (block.type === "text")
391
491
  ev.push({ type: "assistant_text", text: block.text, parentToolUseId });
392
492
  else if (block.type === "thinking")
@@ -411,9 +511,12 @@ export function parseMessage(msg) {
411
511
  : Array.isArray(inp.allowedTools)
412
512
  ? inp.allowedTools.map(String)
413
513
  : []; // the `Agent` tool declares no tools list → []; declared-but-unused is legacy-`Task`-only
514
+ // When block.id is absent, synthesize a fallback to avoid collapsing all anonymous
515
+ // dispatches into the empty-string identity, which breaks dispatch tracking.
516
+ const toolUseId = block.id ? String(block.id) : `unpaired-${blockIndex}`;
414
517
  ev.push({
415
518
  type: "subagent_dispatch",
416
- toolUseId: String(block.id ?? ""),
519
+ toolUseId,
417
520
  parentToolUseId,
418
521
  // Skills often dispatch with only {description, prompt} (no subagent_type) → agentType is
419
522
  // "unknown" but the description still identifies the dispatch (e.g. "TOP_DOWN market sizing").
@@ -423,6 +526,7 @@ export function parseMessage(msg) {
423
526
  });
424
527
  }
425
528
  }
529
+ blockIndex++;
426
530
  }
427
531
  break;
428
532
  }
@@ -438,6 +542,7 @@ export function parseMessage(msg) {
438
542
  isError: !!block.is_error,
439
543
  text: toolResultText(block.content),
440
544
  provenanceText: toolResultRaw(block.content),
545
+ assertText: toolResultAssertText(block.content),
441
546
  });
442
547
  }
443
548
  break;
@@ -447,15 +552,26 @@ export function parseMessage(msg) {
447
552
  }
448
553
  return ev;
449
554
  }
450
- /** Flatten a tool_result `content` (a string, or an array of `{type:"text",text}` blocks), capped at
555
+ /** Flatten a tool_result `content` (a string, or an array of content blocks), capped at
451
556
  * `max` chars. The 500-char DISPLAY value (toolResultText) keeps the recorder/trace compact; the larger
452
- * PROVENANCE value (toolResultRaw) is what seeds web_fetch provenance, so a URL past char 500 isn't lost. */
557
+ * PROVENANCE value (toolResultRaw) is what seeds web_fetch provenance, so a URL past char 500 isn't lost.
558
+ * Bug 35: preserve all content block types — text blocks use their text value; all other block types
559
+ * (json, resource, link, unknown shapes) are JSON-stringified so no content is silently dropped. */
453
560
  function flattenToolResult(content, max) {
454
561
  if (typeof content === "string")
455
562
  return content.slice(0, max);
456
563
  if (Array.isArray(content))
457
564
  return content
458
- .map((b) => (b && typeof b === "object" && "text" in b ? String(b.text) : ""))
565
+ .map((b) => {
566
+ if (b && typeof b === "object" && "text" in b)
567
+ return String(b.text);
568
+ // Non-text blocks (json, resource, link, unknown): JSON-stringify to preserve all content.
569
+ // JSON.stringify returns `undefined` for blocks that contain functions or other
570
+ // non-serializable values; wrap those in an explicit marker so downstream code can identify
571
+ // and skip them rather than treating the literal string "undefined" as block content.
572
+ const serialized = JSON.stringify(b);
573
+ return serialized !== undefined ? serialized : JSON.stringify({ type: "unsupported", raw: "[unserializable]" });
574
+ })
459
575
  .join(" ")
460
576
  .slice(0, max);
461
577
  return "";
@@ -468,8 +584,19 @@ function toolResultText(content) {
468
584
  function toolResultRaw(content) {
469
585
  return flattenToolResult(content, 200_000);
470
586
  }
587
+ /** Assertion-fidelity cap — enough to cover realistic tool outputs for content assertions without
588
+ * blowing up result.json; deliberately larger than the 500-char display cap so text past the display
589
+ * truncation is still assertable. */
590
+ function toolResultAssertText(content) {
591
+ return flattenToolResult(content, 10_240);
592
+ }
471
593
  export function toDecisionRequest(msg) {
472
594
  const sub = msg.request?.subtype;
595
+ // Validate request_id early — every DecisionRequest.id is a non-empty string. A missing or
596
+ // non-string request_id means the agent sent a malformed control_request; reject loudly rather
597
+ // than silently building a DecisionRequest with an unusable id (respond() would always miss it).
598
+ if (typeof msg.request_id !== "string" || msg.request_id === "")
599
+ throw new Error(`control-in: malformed request_id: ${JSON.stringify(msg.request_id)}`);
473
600
  const id = msg.request_id;
474
601
  if (sub === "can_use_tool") {
475
602
  const tool = msg.request.tool_name ?? "";