cowork-harness 0.18.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/cowork-harness/scripts/scenario.py +3 -3
- package/CHANGELOG.md +230 -0
- package/README.md +59 -24
- package/baselines/desktop-1.15962.1.json +220 -0
- package/baselines/desktop-1.17377.1.json +221 -0
- package/dist/agent/session.js +23 -23
- package/dist/assert.js +6 -6
- package/dist/boundary-paths.js +2 -2
- package/dist/boundary.js +1 -1
- package/dist/canary/grants.js +3 -3
- package/dist/cli.js +217 -73
- package/dist/decide/decider.js +82 -29
- package/dist/decide/external-channel.js +12 -12
- package/dist/decide/llm-transport.js +57 -16
- package/dist/egress/proxy.js +5 -5
- package/dist/egress/sidecar.js +7 -7
- package/dist/errors.js +15 -0
- package/dist/hostloop/provenance.js +2 -2
- package/dist/hostloop/workspace-handler.js +18 -18
- package/dist/io.js +30 -1
- package/dist/loop-decision.js +3 -3
- package/dist/prompt.js +2 -2
- package/dist/redact.js +5 -5
- package/dist/run/cassette.js +271 -92
- package/dist/run/chat.js +15 -14
- package/dist/run/doctor.js +1 -1
- package/dist/run/envelope.js +11 -3
- package/dist/run/execute.js +144 -89
- package/dist/run/gate-provenance.js +54 -0
- package/dist/run/renderer.js +96 -20
- package/dist/run/run-status.js +226 -0
- package/dist/run/run.js +45 -36
- package/dist/run/runs-gc.js +2 -2
- package/dist/run/skill-files.js +31 -8
- package/dist/run/skill-hash.js +14 -14
- package/dist/run/trace-view.js +40 -12
- package/dist/run/verdict.js +6 -6
- package/dist/runtime/argv.js +6 -6
- package/dist/runtime/container.js +3 -3
- package/dist/runtime/hostloop.js +11 -10
- package/dist/runtime/image-capabilities.js +1 -1
- package/dist/runtime/lima.js +5 -5
- package/dist/runtime/microvm.js +10 -10
- package/dist/runtime/protocol.js +9 -8
- package/dist/runtime/stage.js +6 -5
- package/dist/scan.js +1 -1
- package/dist/secrets.js +3 -3
- package/dist/session.js +70 -32
- package/dist/staging/resolve.js +2 -2
- package/dist/sync/cowork-sync.js +6 -6
- package/dist/types.js +6 -6
- package/docs/README.md +4 -0
- package/docs/boundary.md +6 -5
- package/docs/cassette.md +24 -4
- package/docs/chat.md +13 -2
- package/docs/cowork-spawn-contract-1.12603.1.md +2 -2
- package/docs/debugging.md +6 -0
- package/docs/decider-dir.md +1 -1
- package/docs/discovery.md +5 -2
- package/docs/fidelity-gaps.md +1 -1
- package/docs/maintenance.md +4 -3
- package/docs/run-status.md +115 -0
- package/docs/scenario.md +38 -12
- package/examples/replays/README.md +1 -1
- package/examples/replays/example-pdf-skill.cassette.json +1 -1
- package/package.json +1 -1
- package/python/README.md +3 -2
- package/schema/run-result.json +10 -1
- package/schema/scenario.schema.json +2 -2
- package/scripts/gen-schema.ts +1 -1
|
@@ -65,7 +65,7 @@ GATE_KEYS = {"question_asked", "questions_count_max", "gate_answers_delivered"}
|
|
|
65
65
|
# manifest-backed: replay-checkable when the cassette carries an `artifacts` manifest (record snapshots one);
|
|
66
66
|
# a manifest-less cassette skips them. Since the 0.3.0 artifact-manifest these are NOT always live-only.
|
|
67
67
|
MANIFEST_KEYS = {"file_exists", "user_visible_artifact", "artifact_json"}
|
|
68
|
-
# live-only: ALWAYS
|
|
68
|
+
# live-only: ALWAYS skipped on replay, with a loud warning (no filesystem, no network on the token-free lane)
|
|
69
69
|
LIVE_ONLY_KEYS = {
|
|
70
70
|
"egress_denied",
|
|
71
71
|
"egress_allowed",
|
|
@@ -328,7 +328,7 @@ def lint_doc(doc, path, raw_lines):
|
|
|
328
328
|
"replay-noop",
|
|
329
329
|
"every assertion is live-only (egress / no_delete_in_outputs / self_heal_ran / "
|
|
330
330
|
"transcript_no_host_path) or a verdict modifier (allow_*, a no-op pass) — on the "
|
|
331
|
-
"token-free `replay` lane the live-only ones are
|
|
331
|
+
"token-free `replay` lane the live-only ones are skipped (with a loud warning) and the verdict "
|
|
332
332
|
"modifiers verify nothing, so a replay PR gate would verify nothing.",
|
|
333
333
|
"Add a content assertion (result / transcript_* / tool_* / subagent_*) or a "
|
|
334
334
|
"manifest-backed one (file_exists / user_visible_artifact / artifact_json), or run this "
|
|
@@ -651,7 +651,7 @@ def build_scenario(args):
|
|
|
651
651
|
L.append(" # --- content / structure: evaluate on the token-free replay PR gate AND live ---")
|
|
652
652
|
L.extend(content_lines)
|
|
653
653
|
if live_lines:
|
|
654
|
-
L.append(" # --- filesystem / egress: LIVE-only (
|
|
654
|
+
L.append(" # --- filesystem / egress: LIVE-only (skipped on replay, with a loud warning) ---")
|
|
655
655
|
L.extend(live_lines)
|
|
656
656
|
else:
|
|
657
657
|
L.append(" # TODO add filesystem/egress checks (file_exists / user_visible_artifact /")
|
package/CHANGELOG.md
CHANGED
|
@@ -6,6 +6,236 @@ All notable changes to this project are documented here. The format is based on
|
|
|
6
6
|
|
|
7
7
|
## [Unreleased]
|
|
8
8
|
|
|
9
|
+
## [0.20.0] — 2026-07-01
|
|
10
|
+
|
|
11
|
+
### Added
|
|
12
|
+
|
|
13
|
+
- **`status.json` + `cowork-harness status <dir> [--follow]`.** Every run now writes a lightweight
|
|
14
|
+
`status.json` into its output directory from the moment `outDir` is created through completion
|
|
15
|
+
(`running` → `done`/`error`), with live tool/sub-agent counts. Two layers keep it from ever getting
|
|
16
|
+
stuck reporting a dead run as `"running"`: an exit-handler crash-safety net for an uncaught
|
|
17
|
+
throw/`SIGTERM`, and `updatedAt`-based staleness detection (both in `status` and `status --follow`) for
|
|
18
|
+
a hard `SIGKILL`/OOM-kill, which no exit handler can catch. `cowork-harness status <run-id | run-dir>`
|
|
19
|
+
reads it (one-shot, or `--follow` streaming one JSON line per change, bounded by a fail-loud
|
|
20
|
+
timeout/staleness check rather than a silent hang) so a script or driving agent can check whether a
|
|
21
|
+
background run is still alive WITHOUT `ps aux` — which only sees processes in the checker's own PID
|
|
22
|
+
namespace and is unreliable from inside a sandbox/container. The harness prints `[status] <outDir>` to
|
|
23
|
+
stderr as soon as it's known, so a caller doesn't need `--session-id` to discover the directory. See
|
|
24
|
+
`docs/run-status.md`.
|
|
25
|
+
- **Gate provenance in run output.** `result.json` now carries a `gateProvenance` block (`total`,
|
|
26
|
+
`bySource` histogram, per-gate `{question, answeredBy, answer, model?}`) recording how each
|
|
27
|
+
AskUserQuestion gate was answered (scripted / decided(llm|external) / first-option / prompt). The
|
|
28
|
+
verdict footer prints a counts-only one-liner (e.g. `gates: 3 · 2 decided(llm), 1 scripted`) and
|
|
29
|
+
`trace --view questions` annotates each gate with its `by`/`model`. Informational — it never changes
|
|
30
|
+
the verdict; it makes the residual non-determinism legible so a reviewer sees which assertions sit
|
|
31
|
+
downstream of a decided (non-reproducible) gate. `bySource` keys are the raw decision sources, so e.g.
|
|
32
|
+
a replay-lane decision reads `replay`; the block itself is a live/partial-lane surface and is absent on
|
|
33
|
+
the replay lane (which reports reproducibility via `nonDeterministic: false`).
|
|
34
|
+
- **`--compact` and `--demo` for shareable output** (`skill`/`run` — `chat` has its own flag parser and
|
|
35
|
+
isn't wired to either yet). `--compact` drops the
|
|
36
|
+
informational `[capability]` `::notice::` lines (the pre-flight, the "image omits…", and the "not used"
|
|
37
|
+
notes) — but the capability probe still runs and a real false-negative still **hard-fails**, unlike
|
|
38
|
+
`COWORK_SKIP_CAPABILITY_PROBE=1` which disables the safety net. `--demo` is the shareable preset:
|
|
39
|
+
`--compact` plus suppression of the `runs →` location header. Runs stay in the durable default location
|
|
40
|
+
(no temp redirect), so `scaffold`/`trace`/`inspect <run-id>` still resolve the run afterward; combined
|
|
41
|
+
with the `$HOME`→`~` collapse, demo output carries no host paths. Under `--compact`, `-V` tool inputs
|
|
42
|
+
also collapse the ephemeral cowork session root (`/sessions/<id>/mnt/` → `mnt/`) — display-only, so the
|
|
43
|
+
long in-container paths don't clutter shareable verbose output (`run.jsonl` keeps the true paths; the
|
|
44
|
+
L0/`protocol` tier uses host `work/` paths and is unaffected).
|
|
45
|
+
- **`replay --assert-from <scenario.yaml>` / `--reassert` — token-free re-check against on-disk assertions.**
|
|
46
|
+
By default `replay` still evaluates the assertions **frozen in the cassette** (byte-deterministic, ignores the
|
|
47
|
+
working tree); a plain `replay` now prints a `::notice::` when a sibling scenario's `assert:` differs, instead
|
|
48
|
+
of silently using the frozen copy. The new flags opt into re-evaluating against the **on-disk** `assert:`
|
|
49
|
+
(+`expect_denied:`) — the "edit the assert, re-check without a paid re-record" loop. `--assert-from <file>`
|
|
50
|
+
takes an explicit sibling scenario; `--reassert` auto-discovers it (persisted `scenarioSource`, else a
|
|
51
|
+
name lookup) — no argument needed. The opt-in path is safe by
|
|
52
|
+
construction: it **hard-fails** on recording-shaping drift (`prompt`/`baseline`/`fidelity`/`answers`/`skills`/
|
|
53
|
+
`requires_capabilities`) and on skill-content staleness (it implies `--fail-on-skill-drift`, when a skill
|
|
54
|
+
fingerprint was recorded), warns on on-disk assert keys that can't be evaluated on replay (filesystem/gate/egress)
|
|
55
|
+
and on an edited `expect_denied`, and notes that the `session` (model/mounts/discovery) is **not** verified.
|
|
56
|
+
- **Per-result `verdict` in the `--output-format json` envelope.** Each entry in `results[]` now carries
|
|
57
|
+
`verdict: { pass, exitCode, signals[], guards[] }` (a non-mutating projection of `computeVerdict`), so a consumer
|
|
58
|
+
can read each result's pass/fail **and why** (e.g. an all-green-assertions run that is `pass:false` purely on a
|
|
59
|
+
`stalled` signal) without recomputing. The top-level `ok` is derived from the same per-result verdicts.
|
|
60
|
+
- **`chat` / `skill` / `run --verbose` live-output legibility pass.** Six small, no-new-dependency
|
|
61
|
+
improvements to the stderr renderer (`src/run/renderer.ts`) and `PromptDecider`'s TTY gate prompts
|
|
62
|
+
(`src/decide/decider.ts`) — informational-only, nothing here touches verdicts, `result.json`, or replay.
|
|
63
|
+
Tool call markers are now category-specific glyphs instead of a uniform `·`: `@` read (Read/Glob/Grep/…),
|
|
64
|
+
`#` mutate (Write/Edit/…), `!` shell (Bash/…), `?` network (WebFetch/…). Truncated `-V` tool-input
|
|
65
|
+
summaries now show how much was cut (`… [+N chars]` instead of a bare `…`). Each turn now ends with a
|
|
66
|
+
`── +N.Ns ──` separator carrying that turn's elapsed time (derived from the SDK's per-turn `result`
|
|
67
|
+
event, which the renderer previously dropped entirely). `tool_result` events — likewise previously
|
|
68
|
+
dropped — now render a one-line `→ …` / `✗ …` outcome under each top-level tool call. Nested sub-agent
|
|
69
|
+
dispatch lines (`--verbose`) now indent proportionally to dispatch depth instead of always rendering
|
|
70
|
+
flat. And permission / `AskUserQuestion` TTY prompts now render inside a `┌─/│/└─` box so they visually
|
|
71
|
+
stand out from the progress markers sharing the same stderr stream.
|
|
72
|
+
|
|
73
|
+
### Changed
|
|
74
|
+
|
|
75
|
+
- **`run` now accepts `--keep` as an explicit no-op** instead of erroring. `--keep` is meaningful on
|
|
76
|
+
`skill` (which otherwise discards runs); `run` always keeps runs, so passing `--keep` (muscle memory
|
|
77
|
+
from `skill`) prints a one-line note that it had no effect rather than the loud "unexpected argument"
|
|
78
|
+
reject. Exact-token only — a genuinely unexpected flag still rejects loudly.
|
|
79
|
+
- **The default `--decider-llm` answering model now floats to the latest Sonnet** (the CLI alias `sonnet`)
|
|
80
|
+
instead of the id pinned in 0.19.0 (`claude-sonnet-4-5`), so the default keeps tracking Anthropic's
|
|
81
|
+
current Sonnet without a repo edit. `gateProvenance`/`result.json`'s `decisions[].model` is unaffected —
|
|
82
|
+
it now records the CONCRETE model the alias resolved to for that run (via `claude -p --output-format
|
|
83
|
+
json`'s `modelUsage`), never the literal string `"sonnet"`, so per-gate auditability is exactly as precise
|
|
84
|
+
as it was under the old pinned default; an envelope that doesn't resolve to exactly one concrete model
|
|
85
|
+
fails loud rather than recording an empty/ambiguous value. `--decider-model <id>` /
|
|
86
|
+
`COWORK_HARNESS_DECIDER_MODEL` still pin an exact id — the way to get byte-for-byte reproducible decider
|
|
87
|
+
behavior across runs (as much as a stochastic model allows), since the floating default can answer
|
|
88
|
+
differently over time as Anthropic ships new Sonnet releases.
|
|
89
|
+
|
|
90
|
+
### Fixed
|
|
91
|
+
|
|
92
|
+
- **A blocking `--on-unanswered prompt` wait now announces itself immediately.** When `skill` blocks at
|
|
93
|
+
the TTY for an unscripted question (the adaptive default when a human is attached), it prints a one-time
|
|
94
|
+
`::notice:: [input] waiting for an answer…` the instant it blocks — instead of only the ~30 s heartbeat —
|
|
95
|
+
so a recording/wrapper/automation context isn't left silently hung. The notice is per-run (a fresh
|
|
96
|
+
decider per scenario, so a `run dir/` batch announces each blocking scenario), and only for the real TTY
|
|
97
|
+
asker (the `chat` REPL's own prompt is left alone). For non-interactive use, `--on-unanswered fail`
|
|
98
|
+
remains the way to never block.
|
|
99
|
+
- **Human output no longer prints absolute `$HOME` paths.** The `runs →` location line, the `--keep`
|
|
100
|
+
run-dir/outputs lines, the `scaffold` tip, the failure `→ full run:` line, and the failure branch's
|
|
101
|
+
own `→ outputs:` line now collapse a leading `$HOME` to `~`, so a screenshot / pasted log / bug report
|
|
102
|
+
doesn't leak your username and filesystem layout. Display-only (`~` re-expands in a shell); set
|
|
103
|
+
`COWORK_HARNESS_RUNS_DIR` for full neutralization.
|
|
104
|
+
- **A plugin/skill mounted from an untracked git working copy no longer fails silently.** Staging delivers
|
|
105
|
+
the git-**tracked** set (the fidelity boundary — real Cowork installs from a repo and sees only committed
|
|
106
|
+
files), but an all-untracked source used to mount **EMPTY** with no signal: the agent reported "the skill
|
|
107
|
+
isn't installed" and did the work itself — a green-looking run where the skill never loaded. Now the filter
|
|
108
|
+
is **visible in both directions**: a would-be-empty plugin/skill mount **hard-fails** with a `BoundaryError`
|
|
109
|
+
(clean exit 3) naming the dir and the fix (`git add`, or `COWORK_HARNESS_GITSET=0`), and a partially-tracked
|
|
110
|
+
source emits a loud `::notice:: [stage]` listing the excluded untracked files. The staged-set count and the
|
|
111
|
+
delivered set now come from one `git ls-files` snapshot (no TOCTOU). The guard is correctly skipped on
|
|
112
|
+
`--resume` (which re-stages nothing) — which also fixes a latent resume false-fail where a since-removed
|
|
113
|
+
skill source would throw. The sibling symlink-escape staging errors are now `BoundaryError`s too (clean
|
|
114
|
+
exit 3 instead of a stack trace).
|
|
115
|
+
- **`trace --view questions` no longer misattributes `by`/`model` after a denied gate.** It paired each
|
|
116
|
+
gate row with `summarizeGateProvenance(...).gates[i]` by array index — but that array **drops**
|
|
117
|
+
denied/mismatched gates (`mismatch→deny`), while the trace rows include every gate asked. One denied
|
|
118
|
+
gate in the middle of a run shifted every later row's `by`/`model` onto the wrong question, and the true
|
|
119
|
+
owner of that data got none. Now pairs against every question-kind decision (answered **or** denied,
|
|
120
|
+
interleaved tool-permission decisions excluded), which keeps the common case aligned; a denied gate is
|
|
121
|
+
correctly left unannotated instead of stealing the next answered gate's provenance. Informational display
|
|
122
|
+
only — never affected pass/fail.
|
|
123
|
+
- **CLI `--help` drift.** The top-level `chat` summary now lists `protocol` (the command already accepted
|
|
124
|
+
it); `--version` documents its `-v` alias; and the `gates` / `answer` / `scaffold` usage strings now show
|
|
125
|
+
the `--output-format` flag they already parse.
|
|
126
|
+
- **`sync --diff` no longer goes silent on a genuine Desktop version bump.** It previously diffed `next`
|
|
127
|
+
against `baselines/desktop-<NEW version>.json` — which doesn't exist yet on a real bump — so it always
|
|
128
|
+
printed `(no committed baseline yet)` instead of the `appVersion`/`agentVersion`/etc. field diff
|
|
129
|
+
`docs/maintenance.md` documents. It now diffs against `base` (the latest committed baseline `next` was
|
|
130
|
+
actually merged onto), which is the previous version on a bump and the exact same content on a
|
|
131
|
+
same-version re-sync. The diff header now names which baseline it's comparing against. `docs/
|
|
132
|
+
maintenance.md`'s example output and noise callout (`$comment` also moves alongside `capturedAt` on every
|
|
133
|
+
run) updated to match.
|
|
134
|
+
|
|
135
|
+
### Documentation
|
|
136
|
+
|
|
137
|
+
- **Doc-vs-code audit — corrected several doc claims that diverged from the implementation.**
|
|
138
|
+
- **Host-loop tier wording.** README, `docs/boundary.md`, `docs/chat.md`, and the skill `SKILL.md` said
|
|
139
|
+
the `hostloop`/`cowork` "agent loop runs host-side." It does not: the agent process runs **in the
|
|
140
|
+
container** like `container`, but native Bash/WebFetch are disabled and routed host-side via the
|
|
141
|
+
workspace SDK-MCP server (bash via `docker exec`, `web_fetch` via host `curl`). Only `protocol` runs
|
|
142
|
+
the agent on the host. Reworded to describe the **tool-routing** split, not an agent-loop split, in
|
|
143
|
+
README, `docs/boundary.md`, `docs/chat.md`, and `SKILL.md` — plus, in a 2026-07-01 follow-up, the three
|
|
144
|
+
files this pass missed: `docs/scenario.md`, `docs/fidelity-gaps.md`, and the skill's
|
|
145
|
+
`fidelity-and-answers.md` (and the misleading code comment in `src/runtime/hostloop.ts`).
|
|
146
|
+
- **Artifact replay.** The skill references (`scenario-schema.md`, `ci-recipe.md`) claimed a replay PR
|
|
147
|
+
gate "cannot verify an artifact's content." It can — `file_exists` / `user_visible_artifact` /
|
|
148
|
+
`artifact_json` evaluate on replay **when the cassette carries an `artifacts` manifest** (already
|
|
149
|
+
correct in `docs/cassette.md`). Fixed the two contradicting copies.
|
|
150
|
+
- **`boundary-check` scope.** Clarified it probes the **L1 Docker** path only (covers `container` and
|
|
151
|
+
`hostloop`, which share that sandbox); the `microvm` guest-iptables firewall is not exercised by it.
|
|
152
|
+
- **`microvm` isolation.** `docs/scenario.md` said microvm "shares the container sandbox"; it actually
|
|
153
|
+
enforces the same allowlist inside a real Lima/Apple-VZ VM via a guest firewall.
|
|
154
|
+
- **Egress mechanism.** `docs/boundary.md` cited `docker/compose.yml` as the live enforcer; the runtime
|
|
155
|
+
creates **per-run** Docker networks in `src/egress/sidecar.ts` and never invokes compose (now marked
|
|
156
|
+
reference-only).
|
|
157
|
+
- **Onboarding / DX.** Added a "Which path am I on?" box (replay / protocol / live tiers / invocation)
|
|
158
|
+
and a three-names note to the README quick start; surfaced `doctor` / `python3` / `vm init` in the
|
|
159
|
+
docs reading order; aligned `docs/discovery.md` with the worked examples; documented the previously
|
|
160
|
+
undocumented `COWORK_*` / `PYTHON` env vars; noted the latest baseline is `desktop-1.15962.1`
|
|
161
|
+
(runtime-identical to `…0`).
|
|
162
|
+
- **Command/assertion reference.** Documented `doctor --tier cowork`, the `prune [<runs-dir>]`
|
|
163
|
+
positional, the `sync --force` alias, the `artifact_json` bare-existence mode, the
|
|
164
|
+
`tool_result_not_contains` fail-loud on truncated evidence, and that `expect_denied` is scenario-level
|
|
165
|
+
shorthand (not an assertion key).
|
|
166
|
+
|
|
167
|
+
### Parity
|
|
168
|
+
|
|
169
|
+
- **Synced the platform baseline to Claude Desktop 1.17377.1** (`baselines/desktop-1.17377.1.json`). The
|
|
170
|
+
staged agent ELF moved **2.1.187 → 2.1.197**. `sync` re-derived egress/gates/mount/web_fetch facts — **no
|
|
171
|
+
unknown deltas**; only `asarFingerprint` moved (the mount-mode and web_fetch drift-guard regexes both
|
|
172
|
+
still matched) and `api.claude.ai` joined `network.allowDomains`. None of the 6 pinned GrowthBook gates
|
|
173
|
+
drifted (loop / dispatch-cap / web_fetch-routing / transport / plugin-sync / CLI-plugin-broker all held
|
|
174
|
+
their prior on/off state). Re-verified end-to-end — the live scenario suite (`protocol` + `container`
|
|
175
|
+
tiers) passes against the new baseline.
|
|
176
|
+
- **Spot-checked the reconstructed system-prompt / host-loop content against the new asar** (this is
|
|
177
|
+
hand-authored, not something `sync` extracts): the `<application_details>` identity block — including the
|
|
178
|
+
load-bearing "is NOT Claude Code" correction — the host-loop `## Shell access` marker, subagent-append
|
|
179
|
+
gating, the `computer://` scheme, `request_cowork_directory`, and `coworkNativeFilePreview` are all
|
|
180
|
+
present and substantively unchanged since the `1.15200.0` reconstruction (only non-substantive
|
|
181
|
+
punctuation-level rewording). No re-authoring of `baselines/prompts/desktop-1.15200.0/` was needed.
|
|
182
|
+
- **Doc-pin sweep to `desktop-1.17377.1` / agent `2.1.197`** across README, DESIGN, SPEC, the spawn-contract
|
|
183
|
+
doc, and the skill's reference docs.
|
|
184
|
+
|
|
185
|
+
## [0.19.0] — 2026-06-30
|
|
186
|
+
|
|
187
|
+
### Changed
|
|
188
|
+
|
|
189
|
+
- **The default `--decider-llm` answering model is now Sonnet (`claude-sonnet-4-5`), not Haiku.** A
|
|
190
|
+
measurement on a real-doc skill with judgment-heavy gates found the prior Haiku default
|
|
191
|
+
**prose-declined ~50% of them** (replying "I don't have information…" instead of picking an option →
|
|
192
|
+
a fail-loud whiff); a Sonnet decider binds those gates and the run proceeds. ⚠️ This **raises per-gate
|
|
193
|
+
token cost** for every `--decider-llm` run, and a gate a weaker model would have prose-declined (and
|
|
194
|
+
failed fast) is now more likely to be answered and the run to continue (longer/costlier). Pin a
|
|
195
|
+
cheaper model to restore the old cost/behavior: `--decider-model <haiku-id>` or
|
|
196
|
+
`COWORK_HARNESS_DECIDER_MODEL`. (n is small — read this as "Haiku is too weak for judgment-heavy
|
|
197
|
+
gates," not a precise rate.) The whiff error now also names the `--decider-model` lever.
|
|
198
|
+
⚠️ **False-green caveat (new guidance, no behavior change):** binding-and-proceeding is the upside,
|
|
199
|
+
but a *decided* answer is the decider's best guess from the transcript tail — it never sees the
|
|
200
|
+
mounted documents — so it can fabricate (oracle-less gate) or get a doc-answerable fact wrong, and a
|
|
201
|
+
green run resting on it is a false pass. **Script any gate whose answer feeds a _semantic_ assertion
|
|
202
|
+
(`--answer` / `--answer-policy`); reserve `--decider-llm` for structural-assertion runs.** See
|
|
203
|
+
`references/fidelity-and-answers.md` in the skill.
|
|
204
|
+
- **Stall detector now also flags a stall AFTER an answered gate (H3).** The `stalled` verdict signal
|
|
205
|
+
previously fired only when a run ended on a question having made *no tool calls at all*. It now fires
|
|
206
|
+
when a run ends on a question and made **no productive tool call after its last `AskUserQuestion` gate** —
|
|
207
|
+
catching the case where the agent answers a gate, then re-asks in plain text and stops with the
|
|
208
|
+
deliverable never produced (previously a `result: "success"` false-green). The no-gate case is unchanged.
|
|
209
|
+
Re-derives identically on replay. ⚠️ This can flip a previously-green scenario to red — set
|
|
210
|
+
`allow_stall: true` to restore the prior verdict if ending on a question is the intended terminal state
|
|
211
|
+
(e.g. a skill that writes its output and then ends on a confirmation/"anything else?" question). The signal
|
|
212
|
+
is a *tool-position heuristic*, not deliverable detection, so it is imprecise in both directions: a post-gate
|
|
213
|
+
tool *call* (successful or errored) clears the flag even if nothing useful was produced (false negative);
|
|
214
|
+
and a deliverable produced *before* the final gate does NOT clear it, so a write-then-confirm-then-question
|
|
215
|
+
run is flagged (false positive). Assertions are the real guard — assert the deliverable
|
|
216
|
+
(`file_exists`/`artifact_json`), never `result` alone — and use `allow_stall` for a deliberate
|
|
217
|
+
question-terminal skill.
|
|
218
|
+
|
|
219
|
+
### Fixed
|
|
220
|
+
|
|
221
|
+
- **`--decider-llm` now binds an echoed grant label on a `web_fetch` approval gate** instead of failing
|
|
222
|
+
loud. The web_fetch permission-approval path matched the model's reply with an exact-label check only,
|
|
223
|
+
so a reply that echoed the option plus a self-glossed tail past a `:` boundary (e.g.
|
|
224
|
+
`Allow once: fetch this URL one time` against option `Allow once`) aborted the run — the same echo shape
|
|
225
|
+
the `AskUserQuestion` path already tolerates. The permission path now applies the same `echoPrefixMatch`
|
|
226
|
+
backstop, so the echoed label binds. Out-of-set replies still fail loud.
|
|
227
|
+
- **The three non-retried `--decider-llm` transport failures now name their mitigation in the error.** A
|
|
228
|
+
timeout, a `maxBytes` overflow, and a spawn failure (e.g. `ENOENT`) forfeit the run by design — they are
|
|
229
|
+
not transient — but the surfaced message previously named only the failure. It now points at the lever:
|
|
230
|
+
`COWORK_HARNESS_LLM_TIMEOUT_MS`, `COWORK_HARNESS_LLM_MAX_BYTES`, and `PATH` / `COWORK_HARNESS_CLAUDE_BIN`
|
|
231
|
+
respectively. No retry behavior changed.
|
|
232
|
+
- `doctor`'s "no token, but a Claude Code Keychain entry exists" remedy now also names the
|
|
233
|
+
`--dotenv` workaround. The macOS Keychain branch previously printed only "copy the token into
|
|
234
|
+
`./.env`", which led an operator whose token lived in a *different* file to conclude `doctor`
|
|
235
|
+
ignores `--dotenv` — it does not (the global `--dotenv <path>` is honored by `doctor` exactly as
|
|
236
|
+
by `skill`/`run`, since it loads into `process.env` before dispatch). The remedy now reads
|
|
237
|
+
"… or, if the token is already in another file, point at it: `cowork-harness --dotenv <path> <cmd>`".
|
|
238
|
+
|
|
9
239
|
## [0.18.0] — 2026-06-27
|
|
10
240
|
|
|
11
241
|
### Fixed
|
package/README.md
CHANGED
|
@@ -73,9 +73,10 @@ L2 microvm parity Optional. Agent inside a real Linux microVM (Lima/Apple-VZ
|
|
|
73
73
|
|
|
74
74
|
─── loop-mode overlays (orthogonal to L0/L1/L2: they pick WHERE the loop runs, not isolation) ───
|
|
75
75
|
|
|
76
|
-
hostloop Cowork's PRODUCTION split-execution
|
|
77
|
-
|
|
78
|
-
(mcp__workspace__bash
|
|
76
|
+
hostloop Cowork's PRODUCTION split-execution. The agent still runs in the container (as in
|
|
77
|
+
L1), but native Bash/WebFetch are disabled and routed host-side via the workspace
|
|
78
|
+
SDK-MCP server (mcp__workspace__bash; web_fetch via host curl) — reproducing the
|
|
79
|
+
host-loop tool boundary. (Only protocol runs the agent on the host.)
|
|
79
80
|
|
|
80
81
|
cowork Auto-picks hostloop vs container the way Cowork itself does, from GrowthBook
|
|
81
82
|
gate 1143815894 decoded in the synced baseline. "Do what real Cowork does."
|
|
@@ -92,9 +93,9 @@ Skill testing is the headline use, but the tool is a general harness over the Co
|
|
|
92
93
|
| Command | What it does | Reach for it when… |
|
|
93
94
|
|---|---|---|
|
|
94
95
|
| `skill <folder> "<prompt>"` | Run a local skill/plugin folder once against the staged agent | ad-hoc "is the skill alive / does it do X?" — the fast inner loop |
|
|
95
|
-
| `run <scenario.yaml \| dir/>` | Run authored scenarios with `assert:` + a CI-ready exit code | you want a repeatable, **asserted regression test** |
|
|
96
|
+
| `run <scenario.yaml \| dir/>` | Run authored scenarios with `assert:` + a CI-ready exit code; a decider (`--decider-cmd`/`--decider-dir`, or a scenario's `on_unanswered: llm`) can answer unscripted gates | you want a repeatable, **asserted regression test** |
|
|
96
97
|
| `chat <folder> [prompt]` | Interactive multi-turn REPL against a skill (TTY); optional seed prompt is sent as the first turn. `--upload <file>` / `--folder <dir>` (repeatable) attach files/project folders; `--verbose` / `-V` shows thinking blocks + tool inputs; `--fidelity protocol\|container\|hostloop` (no `microvm`/`cowork` in the REPL) | debugging a multi-turn flow by hand |
|
|
97
|
-
| `record` / `replay` | **Record a live run once → replay it token-free, Docker-free thereafter.** `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on any stale one, `--fail-on-skill-drift` only on skill-source drift; staleness is also reported per-result in `staleness[]` for a JSON gate; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
|
|
98
|
+
| `record` / `replay` | **Record a live run once → replay it token-free, Docker-free thereafter.** `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on any stale one, `--fail-on-skill-drift` only on skill-source drift; staleness is also reported per-result in `staleness[]` for a JSON gate; exits on the worst verdict). By default it evaluates the assertions **frozen in the cassette** (a sibling scenario whose `assert:` differs only prints a notice); `--assert-from <scenario.yaml>` / `--reassert` opt into re-checking against the **on-disk** `assert:` token-free (hard-fails on recording-shaping or skill drift) | **token-free, Docker-free CI** from a once-recorded run |
|
|
98
99
|
| `verify-cassettes <file\|dir>` | Token-free CI gate over committed cassettes: a privacy scan (email/currency/domain → exit 1; whole-token allows via `--allow` / class-scoped `--allow-domain` / `--allow-email` / `--allow-file`) + a staleness check (`--skip-privacy` or `--skip-staleness` to run only one half). A dir argument scans `*.cassette.json` in that dir only (**non-recursive**) | gating **committed cassettes** against PII leaks + "edited the skill, forgot to re-record" |
|
|
99
100
|
| `verify-run <run-dir> <scenario.yaml>` | Re-evaluate a scenario's `assert:` (and, when the scenario declares `answers:`, whether they still match the run's actual gates) against an already-kept run dir — **no live agent, no tokens, no Docker** (~1s) | iterating on a wrong assertion or a drifted `answer` without a full live re-record |
|
|
100
101
|
| `trace <run-id>` | Digest a run's `events.jsonl` (`--view tools\|questions\|dispatches`; legacy `--tools` / `--gates` / `--dispatches` aliases still accepted) | "how many sub-agents *actually* dispatched, and which?" |
|
|
@@ -105,10 +106,11 @@ Skill testing is the headline use, but the tool is a general harness over the Co
|
|
|
105
106
|
| `assertions --list` | List the available scenario assertions (generated from the schema) | "what can I assert?" without grepping the source |
|
|
106
107
|
| `decide` | Validate a decider against a sample question in ~2 s (no run) | sanity-check a `--decider-*` / `--answer` wiring before a long run |
|
|
107
108
|
| `gates` / `answer` | Stream / answer in-band gates for `--decider-dir` | a **driving agent** answers live questions via a Monitor |
|
|
108
|
-
| `
|
|
109
|
-
| `
|
|
110
|
-
| `
|
|
111
|
-
| `
|
|
109
|
+
| `status <run-id \| run-dir> [--follow]` | Check whether a background run is alive (state/elapsed/tool counts) by reading `status.json` — no `ps aux` needed (unreliable across sandbox/PID-namespace boundaries). `--follow` streams one line per change until done/error; staleness detection catches a `SIGKILL`'d process | a **driving agent** (or script) checking on a run it launched in the background |
|
|
110
|
+
| `boundary-check [baseline] [--session <file>]` | Prove the **L1 Docker** sandbox enforces Cowork's limitations (sealed FS + default-deny egress; `container`/`hostloop` share this sandbox — `microvm`'s guest firewall is not probed here); `--session` folds a session's `egress.extra_allow` into the probe allowlist | verifying the harness's own fidelity |
|
|
111
|
+
| `sync` / `list` | Derive/refresh (`sync [--diff] [--allow-empty\|--force]`) & list platform baselines from the Desktop install | after Claude Desktop updates (baselines ship, so it's optional otherwise) |
|
|
112
|
+
| `doctor [--tier <t>]` | Read-only prerequisite check, per tier (Docker + agent image for `container`/`hostloop`/`cowork`; **Lima** for `microvm`; plus staged agent, token, baseline); prints the exact `docker build` line if the agent image is missing | "can I run the live tiers — what's missing?" before a first live run |
|
|
113
|
+
| `prune [<runs-dir>] [--keep-last <n>]` | Prune accumulated run dirs (keeps the N most recent per scenario; pinned `--session-id` runs are never pruned); the optional positional overrides the runs root; `--dry-run` | the machine-global runs root has grown and you want space back |
|
|
112
114
|
| `rehash <dir/>` | Migrate cassette fingerprints to the current format version when the content is provably unchanged (`--dry-run`); no re-record needed | a cassette-format bump flagged committed fixtures as stale |
|
|
113
115
|
| `vm <init\|status\|delete\|prune>` | Manage the L2 Apple-VZ / Lima microVM (`prune` removes orphaned VMs left by config/agent-version changes) | running `--fidelity microvm` |
|
|
114
116
|
|
|
@@ -140,6 +142,18 @@ cowork-harness replay examples/replays/example-pdf-skill.cassette.json
|
|
|
140
142
|
|
|
141
143
|
Only the committed-cassette `replay` above is fully self-contained. Live `run`/`skill` need the prerequisites in the next section — and note the `protocol` tier skips Docker and the staged agent but **still calls a real model** (via the host `claude`), so it needs the auth token.
|
|
142
144
|
|
|
145
|
+
> **Which path am I on?**
|
|
146
|
+
> - **Replay only (zero setup):** `cowork-harness replay <cassette>` — no token, no Docker, no agent. The command above.
|
|
147
|
+
> - **`protocol` (real model, no Docker):** needs only the auth token (item 3 below).
|
|
148
|
+
> - **Live `container` / `microvm` / `hostloop` / `cowork`:** needs Docker (or Lima for `microvm`), a staged agent, and the token — run `cowork-harness doctor` first.
|
|
149
|
+
> - **Invocation:** from a source checkout, `node dist/cli.js <cmd>` (or `npm link` to get the `cowork-harness` command); from a global install, `cowork-harness <cmd>`; the companion skill falls back to `npx cowork-harness@>=0.20.0`.
|
|
150
|
+
>
|
|
151
|
+
> *(One project, three names: the repo folder is `claude-cowork-headless-emulator`; the npm package, the CLI command, and the GitHub repo are all `cowork-harness`.)*
|
|
152
|
+
|
|
153
|
+
Two more worked examples worth knowing about: `examples/scenarios/protocol-smoke.yaml` (zero-Docker smoke
|
|
154
|
+
test) and `examples/scenarios/skill-loads.yaml` (container-tier acceptance check) — see
|
|
155
|
+
[examples/README.md](./examples/README.md).
|
|
156
|
+
|
|
143
157
|
### Drive it from Claude Code (companion skill)
|
|
144
158
|
|
|
145
159
|
This repo ships a **companion skill** (`.claude/skills/cowork-harness/`) that teaches an agent how to drive the harness — author scenarios, pick a fidelity tier, script answers, place assertions in the right CI lane, and avoid the "✓ passed ≠ correct" traps. Install it via the bundled marketplace — run these **in Claude Code** (slash commands, not your shell):
|
|
@@ -149,7 +163,7 @@ This repo ships a **companion skill** (`.claude/skills/cowork-harness/`) that te
|
|
|
149
163
|
/plugin install cowork-harness@cowork-harness
|
|
150
164
|
```
|
|
151
165
|
|
|
152
|
-
The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.
|
|
166
|
+
The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.20.0` (a version floor that fails loud rather than silently fetching a too-old CLI; Node ≥ 20). Tiers above `protocol` still need Docker/Lima and a Claude Desktop agent binary — see the prerequisites below.
|
|
153
167
|
|
|
154
168
|
It also follows the open [Agent Skills](https://github.com/vercel-labs/skills) spec, so it installs cross-editor (Cursor, Codex, OpenCode, …) by pointing the `npx skills` CLI at `.claude/skills/cowork-harness` in this repo. (Working *inside* this repo, the skill auto-loads as a project skill — no install needed.)
|
|
155
169
|
|
|
@@ -158,7 +172,7 @@ It also follows the open [Agent Skills](https://github.com/vercel-labs/skills) s
|
|
|
158
172
|
### Prerequisites for anything above `protocol` fidelity
|
|
159
173
|
|
|
160
174
|
(The `protocol` tier skips items 1–2 — no Docker, no staged agent — but still calls a real model via the host `claude`, so it needs item 3, the auth token; only a committed-cassette `replay` needs nothing at all):
|
|
161
|
-
1. **Claude Desktop, opened once.** The Cowork agent binary is **bind-mounted from your own install** at run time — nothing Anthropic-owned is bundled. Open Cowork once so the agent ELF is staged (`…/claude-code-vm/<ver>/claude`); the harness auto-detects it, or set `COWORK_AGENT_BINARY=<path>` to point at it. Without a staged agent, container/cowork
|
|
175
|
+
1. **Claude Desktop, opened once.** The Cowork agent binary is **bind-mounted from your own install** at run time — nothing Anthropic-owned is bundled. Open Cowork once so the agent ELF is staged (`…/claude-code-vm/<ver>/claude`); the harness auto-detects it, or set `COWORK_AGENT_BINARY=<path>` to point at it. Without a staged agent, the sandboxed live tiers (container/microvm/hostloop/cowork) fail with "Open Cowork once to stage it…". (`protocol` and a committed-cassette `replay` need no staged agent.)
|
|
162
176
|
2. **Docker (arm64)** + the agent image: `docker build --platform linux/arm64 -t cowork-agent-base:2 -f docker/Dockerfile.agent .` (override the tag with `COWORK_AGENT_IMAGE`). This **core** image mirrors the real Cowork rootfs's document/data toolchain (Node 22, openpyxl/pandas/pdf/docx/pptx, …). For OCR / LibreOffice / markitdown / opencv / PDF-table skills, build the **full-parity** image instead: `docker build --platform linux/arm64 --build-arg COWORK_FULL_PARITY=1 -t cowork-agent-full:2 -f docker/Dockerfile.agent .` and point `COWORK_AGENT_IMAGE` at it. (A run on the core image that uses an omitted capability is flagged — see capability detection.) The `-f docker/Dockerfile.agent .` paths are **repo-relative** — run it from a source checkout, not a global `npm install -g`. (Global install? Run `cowork-harness doctor --tier container` — it prints the exact `docker build` command with the correct package-local Dockerfile path.) The **`microvm` (L2)** tier uses **Lima / Apple-VZ** instead of Docker (`brew install lima`; no agent image to build) — `doctor --tier microvm` checks for that, not Docker.
|
|
163
177
|
3. **An auth token** — either `export CLAUDE_CODE_OAUTH_TOKEN=$(claude setup-token)` (the `claude setup-token` step needs the **`claude` CLI**: `npm i -g @anthropic-ai/claude-code`) or a **`.env`** file (copy `.env.example` → `.env`; gitignored). The token resolves in priority order: exported env > `--dotenv <path>` > `./.env` (cwd) > `<install>/.env` (the package root), so a `npm link`ed install works from any directory. Keep `.env` at a working-dir or install root, never inside a mounted skill/project folder. (Use `--dotenv`, not `--env-file` — Node reserves the latter.)
|
|
164
178
|
|
|
@@ -229,7 +243,7 @@ cowork-harness lint examples/scenarios/*.yaml
|
|
|
229
243
|
> see [Quick start](#quick-start) for both, and run `cowork-harness doctor` to check what's missing. For the
|
|
230
244
|
> zero-infra path (no Docker, no token), start with the token-free [`replay` demo](#quick-start).
|
|
231
245
|
|
|
232
|
-
The fastest path — point at a **local folder**, no repo, no `claude plugin install`, no marketplace, no version bump, no cache layers. The folder is copied **fresh into the session on every run**, so you edit and re-run and your changes are live immediately
|
|
246
|
+
The fastest path — point at a **local folder**, no repo, no `claude plugin install`, no marketplace, no version bump, no cache layers. The folder is copied **fresh into the session on every run**, so you edit and re-run and your changes are live immediately. (Inside a git repo it stages the **git-tracked** files — the fidelity boundary, since real Cowork installs from a repo and sees only committed files; so `git add` a brand-new skill or it would mount empty. The harness now **hard-fails loudly** on a would-be-empty mount and **notices** excluded untracked files, rather than silently staging nothing — `COWORK_HARNESS_GITSET=0` copies untracked files instead.)
|
|
233
247
|
|
|
234
248
|
```bash
|
|
235
249
|
# Auth once — export it, OR put it in a .env file (resolved: env > --dotenv > ./.env > <install>/.env):
|
|
@@ -262,7 +276,9 @@ cowork-harness chat ~/my-plugin # interactive multi-turn REPL (
|
|
|
262
276
|
`run`/CI), `prompt` (ask at the TTY — the default for `skill` when interactive), or `first` (pick
|
|
263
277
|
option 1, loudly warn). Pick with `--on-unanswered`; left unset, `skill` is **adaptive** (`prompt` on
|
|
264
278
|
a TTY, `fail` when piped/CI) and `run` is always `fail`. Exit codes: `0`
|
|
265
|
-
pass · `1` assertion/agent failure · `2` usage / unanswered-under-`fail` / runtime · `3` boundary/integrity
|
|
279
|
+
pass · `1` assertion/agent failure · `2` usage / unanswered-under-`fail` / runtime · `3` boundary/integrity
|
|
280
|
+
(the `boundary-check` **command**'s own probe failures are the exception — they follow the assertion
|
|
281
|
+
convention and exit `1`; see SPEC.md §11).
|
|
266
282
|
After a run, the footer **echoes every auto-answered
|
|
267
283
|
question as a copy-pasteable `--answer "<q>=<choice>"` line** — run once exploratorily, then paste them
|
|
268
284
|
back to lock in a deterministic re-run.
|
|
@@ -270,12 +286,17 @@ back to lock in a deterministic re-run.
|
|
|
270
286
|
**Output.** `skill` **renders the agent's work** (assistant text + tool calls) and a metered footer —
|
|
271
287
|
you see *what it did*, not just a green. `run` is verdict-first but **prints the failing transcript
|
|
272
288
|
inline** on a `FAIL` (no spelunking `runs/…`). Tune with `--quiet` (verdict only) / `--verbose`/`-V`
|
|
273
|
-
(+ thinking, tool inputs, sub-agent tree).
|
|
274
|
-
|
|
289
|
+
(+ thinking, tool inputs, sub-agent tree). For **shareable** output (screenshots, GIFs, README clips),
|
|
290
|
+
`--compact` drops the informational `[capability]` notices (the probe + its hard-fail safety net stay)
|
|
291
|
+
and `--demo` adds `--compact` + hides the `runs →` host-path header (runs stay in the durable location, so
|
|
292
|
+
`scaffold`/`trace` still work); combined with the `$HOME`→`~` collapse, demo output leaks no host paths.
|
|
293
|
+
`--output-format json` emits a stable machine envelope on stdout
|
|
294
|
+
(`{tool, version, command, ok, results[], error}`; each result carries its own `verdict: {pass, signals[], guards[]}`
|
|
295
|
+
so you can read per-result pass/fail **and why** without recomputing; errors are `{ok:false, error:{category,message,hint}}`)
|
|
275
296
|
— see [SPEC §11](./SPEC.md). Human output is stderr, machine output is stdout, so `--output-format json` pipes
|
|
276
297
|
cleanly. Honors `NO_COLOR`.
|
|
277
298
|
|
|
278
|
-
**Test a specific local plugin version** — just point at the folder at that version (it's copied fresh; no install, no version bump). Add more with `--plugin`:
|
|
299
|
+
**Test a specific local plugin version** — just point at the folder at that version (it's copied fresh — git-tracked files inside a repo; no install, no version bump). Add more with `--plugin`:
|
|
279
300
|
```bash
|
|
280
301
|
cowork-harness skill ~/my-plugin "..." --plugin ~/other-plugin
|
|
281
302
|
```
|
|
@@ -337,7 +358,7 @@ assert:
|
|
|
337
358
|
# Relative paths below resolve from THIS file's dir (absolute and ~ are used as-is).
|
|
338
359
|
model: claude-opus-4-8
|
|
339
360
|
effort: high
|
|
340
|
-
max_thinking_tokens: 8000 # kept low for
|
|
361
|
+
max_thinking_tokens: 8000 # kept low for EXAMPLE cost only — don't copy this value; real default is 31999 (see docs/session.md)
|
|
341
362
|
permission_mode: default
|
|
342
363
|
permission_parity: cowork # cowork (allow unscripted tool calls, the default) | strict (deny unscripted)
|
|
343
364
|
folders:
|
|
@@ -391,12 +412,22 @@ run.jsonl harness-observability log: decisions (+who decided), sub-age
|
|
|
391
412
|
trace.json structured run trace: steps, questions, sub-agents, egress, decisions, cost
|
|
392
413
|
egress.log raw allow/deny per outbound connection (microvm: at top level; container: under
|
|
393
414
|
proxy/ — the allow/deny decisions are also folded into run.jsonl/result.json)
|
|
394
|
-
result.json assertion results + decisions + sub-agents + cost/usage + exit status
|
|
415
|
+
result.json assertion results + decisions + sub-agents + cost/usage + exit status +
|
|
416
|
+
gate provenance (how each AskUserQuestion gate was answered)
|
|
395
417
|
agent.stderr.log the agent process's stderr (auth errors, flag rejects)
|
|
396
418
|
```
|
|
397
419
|
|
|
398
420
|
Secrets (the injected OAuth token / API key) are scrubbed from every persisted log by value.
|
|
399
421
|
|
|
422
|
+
**Gate provenance** — `result.json` carries a `gateProvenance` block recording, per AskUserQuestion gate, *how* it was answered — `scripted`, `decided(llm|external)`, `first-option`, or `prompt` — with a `bySource` histogram and per-gate `{question, answeredBy, answer, model?}`. The verdict footer shows a counts-only one-liner and `trace <id> --view questions` annotates each gate with its `by`/`model`, so you can see which assertions sit downstream of a non-reproducible (decided) gate:
|
|
423
|
+
|
|
424
|
+
```
|
|
425
|
+
✓ success [container] · 4 tools · 12.3s ⚠ non-deterministic (LLM-decided)
|
|
426
|
+
gates: 3 · 2 decided(llm), 1 scripted
|
|
427
|
+
```
|
|
428
|
+
|
|
429
|
+
It is informational — it never changes the verdict. The block is a live/`partial`-lane surface (absent on the replay lane, which reports reproducibility via `nonDeterministic: false`).
|
|
430
|
+
|
|
400
431
|
**Debugging a run** — when a run misbehaves or a green looks too good to trust, [docs/debugging.md](./docs/debugging.md) is the map: `inspect` → `trace` → `chat` → `verify-run` for a misbehaving skill, and the false-green hunt for a green you don't trust.
|
|
401
432
|
|
|
402
433
|
---
|
|
@@ -457,7 +488,7 @@ The provided [GitHub Actions workflow](.github/workflows/ci.yml) runs a **four-s
|
|
|
457
488
|
|---|---|---|---|
|
|
458
489
|
| **unit** | format check · typecheck · unit tests · build · CLI smoke · token-free `replay` · `verify-cassettes` · `lint` | nothing | every push/PR |
|
|
459
490
|
| **boundary** | builds the pinned agent image, brings up the default-deny network, runs `boundary-check` | Docker, arm64 runner | proves the sandbox enforces Cowork's limits — **no API key** |
|
|
460
|
-
| **scenarios** | the live scenario suite
|
|
491
|
+
| **scenarios** | the live scenario suite (mixed `protocol` + `container` fidelity across `examples/scenarios/`), uploads transcripts/egress logs as artifacts | `ANTHROPIC_API_KEY` | fork PRs: the whole job is skipped (`if:` guard); same-repo without a key: warns and exits 0 |
|
|
461
492
|
| **parity-drift** | reminder to re-`sync` when Desktop updates | nothing | informational, never blocks |
|
|
462
493
|
|
|
463
494
|
This ordering means cheap checks fail fast, the **boundary parity gate runs without secrets** (so forks get it too), and expensive live runs only happen when a key is present.
|
|
@@ -482,14 +513,15 @@ Most runs need **none** of these — the defaults are correct. They're grouped b
|
|
|
482
513
|
- `COWORK_CONTAINER_RUNTIME=podman` — use Podman instead of Docker.
|
|
483
514
|
- `COWORK_AGENT_IMAGE=<tag>` — override the agent image name (default `cowork-agent-base:2`); `COWORK_AGENT_BINARY=<path>` — override the auto-detected staged agent ELF.
|
|
484
515
|
- `COWORK_SKIP_CAPABILITY_PROBE=1` — skip the per-run capability probe (the harness otherwise probes the agent image/VM for the document/OCR/Office capabilities the real Cowork rootfs ships and **fails a run that uses one the image omits** — a likely false negative; suppress per-scenario with `allow_missing_capability: true`, or rebuild full parity).
|
|
485
|
-
- `COWORK_HARNESS_DECIDER_MODEL` — the default `--decider-llm` answering model (overridden by the `--decider-model` flag; falls back to the
|
|
516
|
+
- `COWORK_HARNESS_DECIDER_MODEL` — the default `--decider-llm` answering model (overridden by the `--decider-model` flag; falls back to the Sonnet default — pin a cheaper model for simple gates to cut cost). `COWORK_HARNESS_DECIDER_DIR_POLL_MS` / `_TIMEOUT_MS` — tune the `--decider-dir` rendezvous poll/backstop; `COWORK_HARNESS_DECIDER_CMD_TIMEOUT_MS` / `COWORK_HARNESS_LLM_TIMEOUT_MS` — backstop a hung `--decider-cmd` helper / `--decider-llm` model call (default 600 s, fail loud); `COWORK_HARNESS_LLM_RETRIES` — bounded retries for a transient non-zero `claude -p` exit in the `--decider-llm` transport (default 2, clamped to 0–10; set `0` to disable, e.g. deterministic CI); `COWORK_HARNESS_LLM_MAX_BYTES` — stdout cap on a `--decider-llm` model call (default 8 MiB, fail loud past it); `COWORK_HARNESS_DIALOG_TIMEOUT_MS` — override the 6 s dialog auto-cancel.
|
|
486
517
|
- `COWORK_HARNESS_RUNS_DIR` (or the `--run-dir <path>` flag — a **global** flag that must precede the subcommand, like `--dotenv`) — override the default run-output root `~/.cowork-harness/runs` (kept out of any working tree so sensitive skill inputs/outputs don't land in a repo). Precedence: `--run-dir` > env > default. The root is flat and machine-global (shared across projects); pinned `--session-id` runs are guarded against cross-project overwrite, and `prune` never prunes them. In CI, set it to a workspace path (e.g. `runs`) so artifact upload can collect the runs. `COWORK_HARNESS_ALLOW_FOREIGN_RESUME=1` overrides the guard that blocks `--resume` onto another project's pinned session.
|
|
487
518
|
- **Networking / loop:** `COWORK_EGRESS_PROXY` overrides the egress-proxy URL injected into the sandbox; `COWORK_PROXY_IMAGE` overrides the egress proxy Docker image name (default `cowork-egress-proxy:1`); `COWORK_DOCKER_NETWORK` pins the Docker network the agent container joins; `CLAUDE_FORCE_HOST_LOOP=1` forces the host-loop path regardless of the baseline's loop decision (the `cowork` tier's auto-pick). `COWORK_LIMACTL` overrides the `limactl` binary path (default `/opt/homebrew/bin/limactl`).
|
|
488
519
|
- **Strictness escape hatches** (the harness fails loud by default): `COWORK_HARNESS_SOFT_MISSING=1` downgrades a missing mount source from a hard error to warn-and-exclude; `COWORK_HARNESS_ALLOW_CONFIG_DIR_WRITE=1` permits writing into an existing pinned `plugins.config_dir` (otherwise refused, to avoid clobbering a real Claude config).
|
|
489
|
-
- **Staleness boundary:** the cassette-staleness skill hash
|
|
520
|
+
- **Staleness boundary:** the cassette-staleness skill hash — **and the staged file set delivered into the sandbox** — use the **git-tracked** file set of each skill/plugin source dir by default (a non-repo dir falls back to a raw walk; OS-junk like `.DS_Store` is always excluded). A plugin/skill that would deliver **zero** tracked files **hard-fails** (so an unadded skill can't silently mount empty); partial exclusions emit a `::notice:: [stage]`. `COWORK_HARNESS_GITSET=0` opts out to the legacy raw walk for every dir (and copies untracked files). `COWORK_HARNESS_DEBUG_SKILLHASH=1` dumps the exact file set feeding the hash on a staleness mismatch (and flags OS-junk) so a drift source is one line. Declare per-plugin non-runtime paths in a `.cowork-hashignore` file (or the session `staleness.hash_ignore`). `COWORK_HARNESS_AGENT_SCOPE=skill` (opt-in) refines a scenario's `skills:` scope so a **skill-named** `agents/<name>.md` re-stales only that skill's cassettes instead of the whole fleet (generic agents stay shared; stamped into the fingerprint, so flipping it is a one-time re-record like `GITSET`).
|
|
490
521
|
- **`skill` / `chat` defaults:** `COWORK_HARNESS_FIDELITY` sets the default fidelity tier for ad-hoc `skill`/`chat` runs (a `--fidelity` flag or a scenario's `fidelity:` still wins); `COWORK_HARNESS_MODEL` sets the default model; `COWORK_HARNESS_OUTPUT_FORMAT` (`text`|`json`) sets the default output format. Each is overridden by the matching explicit flag.
|
|
491
522
|
- **Secret scrubbing:** `COWORK_HARNESS_SCRUB_KEYS=<KEY1,KEY2>` adds extra env-var names whose values are redacted from logs (beyond the known auth tokens + `ANTHROPIC_CUSTOM_HEADERS`); `COWORK_HARNESS_SCRUB_VALUES=<v1,v2>` redacts literal values regardless of env. **Committed-cassette redaction:** `COWORK_HARNESS_REDACT_PATTERNS=<rx1,rx2>` / `COWORK_HARNESS_REDACT_KEYS=<k1,k2>` extend the privacy layer that scrubs recorded `controlOut` before a cassette is written for commit.
|
|
492
523
|
- L2 microVM: `COWORK_VM_GATEWAY` overrides the Lima host-proxy gateway IP (default `192.168.5.2`); `COWORK_VM_PROXY_PORT` the proxy port. The Lima instance is named `cowork-vm-<config-hash>` (a config change → a fresh VM); `COWORK_LIMA_INSTANCE` pins a fixed name, and `vm prune` removes orphaned ones.
|
|
524
|
+
- **Advanced / internal escape hatches** (rarely needed): `PYTHON` overrides the interpreter for `lint` / scenario tooling (default `python3`); `COWORK_HARNESS_DEBUG=1` surfaces which `.env` files were loaded; `COWORK_HARNESS_CLAUDE_BIN=<path>` points the `--decider-llm` transport at a specific `claude` binary; `COWORK_HARNESS_ALLOW_AGENT_FALLBACK=1` lets the harness use the newest sibling agent binary when the baseline-pinned version is missing (a fidelity compromise — off by default); `COWORK_MANAGED_CONFIG=1` forces the managed-config path on `protocol`; `COWORK_HARNESS_ALLOW_MISSING_PROMPT=1` downgrades a missing prompt asset to a warning; `COWORK_HARNESS_SAFE_STAGING_PREFIX=<a,b>` whitelists prefixes under which a delete in `outputs/` is allowed (otherwise delete-in-outputs fails loud); `COWORK_HARNESS_NO_HEARTBEAT=1` disables the idle-run heartbeat, `COWORK_HARNESS_HEARTBEAT_MS` tunes its interval (default per `src/run/renderer.ts`); `COWORK_HARNESS_CLI=/path/to/cli.js` overrides which built CLI the Python `cowork` pytest lane drives (see python/README.md).
|
|
493
525
|
- Pin `baseline: desktop-<ver>` and `model:` in a session for byte-stable runs; use `latest` to track.
|
|
494
526
|
|
|
495
527
|
## Maintenance: parity between releases
|
|
@@ -539,6 +571,9 @@ This repo is built to be driven by agents, not just read by humans:
|
|
|
539
571
|
- **Machine-readable interfaces** — stable `--output-format json` envelope on stdout, deterministic exit codes (`0`/`1`/`2`/`3`), and `--help` on every command.
|
|
540
572
|
- **JSON Schemas** — [`schema/scenario.schema.json`](./schema/scenario.schema.json) and [`schema/session.schema.json`](./schema/session.schema.json) describe every field of the YAML you author (generated from the source schemas; `npm run schema`).
|
|
541
573
|
|
|
574
|
+
`AGENTS.md`, `SPEC.md`, and `DESIGN.md` are **not** shipped in the npm package (see `package.json`
|
|
575
|
+
`files`) — clone the repo (or view it on GitHub) to read them.
|
|
576
|
+
|
|
542
577
|
---
|
|
543
578
|
|
|
544
579
|
## Documentation
|
|
@@ -566,11 +601,11 @@ This repo is built to be driven by agents, not just read by humans:
|
|
|
566
601
|
|
|
567
602
|
## Status
|
|
568
603
|
|
|
569
|
-
**Verified end-to-end against the live staged agent (2.1.
|
|
604
|
+
**Verified end-to-end against the live staged agent (2.1.197 / asar 1.17377.1).** Host-side parity (egress/gates/mount/web_fetch facts) re-derived via `sync` — no unknown deltas; only `asarFingerprint` moved, plus `api.claude.ai` joining `network.allowDomains`. The reconstructed prompt appends (`prompts/desktop-1.15200.0/`) were spot-checked against the new asar: substantively unchanged (minor punctuation-level rewording only) — the load-bearing "is NOT Claude Code" correction, the host-loop `## Shell access` generator, and the web_fetch/mount facts all still hold. The live scenario suite (`protocol` + `container` tiers) passes against this baseline. The latest shipped baseline — what `baseline: latest` resolves to (`cowork-harness list`) — is **`desktop-1.17377.1`**.
|
|
570
605
|
|
|
571
|
-
- ✅ **Three isolation tiers (L0/L1/L2) + two loop-mode overlays** — `protocol` (L0 control loop), `container` (L1 sandboxed arm64 + per-run default-deny egress sidecar), `microvm` (L2 real Apple-VZ Linux microVM + guest firewall); plus the loop-mode overlays `hostloop` (production split-execution: agent
|
|
606
|
+
- ✅ **Three isolation tiers (L0/L1/L2) + two loop-mode overlays** — `protocol` (L0 control loop), `container` (L1 sandboxed arm64 + per-run default-deny egress sidecar), `microvm` (L2 real Apple-VZ Linux microVM + guest firewall); plus the loop-mode overlays `hostloop` (production split-execution: the agent runs in the container with native Bash/WebFetch disabled and routed host-side via the workspace SDK-MCP server) and `cowork` (auto-picks host-loop vs container the way Cowork does — gate `1143815894`). Egress enforced at container/microvm/hostloop; `boundary-check` reports **ALL CONSTRAINTS ENFORCED**.
|
|
572
607
|
- ✅ **Three-seam driver** — `AgentSession` (control protocol) → `Decider` (scripted + `on_unanswered` policy, no silent false-greens) → `Run` (turn loop, sub-agent dispatch tree, `RunRecord`). Multi-turn `chat`, deterministic cassette `record`/`replay` (no token), `run.jsonl`/`trace.json` logging with secret-scrub.
|
|
573
|
-
- ✅ **Answering live questions, every way you'd need** — `--decider-llm --intent "<one line>"` (a
|
|
608
|
+
- ✅ **Answering live questions, every way you'd need** — `--decider-llm --intent "<one line>"` (a model picks per question, steered by your test intent — the ergonomic default; defaults to **Sonnet** (a weaker model prose-declines ambiguous judgment gates → fail-loud), override with **`--decider-model <id>`** — pin a cheaper model for simple gates or a stronger one for the hardest; it replies with the option **number**, mapped to the exact label, so it can't whiff by parroting the label text, and an out-of-set reply fails loud; non-determinism is flagged so a green isn't mistaken for a scripted pass — and ⚠️ a *decided* gate whose answer feeds a **semantic** assertion can still false-green, so script those), `--answer-policy <yaml>`/`--answer "rx=c"` (declarative regex→label, deterministic CI — accepts a **stable partial anchor** for skills whose labels drift run-to-run), `--decider-cmd '<helper>'` (custom logic — the Python `serve_decider(fn)` adapter pre-builds the wire loop so the helper writes only the decision), and **`--decider-dir <dir>`** (the *driving agent* answers each gate **in-band** with full context — it arms a Monitor on `cowork-harness gates <dir> --follow` and replies with `cowork-harness answer <dir> --gate <N> --choose <label>`; the session-under-test stays live, no resume/re-worded question — binary-verified). **multiSelect gates** are answerable on every channel — repeat `--choose` (or, for `--decider-llm`, a comma-list of option numbers; or a JSON-array reply from a `--decider-cmd` helper / hand-written `resp-N.json`); all deliver the binary-verified `", "`-joined wire shape. Every channel keeps stdout free, so all compose with `--output-format json`. A question is **never silently answered with option 1** — unhandled fails loud. Validate any decider in ~2s with `cowork-harness decide`.
|
|
574
609
|
- ✅ **Sub-agent aggregation + `trace`** — recognizes the real `Agent` dispatch tool (binary-verified; `Task` is its alias) so `subagent_dispatched`/`dispatch_count_max` fire under `--fidelity cowork`, excluding the `TaskCreate` todo list; `cowork-harness trace <id> --view tools` digests `events.jsonl` (each tool row now shows its **result status** `ok`/`error`), and `trace <id> --view questions` shows the gate lifecycle (**question → injected answer → delivered result**); `result.json`/`--keep` surface the deep `mnt/outputs` deliverable path.
|
|
575
610
|
- ✅ **Answer delivery is verified, not assumed** — an AskUserQuestion answer is injected as the binary's full tool input (`{questions, answers}`, ELF-verified), so the answer actually reaches the model; `RunResult.gateDeliveries[]` + the `gate_answers_delivered` assertion catch any delivery failure, and `RunResult.toolCounts` gives the **truthful** per-tool call count (host-routed `WebSearch` shows here, not the always-0 `usage.server_tool_use`).
|
|
576
611
|
- ✅ **Binary-grounded fidelity** — cwd `/sessions/<id>`, the three-channel MCP model (`--mcp-config` honored in plain cowork mode; host/API-routed `web_fetch`; SDK-server delivery), host-loop tool partition, auth-env token-only drop, and production GrowthBook gates pinned per release.
|