cowork-harness 1.16.0 → 1.18.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (75) hide show
  1. package/.claude/skills/cowork-harness/SKILL.md +59 -10
  2. package/.claude/skills/cowork-harness/references/ci-recipe.md +49 -8
  3. package/.claude/skills/cowork-harness/references/critique.md +1 -1
  4. package/.claude/skills/cowork-harness/references/fidelity-and-answers.md +2 -2
  5. package/.claude/skills/cowork-harness/references/scenario-schema.md +24 -12
  6. package/.claude/skills/cowork-harness/references/task-recipes.md +17 -2
  7. package/.claude/skills/cowork-harness/scripts/assertion-keys.json +19 -0
  8. package/.claude/skills/cowork-harness/scripts/scenario.py +164 -0
  9. package/CHANGELOG.md +485 -0
  10. package/CONTRIBUTING.md +1 -0
  11. package/DESIGN.md +4 -4
  12. package/README.md +26 -20
  13. package/SPEC.md +7 -4
  14. package/baselines/desktop-1.24012.11.json +546 -0
  15. package/baselines/desktop-1.24012.9.json +60 -21
  16. package/baselines/desktop-1.25927.0.json +547 -0
  17. package/dist/agent/session.js +79 -7
  18. package/dist/assert.js +66 -4
  19. package/dist/cli.js +193 -45
  20. package/dist/critique/command.js +4 -1
  21. package/dist/critique/package-evidence.js +8 -1
  22. package/dist/decide/decider.js +16 -7
  23. package/dist/hostloop/skills-handler.js +48 -11
  24. package/dist/loop-decision.js +36 -13
  25. package/dist/run/analyze-artifact.js +1 -1
  26. package/dist/run/analyze-skill.js +32 -1
  27. package/dist/run/cassette.js +513 -16
  28. package/dist/run/chat.js +3 -3
  29. package/dist/run/execute.js +296 -80
  30. package/dist/run/hook-events.js +115 -0
  31. package/dist/run/inputs.js +17 -5
  32. package/dist/run/mutate.js +197 -0
  33. package/dist/run/pre-run-manifest.js +2 -2
  34. package/dist/run/renderer.js +15 -8
  35. package/dist/run/subagent-reasoning.js +86 -17
  36. package/dist/run/trace-view.js +11 -2
  37. package/dist/run/verdict.js +85 -3
  38. package/dist/runtime/container.js +2 -2
  39. package/dist/runtime/hostloop-stage.js +4 -5
  40. package/dist/runtime/hostloop.js +2 -2
  41. package/dist/scan.js +100 -0
  42. package/dist/session.js +55 -4
  43. package/dist/sync/baseline-diff.js +21 -0
  44. package/dist/sync/cowork-sync.js +518 -101
  45. package/dist/types.js +79 -6
  46. package/docs/README.md +1 -1
  47. package/docs/boundary.md +2 -0
  48. package/docs/cassette.md +104 -4
  49. package/docs/cowork-spawn-contract-1.12603.1.md +3 -1
  50. package/docs/critique.md +24 -6
  51. package/docs/debugging.md +1 -1
  52. package/docs/decider-dir.md +69 -2
  53. package/docs/fidelity-gaps.md +122 -6
  54. package/docs/gotchas.md +19 -3
  55. package/docs/maintenance.md +58 -1
  56. package/docs/protocol.md +42 -3
  57. package/docs/run-status.md +14 -1
  58. package/docs/scenario.md +78 -12
  59. package/docs/session.md +24 -4
  60. package/docs/subagents.md +71 -27
  61. package/examples/README.md +11 -5
  62. package/examples/replays/README.md +1 -1
  63. package/examples/replays/example-multiselect-gate.cassette.json +4 -4
  64. package/examples/replays/example-pdf-skill.cassette.json +1 -1
  65. package/examples/replays/hostloop-computer-links.cassette.json +1 -1
  66. package/llms.txt +4 -0
  67. package/package.json +4 -4
  68. package/python/README.md +6 -4
  69. package/python/cowork_harness.py +4 -4
  70. package/python/test_scenario_lint.py +55 -0
  71. package/schema/run-result.json +80 -48
  72. package/schema/scenario.schema.json +123 -3
  73. package/schema/session.schema.json +23 -1
  74. package/schema/verify-cassettes.json +1 -1
  75. package/scripts/gen-schema.ts +56 -0
@@ -3,8 +3,8 @@ name: cowork-harness
3
3
  description: Test or debug a Claude Code skill/plugin under Claude Cowork's runtime — sandboxed agent, default-deny egress, the can_use_tool permission/question protocol — using the cowork-harness CLI. Use when validating or regression-testing a skill, authoring or debugging a scenario YAML (prompt + scripted answers + assert:), choosing a fidelity tier, scripting AskUserQuestion / tool-permission answers, or asserting artifacts, egress, or sub-agent dispatch. Especially when a harness run no-ops an assertion, fails on an unanswered gate, false-greens, a steered answer never reaches the model, or a web_fetch is unexpectedly denied or gated. Also when iterating or hardening a skill across fixes, or grounding a skill's self-critique against its own run evidence — including a document-analysis skill (cap table, deck, financial model, transcript) that needs an uploaded file attached to be critiqued at all. NOT for generic unit testing (pytest/vitest of your own scripts) or non-Cowork CI. Covers the skill / run / chat / record / replay / trace / decide / assertions / scaffold commands and the session-vs-scenario split.
4
4
  metadata:
5
5
  author: cowork-harness
6
- version: 1.16.0
7
- tracks-harness: cowork-harness 1.16.0 (baseline desktop-1.24012.9)
6
+ version: 1.18.0
7
+ tracks-harness: cowork-harness 1.18.0 (baseline desktop-1.25927.0)
8
8
  ---
9
9
 
10
10
  # cowork-harness
@@ -22,8 +22,8 @@ flagged with a loud `::warning::`, not silent — auto-answer a gate, observe an
22
22
  allowlist). This skill exists mostly to keep you out of those traps — the Gotchas section below is
23
23
  the highest-value part. Read it.
24
24
 
25
- > **Version note:** the facts and `file:line` pointers here track `cowork-harness 1.16.0` (baseline
26
- > `desktop-1.24012.9`). If your checkout is newer, prefer the live `--help` and — in a repo checkout —
25
+ > **Version note:** the facts and `file:line` pointers here track `cowork-harness 1.18.0` (baseline
26
+ > `desktop-1.25927.0`). If your checkout is newer, prefer the live `--help` and — in a repo checkout —
27
27
  > `SPEC.md` / `docs/*.md` over this snapshot, and re-run the bundled linter.
28
28
 
29
29
  ## Preflight — make sure the harness can actually run
@@ -39,7 +39,7 @@ Before the first command, confirm the CLI is reachable and **fail loud** (never
39
39
 
40
40
  - **One-shot check.** Run `cowork-harness doctor [--tier <tier>]` first — a read-only prerequisite check that inspects Docker, the staged agent, the token, and the baseline in one pass. The bullets below explain each thing it checks (and how to fix it).
41
41
  - **Replay-only? Skip `doctor`.** Replaying committed cassettes needs no Docker, no staged agent, and no token — and every tier's `doctor` validates the auth token (the live tiers also Docker + the staged agent), so a ✗ there is expected, not a blocker. Go straight to `cowork-harness replay <cassette>`.
42
- - **CLI on PATH, recent enough?** Run `cowork-harness --version` — this skill needs **≥ 1.16.0**. If it's missing or older, prefix every command with the version floor `npx "cowork-harness@>=1.16.0" <cmd>` (Node ≥ 22), or install once with `npm i -g "cowork-harness@>=1.16.0"`. **Pin `@>=1.16.0`, never `@latest`** — `@latest` can silently fetch an older CLI and the new commands fail as "unknown command", whereas the floor **fails loud** if no compatible version is published.
42
+ - **CLI on PATH, recent enough?** Run `cowork-harness --version` — this skill needs **≥ 1.18.0**. If it's missing or older, prefix every command with the version floor `npx "cowork-harness@>=1.18.0" <cmd>` (Node ≥ 22), or install once with `npm i -g "cowork-harness@>=1.18.0"`. **Pin `@>=1.18.0`, never `@latest`** — `@latest` can silently fetch an older CLI and the new commands fail as "unknown command", whereas the floor **fails loud** if no compatible version is published.
43
43
 
44
44
  This skill documents the CURRENT surface, not release history. If `cowork-harness --version` is
45
45
  OLDER than the floor, the per-release record of what you are missing is [CHANGELOG.md](https://github.com/yaniv-golan/cowork-harness/blob/main/CHANGELOG.md)
@@ -166,6 +166,10 @@ Exact accepted values (teach precisely): `--on-unanswered` takes `fail|prompt|fi
166
166
  only `fail|first` on `run`. **`llm` is NOT an `--on-unanswered` value** — the bare flag
167
167
  `--on-unanswered llm` is rejected (use `--decider-llm`); the YAML spelling is `on_unanswered: llm`.
168
168
  The word `agent` is **retired** — do not write `on_unanswered: agent` (the schema rejects it).
169
+ `--on-unanswered` also conflicts with `--decider-dir`/`--decider-cmd`/`--decider-llm` (the channel or
170
+ model IS the terminal, so a policy alongside it never applies) — pass one, not both. On `record`, a
171
+ scenario setting `on_unanswered: prompt` is rejected too: the YAML field outranks the flag, and a TTY
172
+ wait can't produce a deterministic committed fixture.
169
173
  `--on-unanswered first` is itself flagged `nonDeterministic` — it is *not* a deterministic stand-in
170
174
  for scripted answers. See `references/fidelity-and-answers.md`.
171
175
 
@@ -197,7 +201,7 @@ Conflating these is the **biggest landmine**. An assertion key has two independe
197
201
  `egress_*`, `file_exists`, `user_visible_artifact`, `result`) are robust. Free-text content is
198
202
  not: match prose with `transcript_matches` / `transcript_contains` (stable lexical markers only —
199
203
  not semantic content the model paraphrases, which re-records red); check structured JSON with YAML
200
- `artifact_json` (or the pytest lane for complex predicates), not via a transcript substring.
204
+ `artifact_json` (or the [pytest lane](https://github.com/yaniv-golan/cowork-harness/blob/main/python/README.md) for complex predicates), not via a transcript substring.
201
205
  - **Axis B — survives `replay`?** *Independent of Axis A.* On the token-free `replay` lane, only
202
206
  **content keys** evaluate; filesystem / egress keys are skipped (live-only) — loudly, via an
203
207
  `::warning::` annotation, not a silent no-op. A key
@@ -319,6 +323,8 @@ When the scenario declares `answers:`, verify-run **also** checks they still mat
319
323
  reworded gate or a `choose:` the run never offered fails here in ~1s instead of on a paid re-record). Or skip
320
324
  the discovery/encode/record dance entirely and answer gates **live during the recording** with
321
325
  `record --decider-dir`/`--decider-llm` (the cassette is flagged non-deterministic but replays deterministically).
326
+ `run` takes no `--dry-run`: to check that a scenario **loads** without spending, use
327
+ `cowork-harness record <file.yaml> --dry-run`; `lint` checks the assertion invariants (both above).
322
328
 
323
329
  **Author answers WITHOUT re-paying — the cheap loop.** You don't need a fresh paid record to discover a
324
330
  scenario's gates or their labels: `--keep` ONE run, then `cowork-harness trace <run-dir> --view questions`
@@ -430,9 +436,25 @@ Recognize these before "fixing" a non-bug:
430
436
  only when you thought to ask for it, and the runs that most need this are the ones where nobody did.
431
437
  **Silent when the evidence cannot answer the question** (no workspace walk, or a tier that runs no
432
438
  scratchpad walk, absent delivery telemetry, or a resumed turn) — "cannot tell" never reads as "clean".
433
- Fix by writing deliverables under `outputs/` or a connected folder, or delivering them explicitly; assert
439
+ **The fix is lane-dependent.** On `lane: local`, write deliverables under `outputs/` or a connected
440
+ folder, or deliver them explicitly. **On `lane: remote`, moving a file under `outputs/` does NOT help** —
441
+ nothing is delivered by location there, so only an explicit delivery counts. Assert
434
442
  **`allow_undelivered_deliverables: true`** when the leftovers are intentional (intermediates, caches,
435
443
  downloaded inputs) rather than a delivery gap.
444
+ - **`delivery_unobservable`** (`WARN`, `lane: remote` only) — the run produced file(s) but the harness
445
+ serves **no delivery tool on that lane**, so whether they reached the user is unanswerable. This is the
446
+ honest cannot-verify companion to `undelivered_deliverables`: reporting every remote file as undelivered
447
+ would claim more than the evidence supports, and staying silent would read as clean. Mutually exclusive
448
+ with `undelivered_deliverables`, and quiet on a run that produced nothing to deliver. Not a skill defect —
449
+ a harness coverage gap (see the *File delivery* section of fidelity-gaps).
450
+ - **`mount_delete`** (`WARN`) — a delete touched a **delete-denied mount other than `outputs`**: a `rw`
451
+ connected folder. Production denies `unlink`/`rmdir` on *every* Cowork FUSE mount until per-mount
452
+ approval, not just outputs — a connected folder shows the identical default — so this run diverged from
453
+ what production would have allowed. `WARN` rather than `FAIL` because the harness **detects** post-hoc
454
+ what production **enforces**: by the time the scan sees it, the agent already proceeded where it would
455
+ have hit `EPERM`, so failing the run would overstate what a post-hoc scan knows. Author
456
+ `no_delete_in_mounts: true` to hard-fail on it, or `allow_delete_in: ["<mount>"]` to waive that mount
457
+ (detection still runs and the hit is still recorded — the waiver is a verdict decision).
436
458
  - **`host_path_leak`** — skipped at **`hostloop` and `protocol`** fidelity (the agent runs on real host
437
459
  paths there, so a host path in model-visible text is expected, not a leak); it is *armed* at
438
460
  `container`/`microvm`, but only *fires* on an actual scanned leak with no authored
@@ -476,6 +498,19 @@ newest session's `status.json` and reads that. `--follow` fails loud on a timeou
476
498
  rather than hanging forever. (Fuller recipe in [`docs/run-status.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/run-status.md) — repo-only, not in the installed
477
499
  payload; `cowork-harness status --help` has the flags.)
478
500
 
501
+ **Poll with `--follow`, not with a shell loop over `status`'s stdout.** The one-shot text form prints to
502
+ **stderr** and writes nothing to stdout; `--output-format json` (one envelope) and `--follow` (one JSON
503
+ line per status change) are the **stdout** forms. A poll that greps `status`'s stdout therefore matches
504
+ nothing, exits 1, and returns instantly against a run with minutes left to go — a silent false "done":
505
+
506
+ ```bash
507
+ # WRONG — stdout is empty, so grep exits 1, `!` inverts it, and the loop never sleeps.
508
+ until ! cowork-harness status "$D" | grep -q '● running'; do sleep 30; done
509
+
510
+ # RIGHT — the harness owns the poll loop and exits when the run reaches a terminal state.
511
+ cowork-harness status "$D" --follow
512
+ ```
513
+
479
514
  **A multi-minute `record`/`run` outlives a short-lived wrapper.** Don't launch a long record from a
480
515
  subagent that returns before it finishes — the returning agent tears down its process tree and kills the
481
516
  in-flight run mid-artifact-write. Run it foreground, or detached from any process that will exit first.
@@ -512,7 +547,7 @@ Docker, no re-record.
512
547
  | Situation | Symptom | Reach for (in order) |
513
548
  |---|---|---|
514
549
  | **The skill misbehaved** | wrong output, an unexpected gate, a denied tool, an opaque crash | `inspect` — what did it produce? · `trace <run-dir> --view <view>` — what did it actually do (tools, gates, sub-agent tree)? · `verify-run` — re-assert cheaply when only an assertion is wrong · `diff <old-run> <new-run>` — what changed since it worked · `chat` — reproduce it by hand |
515
- | **A green you don't trust** | an assert that may have tested nothing, a stale cassette, an auto-answered or decided gate | `replay --explain` — the evidence trail behind each *passing* assert · `lint` — assertions on the wrong CI lane / mixed-class keys · `verify-cassettes` — privacy + staleness over committed cassettes · the Gotchas landmine catalog — how a check passes vacuously · `run --repeat N` / `skill --repeat N` — did it pass, or pass once? · `stats` — flaky or expensive over time |
550
+ | **A green you don't trust** | an assert that may have tested nothing, a stale cassette, an auto-answered or decided gate | `replay --explain` — the evidence trail behind each *passing* assert · `replay --mutate` — perturbs each recorded JSON artifact value and reports which perturbations NOTHING caught (reporting only; never moves the verdict/exit code) · `lint` — assertions on the wrong CI lane / mixed-class keys · `verify-cassettes` — privacy + staleness over committed cassettes · the Gotchas landmine catalog — how a check passes vacuously · `run --repeat N` / `skill --repeat N` — did it pass, or pass once? · `stats` — flaky or expensive over time |
516
551
 
517
552
  A failed run also records `errorSource` (where the failure originated) and `stderrLogPath` (the captured
518
553
  agent stderr) — read those before re-running; a re-record rarely tells you more than the captured stderr
@@ -693,11 +728,17 @@ repeats the assertion/replay-relevant ones alongside the schema (a scoped subset
693
728
  `mode:r` mounts get a real `:ro` bind (a write fails in-guest). But `rw` vs `rwd`
694
729
  (write-but-no-delete on `outputs/` / connected folders) is *not* mount-enforced **in the harness** —
695
730
  `rm` succeeds and is only caught post-hoc by `no_delete_in_outputs`. **Real Cowork enforces it live:**
696
- `rm` under `outputs/` fails `Operation not permitted`, and a skill must request approval via
697
- `allow_cowork_file_delete` to delete. Two consequences: a skill should not stage disposable scratch
731
+ outputs is a FUSE mount, and `unlink`/`rmdir` fail `Operation not permitted`; a skill must request
732
+ approval via `allow_cowork_file_delete` (which re-mounts the folder `rwd` mid-session) to delete.
733
+ **Only unlinking is denied.** Emptying a file in place — `truncate -s 0`, `> file`, `shred` without
734
+ `-u` — and renaming *within* outputs both SUCCEED in production, so the harness does not flag them
735
+ either. Renaming a file OUT of outputs fails (`EXDEV`, then `EPERM` on the copy-then-unlink
736
+ fallback), so that stays a delete. Two consequences: a skill should not stage disposable scratch
698
737
  under `outputs/` (in production, cleanup there costs an approval prompt), and a skill's
699
738
  "catch-EPERM-then-request-approval" branch cannot be exercised at any harness tier (the `rm` just
700
739
  succeeds here). Do not read this gotcha as "delete-deny may not be real in production" — it is real.
740
+ If a scenario's deletion IS intended, assert `allow_outputs_delete: true` rather than dropping
741
+ `no_delete_in_outputs` — omitting it does not permit anything.
701
742
 
702
743
  9. **Keep `.env` out of any mounted folder** — it is copied into the sandbox and the token could
703
744
  leak. Put it at a working-dir or install root (token resolution: env > `--dotenv` > `./.env` >
@@ -847,6 +888,14 @@ repeats the assertion/replay-relevant ones alongside the schema (a scoped subset
847
888
  ([`docs/fidelity-gaps.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/fidelity-gaps.md)
848
889
  → "File delivery" has the binary-verified detail; repo-only.)
849
890
 
891
+ 25. **Two distinct host-inventory consent flags — a record-time one and a verify-time one.** `record
892
+ --allow-host-inventory-fixture` is the boolean consent to proceed recording a host-inheriting
893
+ (`protocol`/`hostloop`/`cowork`-resolving-to-hostloop) cassette into a repo-visible path — otherwise
894
+ `record` refuses (freezing this machine's MCP servers/agents/account into a committed fixture is the
895
+ risk). `verify-cassettes --allow-host-inventory <regex>` is unrelated: a per-finding suppressor for the
896
+ scanner's `host-inventory` class on an already-committed cassette. Passing one where the other command
897
+ wants it fails as an unrecognized flag — they don't interchange. Depth: `references/ci-recipe.md`.
898
+
850
899
  For the assertion catalog, the YAML schema, the fidelity/answer tables, and the CI recipe, read the
851
900
  files in `references/` (the gotchas above are the full list; the references repeat only the
852
901
  assertion/replay-relevant ones).
@@ -1,6 +1,6 @@
1
1
  # CI recipe — replay vs live lanes
2
2
 
3
- Self-contained reference. Tracks `cowork-harness 1.16.0` (baseline `desktop-1.24012.9`).
3
+ Self-contained reference. Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`).
4
4
 
5
5
  **Fastest path: the packaged Action.** One step gets you `replay`/`lint`/`verify-cassettes` plus a PR
6
6
  job-summary reporter (verdict table, staleness findings, cost/turns when available):
@@ -13,7 +13,7 @@ job-summary reporter (verdict table, staleness findings, cost/turns when availab
13
13
  ```
14
14
 
15
15
  The Action's `version` input defaults to `latest` — intentional so a copy-pasted recipe tracks the current
16
- release; pin an exact version (e.g. `version: "1.16.0"`) for reproducible CI.
16
+ release; pin an exact version (e.g. `version: "1.18.0"`) for reproducible CI.
17
17
 
18
18
  Reach for the manual multi-step form below only when you need per-step control the Action's inputs don't
19
19
  cover (a custom flag combination, a different runner matrix per step, or `lint`/`verify-cassettes` gated
@@ -32,7 +32,7 @@ jobs:
32
32
  - uses: actions/checkout@v4
33
33
  - name: Stage the agent binary (official channel, sha256-verified — see https://github.com/yaniv-golan/cowork-harness/blob/main/docs/maintenance.md)
34
34
  run: |
35
- V=2.1.219 # match your scenario's pinned baseline's agentVersion
35
+ V=2.1.221 # match your scenario's pinned baseline's agentVersion
36
36
  curl -fSL "https://downloads.claude.ai/claude-code-releases/$V/linux-arm64/claude" -o "$RUNNER_TEMP/claude-$V"
37
37
  chmod +x "$RUNNER_TEMP/claude-$V"
38
38
  # verify against the committed baseline's sha256 (baselines/desktop-*.json → agentBinary.sha256)
@@ -58,12 +58,25 @@ sha256-*checked* but not hard-blocking on mismatch — it's advisory for an inte
58
58
  GitHub-hosted runners, no token/Docker/agent:
59
59
 
60
60
  ```yaml
61
- - run: npm i -g "cowork-harness@>=1.16.0"
62
- - run: cowork-harness lint scenarios/*.yaml # no silent false-greens
63
- - run: cowork-harness verify-cassettes cassettes/ # privacy + staleness
61
+ - run: npm i -g "cowork-harness@>=1.18.0"
62
+ - run: cowork-harness lint scenarios/*.yaml --strict --min-severity WARN
63
+ # no silent false-greens. WITHOUT --strict this
64
+ # step cannot fail on a WARN-class rule (e.g.
65
+ # vacuous-gate-assert) — it would print the
66
+ # finding and still exit 0. --min-severity WARN
67
+ # keeps the advisory INFO class advisory.
68
+ - run: cowork-harness verify-cassettes cassettes/ # privacy + staleness — FAILS on a stale recording
69
+ # ALSO fails on a leaked host inventory: recording at
70
+ # protocol/hostloop freezes YOUR machine's MCP servers,
71
+ # agents and account org into the cassette. Record
72
+ # committed fixtures at `container` (sealed) to avoid it.
64
73
  - run: cowork-harness replay cassettes/ # token-free content/structure
65
74
  ```
66
75
 
76
+ Both lines matter: `replay` alone **warns** on a stale recording and exits 0, so dropping the
77
+ `verify-cassettes` step means a skill edit silently stops being tested. (One command instead of two:
78
+ `replay --fail-on-skill-drift`.)
79
+
67
80
  The rest of this doc explains the lane split, recording, privacy, and the full pipeline + live job.
68
81
 
69
82
  ## The core split: token-free PR gate + live nightly (self-hosted)
@@ -111,6 +124,13 @@ The split is not just about tokens — it decides **where each lane can run**:
111
124
  cowork-harness assertions --list --output-format json # every key, with its replay class
112
125
  ```
113
126
 
127
+ **`replay --mutate`** is a distinct, reporting-only diagnostic on this same lane: it perturbs each
128
+ recorded JSON artifact value one at a time, re-runs the assertions against the perturbed cassette,
129
+ and reports which perturbations NOTHING caught — those are the fields your `assert:` block leaves
130
+ unguarded. It never changes the verdict or exit code (it's coverage information, not a check), so
131
+ don't wire it into a pass/fail gate — read its `::warning::`/`::notice::` output to find `assert:`
132
+ gaps to close.
133
+
114
134
  Placing a key on this gate from a doc list rather than from `assertions --list` is how a valid
115
135
  replay-lane check ends up left off the PR gate. Filesystem/egress assertions are
116
136
  skipped on this token-free lane — loudly: replay emits an `::warning::` annotation whenever it drops
@@ -177,6 +197,12 @@ dollar figures). In a skill repo these cassettes get **committed**. So:
177
197
  (`hostloop`, `protocol`) has an **empty** redaction policy — that combination commits real host paths
178
198
  the `path` scanner then hard-fails at `verify-cassettes` time. The always-on scanner remains the
179
199
  universal net (container-tier recordings can trip it too).
200
+ - **Host-inheriting record refused by default — `--allow-host-inventory-fixture` is the consent.** A
201
+ `protocol`/`hostloop`/`cowork`-resolving-to-hostloop record into a repo-visible cassette path would
202
+ freeze THIS machine's MCP server names, agents, and account metadata into a committed fixture, so
203
+ `record` refuses outright rather than warn. Pass `--allow-host-inventory-fixture` only when the
204
+ recording session genuinely has no personal MCP servers or plugins to leak — it is a per-record
205
+ boolean consent, not a pattern.
180
206
  - **Always-on scan gate** — `verify-cassettes` flags email / currency / bare-domain / local-path /
181
207
  machine-inventory matches it finds in the committed cassettes and **exits non-zero**, so "no leak" is
182
208
  a gate, not discipline. Non-zero is not one thing, though: exit `1` means verification RAN and found a
@@ -189,6 +215,13 @@ dollar figures). In a skill repo these cassettes get **committed**. So:
189
215
  Suppress synthetic / public reference names (NVCA, Cooley GO, …) with `--allow <regex>`. (Multi-word
190
216
  proper names are NOT a default class — too noisy to gate on; add a pattern via config if your corpus
191
217
  needs it.)
218
+ - **`--allow-host-inventory <regex>` — the per-finding sibling of the record-time consent above, and a
219
+ different thing from it.** `verify-cassettes` scans a **structural** `host-inventory` class (an
220
+ `mcp_servers[].name`/`agents[]` entry outside the harness's own known set, or `account.email`/
221
+ `.organization`/`.subscriptionType`) over already-committed cassettes. `--allow-host-inventory <regex>`
222
+ suppresses a specific finding there — it is a scanner allowlist entry, not a record-time green light —
223
+ and, like `--allow-domain`/`--allow-email`/`--allow-path`/`--allow-machine-inventory`, scopes the
224
+ pattern to that one class so it can't accidentally clear an unrelated finding.
192
225
 
193
226
  ```bash
194
227
  cowork-harness verify-cassettes cassettes/ # privacy scan + staleness — exit 1 = verified & failed, exit 3 = could not verify
@@ -246,7 +279,7 @@ jobs:
246
279
  with: { node-version: '24' }
247
280
  - uses: actions/setup-python@v5
248
281
  with: { python-version: '3.x' } # python3 only — PyYAML is bundled with the linter
249
- - run: npm i -g "cowork-harness@>=1.16.0"
282
+ - run: npm i -g "cowork-harness@>=1.18.0"
250
283
  - run: cowork-harness lint scenarios/*.yaml # no-silent-false-green (needs python3; PyYAML bundled)
251
284
  - run: cowork-harness verify-cassettes cassettes/ --output-format json # privacy + staleness gate
252
285
  - run: cowork-harness replay cassettes/ --output-format json # token-free content/structure
@@ -275,7 +308,7 @@ jobs:
275
308
  echo "live=true" >> "$GITHUB_OUTPUT"
276
309
  fi
277
310
  - if: steps.guard.outputs.live == 'true'
278
- run: npm i -g "cowork-harness@>=1.16.0"
311
+ run: npm i -g "cowork-harness@>=1.18.0"
279
312
  - if: steps.guard.outputs.live == 'true'
280
313
  run: cowork-harness run scenarios/ --output-format json
281
314
  env:
@@ -302,6 +335,14 @@ scenario = `result === "success" && assertions.every(pass)`. Exit code is non-ze
302
335
  fails or a run errors, so a plain `cowork-harness run scenarios/` is already CI-ready without parsing
303
336
  JSON.
304
337
 
338
+ **For the commands in this recipe, stdout carries the machine envelope and nothing else.** Without
339
+ `--output-format json`, `run` / `record` / `replay` / `verify-cassettes` / `status` write their whole
340
+ human rendering — warnings, verdict, `status`'s summary line — to **stderr**, and stdout stays empty.
341
+ A wrapper that captures only stdout gets an empty log and, if it greps that for a state, a silent false
342
+ negative. Capture stderr for the human trail (`2> run.stderr.log`), or ask for JSON and parse stdout.
343
+ (Commands whose whole job is to print a value — `--version`, `assertions --list`, `scaffold`, `gates`,
344
+ `skill --dry-run` — write it to stdout by design, with or without the flag.)
345
+
305
346
  `verify-cassettes` emits its **own** envelope (`{command, ok, coverage, results[]}` with per-file
306
347
  `findings`/`staleness`/`unverifiable`/`notes`/`version`/`error`), published as
307
348
  `schema/verify-cassettes.json` in the npm package. `ok:false` doesn't say *why* — read the buckets, or <!-- npm-only-ok -->
@@ -1,6 +1,6 @@
1
1
  # Critique — the facts a plugin install can't otherwise reach
2
2
 
3
- Tracks `cowork-harness 1.16.0` (baseline `desktop-1.24012.9`). This is **not** a trim of the full
3
+ Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`). This is **not** a trim of the full
4
4
  [`docs/critique.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/critique.md) (repo-only —
5
5
  flags, cost, reproduction discipline, known limitations all live there). This file covers exactly what a
6
6
  plugin install cannot otherwise discover: the run-dir artifact a harvester actually reads, the report's
@@ -1,6 +1,6 @@
1
1
  # Fidelity tiers & answer paths
2
2
 
3
- Self-contained reference. Tracks `cowork-harness 1.16.0` (baseline `desktop-1.24012.9`).
3
+ Self-contained reference. Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`).
4
4
 
5
5
  ## Fidelity tiers (`fidelity:` in the scenario)
6
6
 
@@ -74,7 +74,7 @@ regressions.
74
74
  | **Spawned helper** | `--decider-cmd '<helper>'` | depends on the helper |
75
75
  | **In-band (driving agent)** | `--decider-dir <dir>` (+ a Monitor that writes responses) | depends |
76
76
 
77
- Scripted `answers:` and a terminal (`--decider-llm` / `--decider-cmd` / `--decider-dir` / `on_unanswered`) compose as a normal Chain — scripted rules resolve matched gates, the terminal answers whatever's left. What's mutually exclusive is picking **two terminals** at once (e.g. `--decider-llm` with `--decider-cmd`, or `--decider-llm` with an explicit `--on-unanswered`) — the CLI rejects that combination. The in-band path's Monitor drives two
77
+ Scripted `answers:` and a terminal (`--decider-llm` / `--decider-cmd` / `--decider-dir` / `on_unanswered`) compose as a normal Chain — scripted rules resolve matched gates, the terminal answers whatever's left. What's mutually exclusive is picking **two terminals** at once — the CLI rejects every such pairing: two channels (`--decider-dir` with `--decider-cmd`), a model terminal with a channel or a policy (`--decider-llm` with either), and a channel with a policy (`--decider-dir`/`--decider-cmd` with `--on-unanswered`). A channel or the LLM decider *is* the terminal, so an accompanying `--on-unanswered` could only ever be inert — the chain never reaches the policy terminal. The in-band path's Monitor drives two
78
78
  subcommands: `gates <dir>` (stream the pending questions) and `answer <dir> --gate N …` (reply to one).
79
79
 
80
80
  **The `--decider-llm` answer protocol.** The answering model is shown the gate's options **numbered** and
@@ -1,7 +1,7 @@
1
1
  # Scenario & session schema, assertion catalog, web_fetch, full gotchas
2
2
 
3
- Self-contained reference for authoring `cowork-harness` scenarios. Tracks `cowork-harness 1.16.0`
4
- (baseline `desktop-1.24012.9`). If your checkout is newer, prefer the live [`docs/scenario.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/scenario.md),
3
+ Self-contained reference for authoring `cowork-harness` scenarios. Tracks `cowork-harness 1.18.0`
4
+ (baseline `desktop-1.25927.0`). If your checkout is newer, prefer the live [`docs/scenario.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/scenario.md),
5
5
  [`docs/session.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/session.md), and `SPEC.md`.
6
6
 
7
7
  **Minimal scenario** — `prompt` is the only required field:
@@ -32,7 +32,7 @@ references a session for setup.
32
32
  ```yaml
33
33
  name: my-test # OPTIONAL — defaults to the filename; keys runs/<name>/
34
34
  baseline: latest # platform baseline: "latest" or "desktop-<ver>" (NOT "profile:")
35
- session: ../sessions/default.yaml # pre-prompt setup (resolved relative to THIS file)
35
+ session: ../sessions/my-session.yaml # pre-prompt setup (resolved relative to THIS file)
36
36
  fidelity: container # protocol | container | microvm | hostloop | cowork
37
37
  execution: local # OPTIONAL — orthogonal to fidelity (a privilege/sandbox tier, all
38
38
  # local): local (default) | cloud-describe (RESERVED — no runner
@@ -146,6 +146,13 @@ folders:
146
146
  uploads:
147
147
  - ~/Downloads/report.pdf
148
148
 
149
+ # connected PROJECTS (userSelectedProjectUuids) — NOT the same as a work folder above, and not the
150
+ # legacy mnt/.projects/<id> work-folder path older baselines used
151
+ projects:
152
+ - { uuid: 7f3c…, from: ~/code/myproject } # mounted READ-ONLY at mnt/.projects/<uuid>
153
+ # no mode: knob — production hardcodes ro
154
+ # never becomes the cwd; {{workspaceFolder}} stays at outputs
155
+
149
156
  # discovery: marketplaces / plugins / skills / mcp
150
157
  plugins:
151
158
  marketplaces: [] # plugin_marketplaces (git URLs or local paths)
@@ -158,7 +165,7 @@ plugins:
158
165
  skills:
159
166
  local: [] # extra host skill dirs
160
167
  suggest_enabled: true # gate 245679952 override — `mcp__skills__suggest_skills` on/off (default true)
161
- proactive_suggest_enabled: false # gate 1598976391 override — proactive description + `trigger` param (default false)
168
+ proactive_suggest_enabled: false # gate 1598976391 override — proactive description + `trigger` param; unset = synced baseline gate (ON from 1.24012.11)
162
169
  mcp:
163
170
  config: null # --mcp-config file (standard mcpServers map)
164
171
  enabled: []
@@ -274,6 +281,7 @@ same set live from the schema.
274
281
  | `file_exists: <path>` | the path exists under the run's `work/` (anchored at `mnt/`, e.g. `outputs/x.md`). For a user-facing deliverable prefer `user_visible_artifact` — with a connected folder the file lands in `mnt/<folder>` (= `{{workspaceFolder}}`), not `mnt/outputs`, so `file_exists: outputs/x.md` misses it |
275
282
  | `user_visible_artifact: <path>` | exists **and** under a user-visible root (`outputs/` + each connected folder's mount name) — the right primitive for a workspace deliverable when a folder is connected |
276
283
  | `no_delete_in_outputs: true` | no delete op touched `mnt/outputs` — **only `true` is valid**; `false` is rejected (omit to allow deletes) |
284
+ | `no_delete_in_mounts: true` | no delete op touched ANY delete-denied mount — `outputs` plus every `rw` connected folder — except those waived by `allow_delete_in`. Production denies `unlink`/`rmdir` on every such mount, so `no_delete_in_outputs` covers only part of the real rule. **only `true` is valid** |
277
285
  | `no_unexpected_files: [<glob>, …]` | every **newly created** file under a user-visible root matches ≥1 glob (workRoot-relative paths; `**` = whole path segment for any depth — use `outputs/handoff/**` for per-run subdirs); `[]` = no new files; **new-files-only** — overwriting a pre-existing file in place is invisible (use content-level producer stamping); live/verify-run without a pre-run manifest ⇒ evidence-unavailable hard-fail (live runs capture the baseline only when this key is asserted; recordings always capture); an **incomplete post-run filesystem walk** (an unreadable subtree — a permission/I-O error) also fails evidence-unavailable rather than reporting "no strays" over a partial tree — distinct from the missing-manifest case (a `--resume` run), which fails for a different reason; captured on every live sandbox tier including microvm (its outputs are snapshotted from the VM into the run dir); replay needs `cassette.preRunPaths` (≥0.24 recordings) — cassettes without it **exclude** the key with a loud warning |
278
286
  | `input_unmodified: <glob>` or `[<glob>, …]` | a single glob or a list; every **pre-existing** file (incl. uploaded files under `uploads/**`) whose workRoot-relative path matches ≥1 glob keeps an unchanged content hash after the run — the in-place-mutation companion to `no_unexpected_files`'s new-files check (`[]` is rejected by the schema — list at least one glob); a glob that matches **no** pre-run path fails loud (a typo or renamed mount would otherwise verify zero files and pass vacuously); a matched file that was deleted counts as a content change (fails); live/verify-run without a pre-run hash manifest ⇒ evidence-unavailable hard-fail (a `--resume` run); captured on every live sandbox tier including microvm; replay needs `cassette.preRunHashes` — cassettes without it **exclude** the key with a loud warning; on replay it compares against the manifest's recorded `sha256`, never a re-hash of the materialized tree |
279
287
  | `self_heal_ran: <bool>` | a plugin-root self-heal script was (not) invoked |
@@ -334,12 +342,14 @@ same set live from the schema.
334
342
  | `allow_l0_plugin_divergence: true` | verdict modifier — opt into L0/protocol plugin divergence: suppresses the default-fail when a plugin behaves differently at `protocol` (L0) fidelity than under a sandboxed tier. Live tiers only |
335
343
  | `allow_stall: true` | verdict modifier — suppresses the `stalled` default-fail when a run ends on a question having done no productive tool work after its last gate (the agent asked for input and stopped — incl. re-asking in plain text after answering an `AskUserQuestion`); assert only when ending on a question is intended, else script the answer (`answer:` / `--answer` / a decider) |
336
344
  | `allow_undelivered_deliverables: true` | verdict modifier — suppresses the `undelivered_deliverables` WARN. Working in the scratchpad is Cowork's designed pattern, so a skill that legitimately leaves intermediates, caches or downloaded inputs behind can say so instead of carrying permanent noise. The signal is warn-only and never fails a run on its own; reach for this when the scratch activity is intentional, not to silence a real delivery gap |
345
+ | `allow_outputs_delete: true` | verdict modifier — accepts a detected outputs delete instead of failing the run, for a skill whose deletion is intended. Omitting `no_delete_in_outputs` does **not** permit deletes (a detected delete fails via the `outputs_delete` signal precisely because the key was not authored), so this is the way to accept one. **Mutually exclusive** with `no_delete_in_outputs`. Waives the harness's post-hoc detection; it does not model Cowork's `allow_cowork_file_delete` approval handshake |
346
+ | `allow_delete_in: [<mount>…]` | verdict modifier — accepts detected deletes in the named mounts (the per-mount analogue of `allow_outputs_delete`, mirroring production's `fileDeleteApprovedMounts`). Waives the verdict only; detection still runs and the hits stay in `result.json`. Listing `outputs` conflicts with `no_delete_in_outputs` |
337
347
  | `transcript_no_host_path: true` | no host path (`/Users/`, `/opt/cowork/`, `/home/`, `/root/`) leaked into model-visible text — **incompatible with `hostloop` AND `protocol`**: hostloop's native file tools legitimately expose real host paths, and protocol (L0) runs the agent's file tools on the real host cwd with no sealed filesystem, so this fails BY DESIGN on both (the harness warns at run start if asserted anyway); use `container`/`microvm` for this check |
338
348
  | `egress_denied: <host>` | the host was blocked by the egress proxy |
339
349
  | `egress_allowed: <host>` | the host was allowed through |
340
350
  | `no_mcp_error: true` | no MCP round-trip failed (`RunResult.mcpErrors` is empty — no unhandled server, no handler throw) — live-only: MCP round-trips are harness-computed, not in the SDK stdout stream, so evidence-unavailable on replay (never a vacuous pass). **Only `true` is valid** |
341
351
  | `max_peak_rss_bytes: <N>` | peak sampled RSS of the agent sandbox ≤ N bytes (`RunResult.resources.peakRssBytes`) — live-only: replay never spawns a sandbox to sample, so evidence-unavailable on replay/protocol (never a vacuous pass); also evidence-unavailable when sampling captured no RSS value |
342
- | `semantic_matches: {rubric: [...], min_pass?, judge_model?}` | a pinned LLM judge grades each fixed `rubric` claim against the run's answer — the **union of the agent's final result text (`RunResult.finalMessage`), the transcript, and the final on-disk content of any files the agent authored during the run** — so a claim about content the skill led the agent to *write to a file* grades as reliably as one about inlined prose (authored-file evidence is captured on every live sandbox tier including **microvm** — its session tree is snapshotted from the VM into the run dir). When the authored-file evidence backing the judged document is **incomplete** — a file dropped at the capture-size cap, unreadable at read-back, or (on `--resume`) the scratchpad walk skipped — the assert fails evidence-unavailable rather than trusting a judge grade over a partial document; this is separate from the malformed-grade `judgeInvalid` path below. The assert passes iff ≥ `min_pass` claims pass (default: all — avoid for a gating scenario). Results align by claim index and are recorded per-claim in `RunResult.assertions[].semanticClaims` (`[{index, claim, pass}]`, so a consumer can diff the per-claim profile across runs); a rep whose grade can't be parsed (after one retry) is marked `RunResult.assertions[].judgeInvalid` and **never silently dropped** — it is excluded from the pass denominator, and the guard against a misleading score from that exclusion is the gate's minimum-valid-rep floor (`MIN_VALID` ≥ 4) plus this visibility, not a claim that denominator-shrinking inflation is impossible. Within a rep, a grade that's still unparseable after the retry **fails that assert outright** (evidence-unavailable, not a vacuous pass) — a persistently-flaky judge reds the run rather than silently passing. `judge_model` pins the grader (default when neither it nor `COWORK_HARNESS_JUDGE_MODEL` is set: `claude-opus-4-8`; a dated id keeps a before/after comparison reproducible). Live-only: the judge is a live model call, so evidence-unavailable / skipped-loud on replay (never a vacuous pass) |
352
+ | `semantic_matches: {rubric: [...], min_pass?, judge_model?, include_subagent_text?}` | a pinned LLM judge grades each fixed `rubric` claim against the run's answer — the **union of the agent's final result text (`RunResult.finalMessage`), the transcript, and the final on-disk content of any files the agent authored during the run** — so a claim about content the skill led the agent to *write to a file* grades as reliably as one about inlined prose. **"The transcript" is narrower than it reads: top-level `assistant_text` ONLY.** It excludes every `tool_use`/`tool_result` and **all sub-agent-originated text** (including fork-scoped `Skill`/`Agent(fork)` dispatches — the harness attributes their *tool* calls to the main agent, but not their text). ⚠️ **Consequence: a rubric claim about whether a tool was called can NEVER grade true** — the evidence is not in the judged document. Such a claim looks reasonable and silently caps your pass rate; assert tool use with `tool_called` / `present_files_called` / `subagent_dispatched` / `hook_blocked` instead. Sub-agent text is captured in `RunResult.subagents[].reasoning` and reaches the judge only via opt-in `include_subagent_text: true` (`kind:"text"` turns only — sub-agent *thinking* arrives empty with `redacted:true`, so it would pad the document with blanks) (authored-file evidence is captured on every live sandbox tier including **microvm** — its session tree is snapshotted from the VM into the run dir). When the authored-file evidence backing the judged document is **incomplete** — a file dropped at the capture-size cap, unreadable at read-back, or (on `--resume`) the scratchpad walk skipped — the assert fails evidence-unavailable rather than trusting a judge grade over a partial document; this is separate from the malformed-grade `judgeInvalid` path below. The assert passes iff ≥ `min_pass` claims pass (default: all — avoid for a gating scenario). Results align by claim index and are recorded per-claim in `RunResult.assertions[].semanticClaims` (`[{index, claim, pass}]`, so a consumer can diff the per-claim profile across runs); a rep whose grade can't be parsed (after one retry) is marked `RunResult.assertions[].judgeInvalid` and **never silently dropped** — it is excluded from the pass denominator, and the guard against a misleading score from that exclusion is the gate's minimum-valid-rep floor (`MIN_VALID` ≥ 4) plus this visibility, not a claim that denominator-shrinking inflation is impossible. Within a rep, a grade that's still unparseable after the retry **fails that assert outright** (evidence-unavailable, not a vacuous pass) — a persistently-flaky judge reds the run rather than silently passing. `judge_model` pins the grader (default when neither it nor `COWORK_HARNESS_JUDGE_MODEL` is set: `claude-opus-4-8`; a dated id keeps a before/after comparison reproducible). Live-only: the judge is a live model call, so evidence-unavailable / skipped-loud on replay (never a vacuous pass) |
343
353
  | `artifact_json: {artifact, path, …}` | assert a JSON artifact's contents — `equals`/`gt`/`in`/`exists`/`absent`/`is_null` over a dotted `path` (`in` = membership in a list, for a stochastic/LLM value; `absent` ≠ `is_null`; an unresolved intermediate fails loud) |
344
354
  | `computer_links_resolve: true` | every `computer://` link in the model-visible transcript resolves to an artifact that exists in the run's collected outputs/mounts — a dangling link fails, naming which target was checked (a live host path, the collected work tree, or the replay manifest). **Requires ≥1 link** (zero links fails — use `computer_links_resolve_if_present` for the presence-free variant). **Only `true` is valid** (`false` is rejected by the schema) |
345
355
  | `computer_links_resolve_if_present: true` | like `computer_links_resolve` but passes vacuously when the transcript has zero `computer://` links — the presence-free variant. **Only `true` is valid** |
@@ -369,6 +379,7 @@ codes (`VerdictSignal["code"]` in `src/run/verdict.ts`):
369
379
  | `transport_error` | fail | The connection dropped mid/after-run |
370
380
  | `permissive_auto_allow` | fail | A cowork-parity auto-allow real Cowork would block (opt out: `allow_permissive_auto_allow`) |
371
381
  | `outputs_delete` | fail | An unauthorized delete touched `mnt/outputs` (opt out: author `no_delete_in_outputs`) |
382
+ | `mount_delete` | warn | A delete touched a delete-denied mount other than `outputs` (a `rw` connected folder). Production denies `unlink`/`rmdir` there until per-mount approval, so the run diverged. Warn, not fail: the harness detects post-hoc what production enforces. Author `no_delete_in_mounts` to hard-fail, or `allow_delete_in` to waive |
372
383
  | `host_path_leak` | fail | A host path leaked into model-visible text (opt out: author `transcript_no_host_path`) |
373
384
  | `l0_plugin_divergence` | fail | L0/protocol plugin loading diverged from Cowork (opt out: `allow_l0_plugin_divergence`) |
374
385
  | `missing_capability` | fail | A `requires_capabilities` need was unmet, or the skill used a capability the image omits (opt out: `allow_missing_capability`, or `skill --allow-missing-capability` on an open-ended run) |
@@ -378,7 +389,8 @@ codes (`VerdictSignal["code"]` in `src/run/verdict.ts`):
378
389
  | `prompt_asset_missing` | warn | The run proceeded with a missing prompt asset (`COWORK_HARNESS_ALLOW_MISSING_PROMPT=1`); fidelity is degraded |
379
390
  | `scan_unavailable` | warn | Post-run scan evidence unavailable (`RunResult.scan` undefined) — the host-path and outputs-delete guards did not run this run |
380
391
  | `ended_with_question` | warn | Live-lane heuristic: the final answer contains a question and the run wrote no deliverable to `outputs/` — the lenient sibling of `stalled` (covers a mid-message `?`, or tool work after the last gate that still ended asking). Opt out: `allow_stall` |
381
- | `undelivered_deliverables` | warn | The skill produced file(s) OUTSIDE every user-visible root and never delivered them — on a remote Cowork session the workspace is reclaimed at session end; on a local one they stay invisible to the user. Fires on every run without opting in, because the scenarios that most need it are the ones whose author never considered delivery. Silent when the evidence cannot answer the question (no workspace walk, a tier that runs no scratchpad walk, absent delivery telemetry, or a resumed turn) — never a vacuous clean. Opt out: `allow_undelivered_deliverables` |
392
+ | `undelivered_deliverables` | warn | The skill produced file(s) OUTSIDE every user-visible root and never delivered them, so they stay invisible to the user. Fires on every run without opting in, because the scenarios that most need it are the ones whose author never considered delivery. Silent when the evidence cannot answer the question (no workspace walk, a tier that runs no scratchpad walk, absent delivery telemetry, a resumed turn, or a lane where delivery is unobservable — see `delivery_unobservable`) — never a vacuous clean. **`lane: local` only**: on remote, delivery cannot be measured at all, so that lane reports `delivery_unobservable` instead of guessing. Opt out: `allow_undelivered_deliverables` |
393
+ | `delivery_unobservable` | warn | `lane: remote` only — the run produced file(s), but whether any reached the user CANNOT be verified: nothing is delivered by location on that lane and the harness models no remote delivery tool (production uses the agent-native `SendUserFile`). The honest counterpart to `undelivered_deliverables`, which would otherwise fire on every remote run that writes anything — a signal that always fires carries no information. Mutually exclusive with it; quiet when the run produced nothing to deliver. A harness coverage gap, not a skill defect. Opt out: `allow_undelivered_deliverables` |
382
394
  | `exec_infra_error` | warn | Host-loop: one or more container `exec` calls failed for infrastructure reasons, so those tool calls returned an error to the agent. Warns rather than fails because the run's other evidence is intact — unlike `infra_error`, where a dead supervisor contaminates everything. Caveat: if *every* exec failed, the agent ran nothing and this still only warns — check `result.infraErrors` |
383
395
 
384
396
  A **fail**-severity signal does not change `result.result` (still `"success"`), but it DOES fail the
@@ -390,12 +402,12 @@ overall run verdict and exit code — `assert result: success` alone won't catch
390
402
  A cassette (`record`/`replay`) has **no filesystem and no network**. `replay` re-evaluates only the
391
403
  **content** assertions. The authoritative list is `ALWAYS_CONTENT_KEYS`/`QUESTION_GATE_KEYS`/`MANIFEST_KEYS` (composed) in `src/run/cassette.ts`.
392
404
 
393
- **Scenario source — the WHOLE scenario is frozen; only `assert:` can be opted back to disk.** A cassette
394
- captures every key (`name`/`prompt`/`session`/`baseline`/`fidelity`/`lane`/`skills`/`answers`/`execution`/
395
- `requires_capabilities`/`expect_denied`/`assert`), and a plain `replay` evaluates **all** of them from that
396
- frozen copy (byte-deterministic, ignores the working tree); editing `scenarios/<name>.yaml` does not change
397
- it — replay only prints a `::notice::` when a sibling's `assert:`/`prompt:` differs, or when it fails to
398
- load at all. An edited `lane:`/`fidelity:`/`baseline:` reaches a replay ONLY by re-recording. `replay --assert-from
405
+ **Scenario source — the WHOLE scenario is frozen; only `assert:` (+`expect_denied:`) can be opted back to
406
+ disk.** A cassette captures every key (`name`/`prompt`/`session`/`baseline`/`fidelity`/`lane`/`skills`/
407
+ `answers`/`execution`/`requires_capabilities`/`expect_denied`/`assert`), and a plain `replay` evaluates
408
+ **all** of them from that frozen copy (byte-deterministic, ignores the working tree); editing
409
+ `scenarios/<name>.yaml` does not change it — replay only prints a `::notice::` when a sibling's
410
+ `assert:`/`prompt:` differs, or when it fails to load at all. An edited `lane:`/`fidelity:`/`baseline:` reaches a replay ONLY by re-recording. `replay --assert-from
399
411
  <scenario.yaml>` / `--reassert` is the opt-in token-free re-check against the on-disk block; it **hard-fails**
400
412
  on recording-shaping drift (`prompt`/`answers`/`baseline`/`skills`) or skill-content staleness (it implies
401
413
  `--fail-on-skill-drift`). `expect_denied`/filesystem/egress keys are sourced from on-disk but stay live-only —
@@ -2,7 +2,7 @@
2
2
 
3
3
  Each recipe composes facts that live scattered across SKILL.md and the other references into one
4
4
  decision path. Every one answers a question a real fleet owner had to work out the hard way.
5
- Tracks `cowork-harness 1.16.0` (baseline `desktop-1.24012.9`), same as SKILL.md's front-matter. Recipe 2's `resolved-tier`/`unverifiable-tier` staleness classes and
5
+ Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`), same as SKILL.md's front-matter. Recipe 2's `resolved-tier`/`unverifiable-tier` staleness classes and
6
6
  Recipe 3's `init-redact` shipped in 0.24.0 and are part of the current feature set — no version gate
7
7
  needed if your CLI meets SKILL.md's version floor.
8
8
 
@@ -131,7 +131,8 @@ degrade the advice. It is real work to calibrate; these steps are the traps that
131
131
  1. **Author a suite of Q&A scenarios — one per representative question.** Each installs your skill and
132
132
  asserts the answer with a rubric. The judge grades the **union of the agent's final answer, the
133
133
  transcript, and any files it wrote**, so a claim about content your skill leads the agent to *write to a
134
- file* grades as reliably as one about inlined prose — you don't have to force an inline answer:
134
+ file* grades as reliably as one about inlined prose — you don't have to force an inline answer.
135
+ **But "the transcript" is narrower than it sounds — see step 2a before writing a claim about tool use:**
135
136
  ```yaml
136
137
  session: ./_session.yaml # plugins.local_plugins + enabled: [<your-skill>@local]
137
138
  fidelity: container
@@ -142,6 +143,20 @@ degrade the advice. It is real work to calibrate; these steps are the traps that
142
143
  - <one discrete, checkable claim a correct answer MUST make>
143
144
  - <another>
144
145
  ```
146
+ 2a. **NEVER write a claim about whether a TOOL was called — the judge cannot see tool calls.** The
147
+ "transcript" the judge receives is **top-level assistant prose only**. It excludes every `tool_use`
148
+ and `tool_result`, and it excludes **all sub-agent text** (including `Skill` and `Agent(fork)`
149
+ dispatches, whose *tool* calls the harness does attribute to the main agent — the text path does not).
150
+ Sub-agent text is captured in `RunResult.subagents[].reasoning` but is **not** sent to the judge
151
+ unless you opt in with `include_subagent_text: true`.
152
+
153
+ So a rubric claim like *"the agent used a tool to surface the file, or said none was available"* has a
154
+ first branch that **can never grade true**, no matter how the skill behaves — the evidence simply
155
+ isn't in the document. Such a claim looks reasonable, survives drafting, and silently caps your pass
156
+ rate. Assert tool use with the structural keys instead (`tool_called`, `present_files_called`,
157
+ `subagent_dispatched`, `hook_blocked`), and reserve `semantic_matches` for what the agent *said* or
158
+ *wrote*. For a fan-out skill whose real work happens in sub-agents, add `include_subagent_text: true`.
159
+
145
160
  2. **Write DISCRIMINATING claims, and verify each against ground truth — not memory.** A claim that
146
161
  contradicts how the tool actually behaves can *never* pass (the correct skill will contradict it), and
147
162
  it silently poisons the gate. Check each claim against the code/docs. Decompose into single,
@@ -2,8 +2,10 @@
2
2
  "$comment": "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
3
3
  "keys": [
4
4
  "all_tasks_completed",
5
+ "allow_delete_in",
5
6
  "allow_l0_plugin_divergence",
6
7
  "allow_missing_capability",
8
+ "allow_outputs_delete",
7
9
  "allow_permissive_auto_allow",
8
10
  "allow_stall",
9
11
  "allow_undelivered_deliverables",
@@ -26,6 +28,7 @@
26
28
  "max_tokens",
27
29
  "max_tool_errors",
28
30
  "max_turns",
31
+ "no_delete_in_mounts",
29
32
  "no_delete_in_outputs",
30
33
  "no_hook_blocked",
31
34
  "no_lost_write_back",
@@ -91,10 +94,26 @@
91
94
  "timeout_ms"
92
95
  ],
93
96
  "verdictModifierKeys": [
97
+ "allow_delete_in",
94
98
  "allow_l0_plugin_divergence",
95
99
  "allow_missing_capability",
100
+ "allow_outputs_delete",
96
101
  "allow_permissive_auto_allow",
97
102
  "allow_stall",
98
103
  "allow_undelivered_deliverables"
104
+ ],
105
+ "servedHookEvents": [
106
+ "PreToolUse"
107
+ ],
108
+ "knownHookEvents": [
109
+ "Notification",
110
+ "PostToolUse",
111
+ "PreCompact",
112
+ "PreToolUse",
113
+ "SessionEnd",
114
+ "SessionStart",
115
+ "Stop",
116
+ "SubagentStop",
117
+ "UserPromptSubmit"
99
118
  ]
100
119
  }