cowork-harness 1.16.0 → 1.18.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/cowork-harness/SKILL.md +59 -10
- package/.claude/skills/cowork-harness/references/ci-recipe.md +49 -8
- package/.claude/skills/cowork-harness/references/critique.md +1 -1
- package/.claude/skills/cowork-harness/references/fidelity-and-answers.md +2 -2
- package/.claude/skills/cowork-harness/references/scenario-schema.md +24 -12
- package/.claude/skills/cowork-harness/references/task-recipes.md +17 -2
- package/.claude/skills/cowork-harness/scripts/assertion-keys.json +19 -0
- package/.claude/skills/cowork-harness/scripts/scenario.py +164 -0
- package/CHANGELOG.md +485 -0
- package/CONTRIBUTING.md +1 -0
- package/DESIGN.md +4 -4
- package/README.md +26 -20
- package/SPEC.md +7 -4
- package/baselines/desktop-1.24012.11.json +546 -0
- package/baselines/desktop-1.24012.9.json +60 -21
- package/baselines/desktop-1.25927.0.json +547 -0
- package/dist/agent/session.js +79 -7
- package/dist/assert.js +66 -4
- package/dist/cli.js +193 -45
- package/dist/critique/command.js +4 -1
- package/dist/critique/package-evidence.js +8 -1
- package/dist/decide/decider.js +16 -7
- package/dist/hostloop/skills-handler.js +48 -11
- package/dist/loop-decision.js +36 -13
- package/dist/run/analyze-artifact.js +1 -1
- package/dist/run/analyze-skill.js +32 -1
- package/dist/run/cassette.js +513 -16
- package/dist/run/chat.js +3 -3
- package/dist/run/execute.js +296 -80
- package/dist/run/hook-events.js +115 -0
- package/dist/run/inputs.js +17 -5
- package/dist/run/mutate.js +197 -0
- package/dist/run/pre-run-manifest.js +2 -2
- package/dist/run/renderer.js +15 -8
- package/dist/run/subagent-reasoning.js +86 -17
- package/dist/run/trace-view.js +11 -2
- package/dist/run/verdict.js +85 -3
- package/dist/runtime/container.js +2 -2
- package/dist/runtime/hostloop-stage.js +4 -5
- package/dist/runtime/hostloop.js +2 -2
- package/dist/scan.js +100 -0
- package/dist/session.js +55 -4
- package/dist/sync/baseline-diff.js +21 -0
- package/dist/sync/cowork-sync.js +518 -101
- package/dist/types.js +79 -6
- package/docs/README.md +1 -1
- package/docs/boundary.md +2 -0
- package/docs/cassette.md +104 -4
- package/docs/cowork-spawn-contract-1.12603.1.md +3 -1
- package/docs/critique.md +24 -6
- package/docs/debugging.md +1 -1
- package/docs/decider-dir.md +69 -2
- package/docs/fidelity-gaps.md +122 -6
- package/docs/gotchas.md +19 -3
- package/docs/maintenance.md +58 -1
- package/docs/protocol.md +42 -3
- package/docs/run-status.md +14 -1
- package/docs/scenario.md +78 -12
- package/docs/session.md +24 -4
- package/docs/subagents.md +71 -27
- package/examples/README.md +11 -5
- package/examples/replays/README.md +1 -1
- package/examples/replays/example-multiselect-gate.cassette.json +4 -4
- package/examples/replays/example-pdf-skill.cassette.json +1 -1
- package/examples/replays/hostloop-computer-links.cassette.json +1 -1
- package/llms.txt +4 -0
- package/package.json +4 -4
- package/python/README.md +6 -4
- package/python/cowork_harness.py +4 -4
- package/python/test_scenario_lint.py +55 -0
- package/schema/run-result.json +80 -48
- package/schema/scenario.schema.json +123 -3
- package/schema/session.schema.json +23 -1
- package/schema/verify-cassettes.json +1 -1
- package/scripts/gen-schema.ts +56 -0
|
@@ -3,8 +3,8 @@ name: cowork-harness
|
|
|
3
3
|
description: Test or debug a Claude Code skill/plugin under Claude Cowork's runtime — sandboxed agent, default-deny egress, the can_use_tool permission/question protocol — using the cowork-harness CLI. Use when validating or regression-testing a skill, authoring or debugging a scenario YAML (prompt + scripted answers + assert:), choosing a fidelity tier, scripting AskUserQuestion / tool-permission answers, or asserting artifacts, egress, or sub-agent dispatch. Especially when a harness run no-ops an assertion, fails on an unanswered gate, false-greens, a steered answer never reaches the model, or a web_fetch is unexpectedly denied or gated. Also when iterating or hardening a skill across fixes, or grounding a skill's self-critique against its own run evidence — including a document-analysis skill (cap table, deck, financial model, transcript) that needs an uploaded file attached to be critiqued at all. NOT for generic unit testing (pytest/vitest of your own scripts) or non-Cowork CI. Covers the skill / run / chat / record / replay / trace / decide / assertions / scaffold commands and the session-vs-scenario split.
|
|
4
4
|
metadata:
|
|
5
5
|
author: cowork-harness
|
|
6
|
-
version: 1.
|
|
7
|
-
tracks-harness: cowork-harness 1.
|
|
6
|
+
version: 1.18.0
|
|
7
|
+
tracks-harness: cowork-harness 1.18.0 (baseline desktop-1.25927.0)
|
|
8
8
|
---
|
|
9
9
|
|
|
10
10
|
# cowork-harness
|
|
@@ -22,8 +22,8 @@ flagged with a loud `::warning::`, not silent — auto-answer a gate, observe an
|
|
|
22
22
|
allowlist). This skill exists mostly to keep you out of those traps — the Gotchas section below is
|
|
23
23
|
the highest-value part. Read it.
|
|
24
24
|
|
|
25
|
-
> **Version note:** the facts and `file:line` pointers here track `cowork-harness 1.
|
|
26
|
-
> `desktop-1.
|
|
25
|
+
> **Version note:** the facts and `file:line` pointers here track `cowork-harness 1.18.0` (baseline
|
|
26
|
+
> `desktop-1.25927.0`). If your checkout is newer, prefer the live `--help` and — in a repo checkout —
|
|
27
27
|
> `SPEC.md` / `docs/*.md` over this snapshot, and re-run the bundled linter.
|
|
28
28
|
|
|
29
29
|
## Preflight — make sure the harness can actually run
|
|
@@ -39,7 +39,7 @@ Before the first command, confirm the CLI is reachable and **fail loud** (never
|
|
|
39
39
|
|
|
40
40
|
- **One-shot check.** Run `cowork-harness doctor [--tier <tier>]` first — a read-only prerequisite check that inspects Docker, the staged agent, the token, and the baseline in one pass. The bullets below explain each thing it checks (and how to fix it).
|
|
41
41
|
- **Replay-only? Skip `doctor`.** Replaying committed cassettes needs no Docker, no staged agent, and no token — and every tier's `doctor` validates the auth token (the live tiers also Docker + the staged agent), so a ✗ there is expected, not a blocker. Go straight to `cowork-harness replay <cassette>`.
|
|
42
|
-
- **CLI on PATH, recent enough?** Run `cowork-harness --version` — this skill needs **≥ 1.
|
|
42
|
+
- **CLI on PATH, recent enough?** Run `cowork-harness --version` — this skill needs **≥ 1.18.0**. If it's missing or older, prefix every command with the version floor `npx "cowork-harness@>=1.18.0" <cmd>` (Node ≥ 22), or install once with `npm i -g "cowork-harness@>=1.18.0"`. **Pin `@>=1.18.0`, never `@latest`** — `@latest` can silently fetch an older CLI and the new commands fail as "unknown command", whereas the floor **fails loud** if no compatible version is published.
|
|
43
43
|
|
|
44
44
|
This skill documents the CURRENT surface, not release history. If `cowork-harness --version` is
|
|
45
45
|
OLDER than the floor, the per-release record of what you are missing is [CHANGELOG.md](https://github.com/yaniv-golan/cowork-harness/blob/main/CHANGELOG.md)
|
|
@@ -166,6 +166,10 @@ Exact accepted values (teach precisely): `--on-unanswered` takes `fail|prompt|fi
|
|
|
166
166
|
only `fail|first` on `run`. **`llm` is NOT an `--on-unanswered` value** — the bare flag
|
|
167
167
|
`--on-unanswered llm` is rejected (use `--decider-llm`); the YAML spelling is `on_unanswered: llm`.
|
|
168
168
|
The word `agent` is **retired** — do not write `on_unanswered: agent` (the schema rejects it).
|
|
169
|
+
`--on-unanswered` also conflicts with `--decider-dir`/`--decider-cmd`/`--decider-llm` (the channel or
|
|
170
|
+
model IS the terminal, so a policy alongside it never applies) — pass one, not both. On `record`, a
|
|
171
|
+
scenario setting `on_unanswered: prompt` is rejected too: the YAML field outranks the flag, and a TTY
|
|
172
|
+
wait can't produce a deterministic committed fixture.
|
|
169
173
|
`--on-unanswered first` is itself flagged `nonDeterministic` — it is *not* a deterministic stand-in
|
|
170
174
|
for scripted answers. See `references/fidelity-and-answers.md`.
|
|
171
175
|
|
|
@@ -197,7 +201,7 @@ Conflating these is the **biggest landmine**. An assertion key has two independe
|
|
|
197
201
|
`egress_*`, `file_exists`, `user_visible_artifact`, `result`) are robust. Free-text content is
|
|
198
202
|
not: match prose with `transcript_matches` / `transcript_contains` (stable lexical markers only —
|
|
199
203
|
not semantic content the model paraphrases, which re-records red); check structured JSON with YAML
|
|
200
|
-
`artifact_json` (or the pytest lane for complex predicates), not via a transcript substring.
|
|
204
|
+
`artifact_json` (or the [pytest lane](https://github.com/yaniv-golan/cowork-harness/blob/main/python/README.md) for complex predicates), not via a transcript substring.
|
|
201
205
|
- **Axis B — survives `replay`?** *Independent of Axis A.* On the token-free `replay` lane, only
|
|
202
206
|
**content keys** evaluate; filesystem / egress keys are skipped (live-only) — loudly, via an
|
|
203
207
|
`::warning::` annotation, not a silent no-op. A key
|
|
@@ -319,6 +323,8 @@ When the scenario declares `answers:`, verify-run **also** checks they still mat
|
|
|
319
323
|
reworded gate or a `choose:` the run never offered fails here in ~1s instead of on a paid re-record). Or skip
|
|
320
324
|
the discovery/encode/record dance entirely and answer gates **live during the recording** with
|
|
321
325
|
`record --decider-dir`/`--decider-llm` (the cassette is flagged non-deterministic but replays deterministically).
|
|
326
|
+
`run` takes no `--dry-run`: to check that a scenario **loads** without spending, use
|
|
327
|
+
`cowork-harness record <file.yaml> --dry-run`; `lint` checks the assertion invariants (both above).
|
|
322
328
|
|
|
323
329
|
**Author answers WITHOUT re-paying — the cheap loop.** You don't need a fresh paid record to discover a
|
|
324
330
|
scenario's gates or their labels: `--keep` ONE run, then `cowork-harness trace <run-dir> --view questions`
|
|
@@ -430,9 +436,25 @@ Recognize these before "fixing" a non-bug:
|
|
|
430
436
|
only when you thought to ask for it, and the runs that most need this are the ones where nobody did.
|
|
431
437
|
**Silent when the evidence cannot answer the question** (no workspace walk, or a tier that runs no
|
|
432
438
|
scratchpad walk, absent delivery telemetry, or a resumed turn) — "cannot tell" never reads as "clean".
|
|
433
|
-
|
|
439
|
+
**The fix is lane-dependent.** On `lane: local`, write deliverables under `outputs/` or a connected
|
|
440
|
+
folder, or deliver them explicitly. **On `lane: remote`, moving a file under `outputs/` does NOT help** —
|
|
441
|
+
nothing is delivered by location there, so only an explicit delivery counts. Assert
|
|
434
442
|
**`allow_undelivered_deliverables: true`** when the leftovers are intentional (intermediates, caches,
|
|
435
443
|
downloaded inputs) rather than a delivery gap.
|
|
444
|
+
- **`delivery_unobservable`** (`WARN`, `lane: remote` only) — the run produced file(s) but the harness
|
|
445
|
+
serves **no delivery tool on that lane**, so whether they reached the user is unanswerable. This is the
|
|
446
|
+
honest cannot-verify companion to `undelivered_deliverables`: reporting every remote file as undelivered
|
|
447
|
+
would claim more than the evidence supports, and staying silent would read as clean. Mutually exclusive
|
|
448
|
+
with `undelivered_deliverables`, and quiet on a run that produced nothing to deliver. Not a skill defect —
|
|
449
|
+
a harness coverage gap (see the *File delivery* section of fidelity-gaps).
|
|
450
|
+
- **`mount_delete`** (`WARN`) — a delete touched a **delete-denied mount other than `outputs`**: a `rw`
|
|
451
|
+
connected folder. Production denies `unlink`/`rmdir` on *every* Cowork FUSE mount until per-mount
|
|
452
|
+
approval, not just outputs — a connected folder shows the identical default — so this run diverged from
|
|
453
|
+
what production would have allowed. `WARN` rather than `FAIL` because the harness **detects** post-hoc
|
|
454
|
+
what production **enforces**: by the time the scan sees it, the agent already proceeded where it would
|
|
455
|
+
have hit `EPERM`, so failing the run would overstate what a post-hoc scan knows. Author
|
|
456
|
+
`no_delete_in_mounts: true` to hard-fail on it, or `allow_delete_in: ["<mount>"]` to waive that mount
|
|
457
|
+
(detection still runs and the hit is still recorded — the waiver is a verdict decision).
|
|
436
458
|
- **`host_path_leak`** — skipped at **`hostloop` and `protocol`** fidelity (the agent runs on real host
|
|
437
459
|
paths there, so a host path in model-visible text is expected, not a leak); it is *armed* at
|
|
438
460
|
`container`/`microvm`, but only *fires* on an actual scanned leak with no authored
|
|
@@ -476,6 +498,19 @@ newest session's `status.json` and reads that. `--follow` fails loud on a timeou
|
|
|
476
498
|
rather than hanging forever. (Fuller recipe in [`docs/run-status.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/run-status.md) — repo-only, not in the installed
|
|
477
499
|
payload; `cowork-harness status --help` has the flags.)
|
|
478
500
|
|
|
501
|
+
**Poll with `--follow`, not with a shell loop over `status`'s stdout.** The one-shot text form prints to
|
|
502
|
+
**stderr** and writes nothing to stdout; `--output-format json` (one envelope) and `--follow` (one JSON
|
|
503
|
+
line per status change) are the **stdout** forms. A poll that greps `status`'s stdout therefore matches
|
|
504
|
+
nothing, exits 1, and returns instantly against a run with minutes left to go — a silent false "done":
|
|
505
|
+
|
|
506
|
+
```bash
|
|
507
|
+
# WRONG — stdout is empty, so grep exits 1, `!` inverts it, and the loop never sleeps.
|
|
508
|
+
until ! cowork-harness status "$D" | grep -q '● running'; do sleep 30; done
|
|
509
|
+
|
|
510
|
+
# RIGHT — the harness owns the poll loop and exits when the run reaches a terminal state.
|
|
511
|
+
cowork-harness status "$D" --follow
|
|
512
|
+
```
|
|
513
|
+
|
|
479
514
|
**A multi-minute `record`/`run` outlives a short-lived wrapper.** Don't launch a long record from a
|
|
480
515
|
subagent that returns before it finishes — the returning agent tears down its process tree and kills the
|
|
481
516
|
in-flight run mid-artifact-write. Run it foreground, or detached from any process that will exit first.
|
|
@@ -512,7 +547,7 @@ Docker, no re-record.
|
|
|
512
547
|
| Situation | Symptom | Reach for (in order) |
|
|
513
548
|
|---|---|---|
|
|
514
549
|
| **The skill misbehaved** | wrong output, an unexpected gate, a denied tool, an opaque crash | `inspect` — what did it produce? · `trace <run-dir> --view <view>` — what did it actually do (tools, gates, sub-agent tree)? · `verify-run` — re-assert cheaply when only an assertion is wrong · `diff <old-run> <new-run>` — what changed since it worked · `chat` — reproduce it by hand |
|
|
515
|
-
| **A green you don't trust** | an assert that may have tested nothing, a stale cassette, an auto-answered or decided gate | `replay --explain` — the evidence trail behind each *passing* assert · `lint` — assertions on the wrong CI lane / mixed-class keys · `verify-cassettes` — privacy + staleness over committed cassettes · the Gotchas landmine catalog — how a check passes vacuously · `run --repeat N` / `skill --repeat N` — did it pass, or pass once? · `stats` — flaky or expensive over time |
|
|
550
|
+
| **A green you don't trust** | an assert that may have tested nothing, a stale cassette, an auto-answered or decided gate | `replay --explain` — the evidence trail behind each *passing* assert · `replay --mutate` — perturbs each recorded JSON artifact value and reports which perturbations NOTHING caught (reporting only; never moves the verdict/exit code) · `lint` — assertions on the wrong CI lane / mixed-class keys · `verify-cassettes` — privacy + staleness over committed cassettes · the Gotchas landmine catalog — how a check passes vacuously · `run --repeat N` / `skill --repeat N` — did it pass, or pass once? · `stats` — flaky or expensive over time |
|
|
516
551
|
|
|
517
552
|
A failed run also records `errorSource` (where the failure originated) and `stderrLogPath` (the captured
|
|
518
553
|
agent stderr) — read those before re-running; a re-record rarely tells you more than the captured stderr
|
|
@@ -693,11 +728,17 @@ repeats the assertion/replay-relevant ones alongside the schema (a scoped subset
|
|
|
693
728
|
`mode:r` mounts get a real `:ro` bind (a write fails in-guest). But `rw` vs `rwd`
|
|
694
729
|
(write-but-no-delete on `outputs/` / connected folders) is *not* mount-enforced **in the harness** —
|
|
695
730
|
`rm` succeeds and is only caught post-hoc by `no_delete_in_outputs`. **Real Cowork enforces it live:**
|
|
696
|
-
|
|
697
|
-
`allow_cowork_file_delete`
|
|
731
|
+
outputs is a FUSE mount, and `unlink`/`rmdir` fail `Operation not permitted`; a skill must request
|
|
732
|
+
approval via `allow_cowork_file_delete` (which re-mounts the folder `rwd` mid-session) to delete.
|
|
733
|
+
**Only unlinking is denied.** Emptying a file in place — `truncate -s 0`, `> file`, `shred` without
|
|
734
|
+
`-u` — and renaming *within* outputs both SUCCEED in production, so the harness does not flag them
|
|
735
|
+
either. Renaming a file OUT of outputs fails (`EXDEV`, then `EPERM` on the copy-then-unlink
|
|
736
|
+
fallback), so that stays a delete. Two consequences: a skill should not stage disposable scratch
|
|
698
737
|
under `outputs/` (in production, cleanup there costs an approval prompt), and a skill's
|
|
699
738
|
"catch-EPERM-then-request-approval" branch cannot be exercised at any harness tier (the `rm` just
|
|
700
739
|
succeeds here). Do not read this gotcha as "delete-deny may not be real in production" — it is real.
|
|
740
|
+
If a scenario's deletion IS intended, assert `allow_outputs_delete: true` rather than dropping
|
|
741
|
+
`no_delete_in_outputs` — omitting it does not permit anything.
|
|
701
742
|
|
|
702
743
|
9. **Keep `.env` out of any mounted folder** — it is copied into the sandbox and the token could
|
|
703
744
|
leak. Put it at a working-dir or install root (token resolution: env > `--dotenv` > `./.env` >
|
|
@@ -847,6 +888,14 @@ repeats the assertion/replay-relevant ones alongside the schema (a scoped subset
|
|
|
847
888
|
([`docs/fidelity-gaps.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/fidelity-gaps.md)
|
|
848
889
|
→ "File delivery" has the binary-verified detail; repo-only.)
|
|
849
890
|
|
|
891
|
+
25. **Two distinct host-inventory consent flags — a record-time one and a verify-time one.** `record
|
|
892
|
+
--allow-host-inventory-fixture` is the boolean consent to proceed recording a host-inheriting
|
|
893
|
+
(`protocol`/`hostloop`/`cowork`-resolving-to-hostloop) cassette into a repo-visible path — otherwise
|
|
894
|
+
`record` refuses (freezing this machine's MCP servers/agents/account into a committed fixture is the
|
|
895
|
+
risk). `verify-cassettes --allow-host-inventory <regex>` is unrelated: a per-finding suppressor for the
|
|
896
|
+
scanner's `host-inventory` class on an already-committed cassette. Passing one where the other command
|
|
897
|
+
wants it fails as an unrecognized flag — they don't interchange. Depth: `references/ci-recipe.md`.
|
|
898
|
+
|
|
850
899
|
For the assertion catalog, the YAML schema, the fidelity/answer tables, and the CI recipe, read the
|
|
851
900
|
files in `references/` (the gotchas above are the full list; the references repeat only the
|
|
852
901
|
assertion/replay-relevant ones).
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# CI recipe — replay vs live lanes
|
|
2
2
|
|
|
3
|
-
Self-contained reference. Tracks `cowork-harness 1.
|
|
3
|
+
Self-contained reference. Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`).
|
|
4
4
|
|
|
5
5
|
**Fastest path: the packaged Action.** One step gets you `replay`/`lint`/`verify-cassettes` plus a PR
|
|
6
6
|
job-summary reporter (verdict table, staleness findings, cost/turns when available):
|
|
@@ -13,7 +13,7 @@ job-summary reporter (verdict table, staleness findings, cost/turns when availab
|
|
|
13
13
|
```
|
|
14
14
|
|
|
15
15
|
The Action's `version` input defaults to `latest` — intentional so a copy-pasted recipe tracks the current
|
|
16
|
-
release; pin an exact version (e.g. `version: "1.
|
|
16
|
+
release; pin an exact version (e.g. `version: "1.18.0"`) for reproducible CI.
|
|
17
17
|
|
|
18
18
|
Reach for the manual multi-step form below only when you need per-step control the Action's inputs don't
|
|
19
19
|
cover (a custom flag combination, a different runner matrix per step, or `lint`/`verify-cassettes` gated
|
|
@@ -32,7 +32,7 @@ jobs:
|
|
|
32
32
|
- uses: actions/checkout@v4
|
|
33
33
|
- name: Stage the agent binary (official channel, sha256-verified — see https://github.com/yaniv-golan/cowork-harness/blob/main/docs/maintenance.md)
|
|
34
34
|
run: |
|
|
35
|
-
V=2.1.
|
|
35
|
+
V=2.1.221 # match your scenario's pinned baseline's agentVersion
|
|
36
36
|
curl -fSL "https://downloads.claude.ai/claude-code-releases/$V/linux-arm64/claude" -o "$RUNNER_TEMP/claude-$V"
|
|
37
37
|
chmod +x "$RUNNER_TEMP/claude-$V"
|
|
38
38
|
# verify against the committed baseline's sha256 (baselines/desktop-*.json → agentBinary.sha256)
|
|
@@ -58,12 +58,25 @@ sha256-*checked* but not hard-blocking on mismatch — it's advisory for an inte
|
|
|
58
58
|
GitHub-hosted runners, no token/Docker/agent:
|
|
59
59
|
|
|
60
60
|
```yaml
|
|
61
|
-
- run: npm i -g "cowork-harness@>=1.
|
|
62
|
-
- run: cowork-harness lint scenarios/*.yaml
|
|
63
|
-
|
|
61
|
+
- run: npm i -g "cowork-harness@>=1.18.0"
|
|
62
|
+
- run: cowork-harness lint scenarios/*.yaml --strict --min-severity WARN
|
|
63
|
+
# no silent false-greens. WITHOUT --strict this
|
|
64
|
+
# step cannot fail on a WARN-class rule (e.g.
|
|
65
|
+
# vacuous-gate-assert) — it would print the
|
|
66
|
+
# finding and still exit 0. --min-severity WARN
|
|
67
|
+
# keeps the advisory INFO class advisory.
|
|
68
|
+
- run: cowork-harness verify-cassettes cassettes/ # privacy + staleness — FAILS on a stale recording
|
|
69
|
+
# ALSO fails on a leaked host inventory: recording at
|
|
70
|
+
# protocol/hostloop freezes YOUR machine's MCP servers,
|
|
71
|
+
# agents and account org into the cassette. Record
|
|
72
|
+
# committed fixtures at `container` (sealed) to avoid it.
|
|
64
73
|
- run: cowork-harness replay cassettes/ # token-free content/structure
|
|
65
74
|
```
|
|
66
75
|
|
|
76
|
+
Both lines matter: `replay` alone **warns** on a stale recording and exits 0, so dropping the
|
|
77
|
+
`verify-cassettes` step means a skill edit silently stops being tested. (One command instead of two:
|
|
78
|
+
`replay --fail-on-skill-drift`.)
|
|
79
|
+
|
|
67
80
|
The rest of this doc explains the lane split, recording, privacy, and the full pipeline + live job.
|
|
68
81
|
|
|
69
82
|
## The core split: token-free PR gate + live nightly (self-hosted)
|
|
@@ -111,6 +124,13 @@ The split is not just about tokens — it decides **where each lane can run**:
|
|
|
111
124
|
cowork-harness assertions --list --output-format json # every key, with its replay class
|
|
112
125
|
```
|
|
113
126
|
|
|
127
|
+
**`replay --mutate`** is a distinct, reporting-only diagnostic on this same lane: it perturbs each
|
|
128
|
+
recorded JSON artifact value one at a time, re-runs the assertions against the perturbed cassette,
|
|
129
|
+
and reports which perturbations NOTHING caught — those are the fields your `assert:` block leaves
|
|
130
|
+
unguarded. It never changes the verdict or exit code (it's coverage information, not a check), so
|
|
131
|
+
don't wire it into a pass/fail gate — read its `::warning::`/`::notice::` output to find `assert:`
|
|
132
|
+
gaps to close.
|
|
133
|
+
|
|
114
134
|
Placing a key on this gate from a doc list rather than from `assertions --list` is how a valid
|
|
115
135
|
replay-lane check ends up left off the PR gate. Filesystem/egress assertions are
|
|
116
136
|
skipped on this token-free lane — loudly: replay emits an `::warning::` annotation whenever it drops
|
|
@@ -177,6 +197,12 @@ dollar figures). In a skill repo these cassettes get **committed**. So:
|
|
|
177
197
|
(`hostloop`, `protocol`) has an **empty** redaction policy — that combination commits real host paths
|
|
178
198
|
the `path` scanner then hard-fails at `verify-cassettes` time. The always-on scanner remains the
|
|
179
199
|
universal net (container-tier recordings can trip it too).
|
|
200
|
+
- **Host-inheriting record refused by default — `--allow-host-inventory-fixture` is the consent.** A
|
|
201
|
+
`protocol`/`hostloop`/`cowork`-resolving-to-hostloop record into a repo-visible cassette path would
|
|
202
|
+
freeze THIS machine's MCP server names, agents, and account metadata into a committed fixture, so
|
|
203
|
+
`record` refuses outright rather than warn. Pass `--allow-host-inventory-fixture` only when the
|
|
204
|
+
recording session genuinely has no personal MCP servers or plugins to leak — it is a per-record
|
|
205
|
+
boolean consent, not a pattern.
|
|
180
206
|
- **Always-on scan gate** — `verify-cassettes` flags email / currency / bare-domain / local-path /
|
|
181
207
|
machine-inventory matches it finds in the committed cassettes and **exits non-zero**, so "no leak" is
|
|
182
208
|
a gate, not discipline. Non-zero is not one thing, though: exit `1` means verification RAN and found a
|
|
@@ -189,6 +215,13 @@ dollar figures). In a skill repo these cassettes get **committed**. So:
|
|
|
189
215
|
Suppress synthetic / public reference names (NVCA, Cooley GO, …) with `--allow <regex>`. (Multi-word
|
|
190
216
|
proper names are NOT a default class — too noisy to gate on; add a pattern via config if your corpus
|
|
191
217
|
needs it.)
|
|
218
|
+
- **`--allow-host-inventory <regex>` — the per-finding sibling of the record-time consent above, and a
|
|
219
|
+
different thing from it.** `verify-cassettes` scans a **structural** `host-inventory` class (an
|
|
220
|
+
`mcp_servers[].name`/`agents[]` entry outside the harness's own known set, or `account.email`/
|
|
221
|
+
`.organization`/`.subscriptionType`) over already-committed cassettes. `--allow-host-inventory <regex>`
|
|
222
|
+
suppresses a specific finding there — it is a scanner allowlist entry, not a record-time green light —
|
|
223
|
+
and, like `--allow-domain`/`--allow-email`/`--allow-path`/`--allow-machine-inventory`, scopes the
|
|
224
|
+
pattern to that one class so it can't accidentally clear an unrelated finding.
|
|
192
225
|
|
|
193
226
|
```bash
|
|
194
227
|
cowork-harness verify-cassettes cassettes/ # privacy scan + staleness — exit 1 = verified & failed, exit 3 = could not verify
|
|
@@ -246,7 +279,7 @@ jobs:
|
|
|
246
279
|
with: { node-version: '24' }
|
|
247
280
|
- uses: actions/setup-python@v5
|
|
248
281
|
with: { python-version: '3.x' } # python3 only — PyYAML is bundled with the linter
|
|
249
|
-
- run: npm i -g "cowork-harness@>=1.
|
|
282
|
+
- run: npm i -g "cowork-harness@>=1.18.0"
|
|
250
283
|
- run: cowork-harness lint scenarios/*.yaml # no-silent-false-green (needs python3; PyYAML bundled)
|
|
251
284
|
- run: cowork-harness verify-cassettes cassettes/ --output-format json # privacy + staleness gate
|
|
252
285
|
- run: cowork-harness replay cassettes/ --output-format json # token-free content/structure
|
|
@@ -275,7 +308,7 @@ jobs:
|
|
|
275
308
|
echo "live=true" >> "$GITHUB_OUTPUT"
|
|
276
309
|
fi
|
|
277
310
|
- if: steps.guard.outputs.live == 'true'
|
|
278
|
-
run: npm i -g "cowork-harness@>=1.
|
|
311
|
+
run: npm i -g "cowork-harness@>=1.18.0"
|
|
279
312
|
- if: steps.guard.outputs.live == 'true'
|
|
280
313
|
run: cowork-harness run scenarios/ --output-format json
|
|
281
314
|
env:
|
|
@@ -302,6 +335,14 @@ scenario = `result === "success" && assertions.every(pass)`. Exit code is non-ze
|
|
|
302
335
|
fails or a run errors, so a plain `cowork-harness run scenarios/` is already CI-ready without parsing
|
|
303
336
|
JSON.
|
|
304
337
|
|
|
338
|
+
**For the commands in this recipe, stdout carries the machine envelope and nothing else.** Without
|
|
339
|
+
`--output-format json`, `run` / `record` / `replay` / `verify-cassettes` / `status` write their whole
|
|
340
|
+
human rendering — warnings, verdict, `status`'s summary line — to **stderr**, and stdout stays empty.
|
|
341
|
+
A wrapper that captures only stdout gets an empty log and, if it greps that for a state, a silent false
|
|
342
|
+
negative. Capture stderr for the human trail (`2> run.stderr.log`), or ask for JSON and parse stdout.
|
|
343
|
+
(Commands whose whole job is to print a value — `--version`, `assertions --list`, `scaffold`, `gates`,
|
|
344
|
+
`skill --dry-run` — write it to stdout by design, with or without the flag.)
|
|
345
|
+
|
|
305
346
|
`verify-cassettes` emits its **own** envelope (`{command, ok, coverage, results[]}` with per-file
|
|
306
347
|
`findings`/`staleness`/`unverifiable`/`notes`/`version`/`error`), published as
|
|
307
348
|
`schema/verify-cassettes.json` in the npm package. `ok:false` doesn't say *why* — read the buckets, or <!-- npm-only-ok -->
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Critique — the facts a plugin install can't otherwise reach
|
|
2
2
|
|
|
3
|
-
Tracks `cowork-harness 1.
|
|
3
|
+
Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`). This is **not** a trim of the full
|
|
4
4
|
[`docs/critique.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/critique.md) (repo-only —
|
|
5
5
|
flags, cost, reproduction discipline, known limitations all live there). This file covers exactly what a
|
|
6
6
|
plugin install cannot otherwise discover: the run-dir artifact a harvester actually reads, the report's
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
# Fidelity tiers & answer paths
|
|
2
2
|
|
|
3
|
-
Self-contained reference. Tracks `cowork-harness 1.
|
|
3
|
+
Self-contained reference. Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`).
|
|
4
4
|
|
|
5
5
|
## Fidelity tiers (`fidelity:` in the scenario)
|
|
6
6
|
|
|
@@ -74,7 +74,7 @@ regressions.
|
|
|
74
74
|
| **Spawned helper** | `--decider-cmd '<helper>'` | depends on the helper |
|
|
75
75
|
| **In-band (driving agent)** | `--decider-dir <dir>` (+ a Monitor that writes responses) | depends |
|
|
76
76
|
|
|
77
|
-
Scripted `answers:` and a terminal (`--decider-llm` / `--decider-cmd` / `--decider-dir` / `on_unanswered`) compose as a normal Chain — scripted rules resolve matched gates, the terminal answers whatever's left. What's mutually exclusive is picking **two terminals** at once (
|
|
77
|
+
Scripted `answers:` and a terminal (`--decider-llm` / `--decider-cmd` / `--decider-dir` / `on_unanswered`) compose as a normal Chain — scripted rules resolve matched gates, the terminal answers whatever's left. What's mutually exclusive is picking **two terminals** at once — the CLI rejects every such pairing: two channels (`--decider-dir` with `--decider-cmd`), a model terminal with a channel or a policy (`--decider-llm` with either), and a channel with a policy (`--decider-dir`/`--decider-cmd` with `--on-unanswered`). A channel or the LLM decider *is* the terminal, so an accompanying `--on-unanswered` could only ever be inert — the chain never reaches the policy terminal. The in-band path's Monitor drives two
|
|
78
78
|
subcommands: `gates <dir>` (stream the pending questions) and `answer <dir> --gate N …` (reply to one).
|
|
79
79
|
|
|
80
80
|
**The `--decider-llm` answer protocol.** The answering model is shown the gate's options **numbered** and
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# Scenario & session schema, assertion catalog, web_fetch, full gotchas
|
|
2
2
|
|
|
3
|
-
Self-contained reference for authoring `cowork-harness` scenarios. Tracks `cowork-harness 1.
|
|
4
|
-
(baseline `desktop-1.
|
|
3
|
+
Self-contained reference for authoring `cowork-harness` scenarios. Tracks `cowork-harness 1.18.0`
|
|
4
|
+
(baseline `desktop-1.25927.0`). If your checkout is newer, prefer the live [`docs/scenario.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/scenario.md),
|
|
5
5
|
[`docs/session.md`](https://github.com/yaniv-golan/cowork-harness/blob/main/docs/session.md), and `SPEC.md`.
|
|
6
6
|
|
|
7
7
|
**Minimal scenario** — `prompt` is the only required field:
|
|
@@ -32,7 +32,7 @@ references a session for setup.
|
|
|
32
32
|
```yaml
|
|
33
33
|
name: my-test # OPTIONAL — defaults to the filename; keys runs/<name>/
|
|
34
34
|
baseline: latest # platform baseline: "latest" or "desktop-<ver>" (NOT "profile:")
|
|
35
|
-
session: ../sessions/
|
|
35
|
+
session: ../sessions/my-session.yaml # pre-prompt setup (resolved relative to THIS file)
|
|
36
36
|
fidelity: container # protocol | container | microvm | hostloop | cowork
|
|
37
37
|
execution: local # OPTIONAL — orthogonal to fidelity (a privilege/sandbox tier, all
|
|
38
38
|
# local): local (default) | cloud-describe (RESERVED — no runner
|
|
@@ -146,6 +146,13 @@ folders:
|
|
|
146
146
|
uploads:
|
|
147
147
|
- ~/Downloads/report.pdf
|
|
148
148
|
|
|
149
|
+
# connected PROJECTS (userSelectedProjectUuids) — NOT the same as a work folder above, and not the
|
|
150
|
+
# legacy mnt/.projects/<id> work-folder path older baselines used
|
|
151
|
+
projects:
|
|
152
|
+
- { uuid: 7f3c…, from: ~/code/myproject } # mounted READ-ONLY at mnt/.projects/<uuid>
|
|
153
|
+
# no mode: knob — production hardcodes ro
|
|
154
|
+
# never becomes the cwd; {{workspaceFolder}} stays at outputs
|
|
155
|
+
|
|
149
156
|
# discovery: marketplaces / plugins / skills / mcp
|
|
150
157
|
plugins:
|
|
151
158
|
marketplaces: [] # plugin_marketplaces (git URLs or local paths)
|
|
@@ -158,7 +165,7 @@ plugins:
|
|
|
158
165
|
skills:
|
|
159
166
|
local: [] # extra host skill dirs
|
|
160
167
|
suggest_enabled: true # gate 245679952 override — `mcp__skills__suggest_skills` on/off (default true)
|
|
161
|
-
proactive_suggest_enabled: false # gate 1598976391 override — proactive description + `trigger` param (
|
|
168
|
+
proactive_suggest_enabled: false # gate 1598976391 override — proactive description + `trigger` param; unset = synced baseline gate (ON from 1.24012.11)
|
|
162
169
|
mcp:
|
|
163
170
|
config: null # --mcp-config file (standard mcpServers map)
|
|
164
171
|
enabled: []
|
|
@@ -274,6 +281,7 @@ same set live from the schema.
|
|
|
274
281
|
| `file_exists: <path>` | the path exists under the run's `work/` (anchored at `mnt/`, e.g. `outputs/x.md`). For a user-facing deliverable prefer `user_visible_artifact` — with a connected folder the file lands in `mnt/<folder>` (= `{{workspaceFolder}}`), not `mnt/outputs`, so `file_exists: outputs/x.md` misses it |
|
|
275
282
|
| `user_visible_artifact: <path>` | exists **and** under a user-visible root (`outputs/` + each connected folder's mount name) — the right primitive for a workspace deliverable when a folder is connected |
|
|
276
283
|
| `no_delete_in_outputs: true` | no delete op touched `mnt/outputs` — **only `true` is valid**; `false` is rejected (omit to allow deletes) |
|
|
284
|
+
| `no_delete_in_mounts: true` | no delete op touched ANY delete-denied mount — `outputs` plus every `rw` connected folder — except those waived by `allow_delete_in`. Production denies `unlink`/`rmdir` on every such mount, so `no_delete_in_outputs` covers only part of the real rule. **only `true` is valid** |
|
|
277
285
|
| `no_unexpected_files: [<glob>, …]` | every **newly created** file under a user-visible root matches ≥1 glob (workRoot-relative paths; `**` = whole path segment for any depth — use `outputs/handoff/**` for per-run subdirs); `[]` = no new files; **new-files-only** — overwriting a pre-existing file in place is invisible (use content-level producer stamping); live/verify-run without a pre-run manifest ⇒ evidence-unavailable hard-fail (live runs capture the baseline only when this key is asserted; recordings always capture); an **incomplete post-run filesystem walk** (an unreadable subtree — a permission/I-O error) also fails evidence-unavailable rather than reporting "no strays" over a partial tree — distinct from the missing-manifest case (a `--resume` run), which fails for a different reason; captured on every live sandbox tier including microvm (its outputs are snapshotted from the VM into the run dir); replay needs `cassette.preRunPaths` (≥0.24 recordings) — cassettes without it **exclude** the key with a loud warning |
|
|
278
286
|
| `input_unmodified: <glob>` or `[<glob>, …]` | a single glob or a list; every **pre-existing** file (incl. uploaded files under `uploads/**`) whose workRoot-relative path matches ≥1 glob keeps an unchanged content hash after the run — the in-place-mutation companion to `no_unexpected_files`'s new-files check (`[]` is rejected by the schema — list at least one glob); a glob that matches **no** pre-run path fails loud (a typo or renamed mount would otherwise verify zero files and pass vacuously); a matched file that was deleted counts as a content change (fails); live/verify-run without a pre-run hash manifest ⇒ evidence-unavailable hard-fail (a `--resume` run); captured on every live sandbox tier including microvm; replay needs `cassette.preRunHashes` — cassettes without it **exclude** the key with a loud warning; on replay it compares against the manifest's recorded `sha256`, never a re-hash of the materialized tree |
|
|
279
287
|
| `self_heal_ran: <bool>` | a plugin-root self-heal script was (not) invoked |
|
|
@@ -334,12 +342,14 @@ same set live from the schema.
|
|
|
334
342
|
| `allow_l0_plugin_divergence: true` | verdict modifier — opt into L0/protocol plugin divergence: suppresses the default-fail when a plugin behaves differently at `protocol` (L0) fidelity than under a sandboxed tier. Live tiers only |
|
|
335
343
|
| `allow_stall: true` | verdict modifier — suppresses the `stalled` default-fail when a run ends on a question having done no productive tool work after its last gate (the agent asked for input and stopped — incl. re-asking in plain text after answering an `AskUserQuestion`); assert only when ending on a question is intended, else script the answer (`answer:` / `--answer` / a decider) |
|
|
336
344
|
| `allow_undelivered_deliverables: true` | verdict modifier — suppresses the `undelivered_deliverables` WARN. Working in the scratchpad is Cowork's designed pattern, so a skill that legitimately leaves intermediates, caches or downloaded inputs behind can say so instead of carrying permanent noise. The signal is warn-only and never fails a run on its own; reach for this when the scratch activity is intentional, not to silence a real delivery gap |
|
|
345
|
+
| `allow_outputs_delete: true` | verdict modifier — accepts a detected outputs delete instead of failing the run, for a skill whose deletion is intended. Omitting `no_delete_in_outputs` does **not** permit deletes (a detected delete fails via the `outputs_delete` signal precisely because the key was not authored), so this is the way to accept one. **Mutually exclusive** with `no_delete_in_outputs`. Waives the harness's post-hoc detection; it does not model Cowork's `allow_cowork_file_delete` approval handshake |
|
|
346
|
+
| `allow_delete_in: [<mount>…]` | verdict modifier — accepts detected deletes in the named mounts (the per-mount analogue of `allow_outputs_delete`, mirroring production's `fileDeleteApprovedMounts`). Waives the verdict only; detection still runs and the hits stay in `result.json`. Listing `outputs` conflicts with `no_delete_in_outputs` |
|
|
337
347
|
| `transcript_no_host_path: true` | no host path (`/Users/`, `/opt/cowork/`, `/home/`, `/root/`) leaked into model-visible text — **incompatible with `hostloop` AND `protocol`**: hostloop's native file tools legitimately expose real host paths, and protocol (L0) runs the agent's file tools on the real host cwd with no sealed filesystem, so this fails BY DESIGN on both (the harness warns at run start if asserted anyway); use `container`/`microvm` for this check |
|
|
338
348
|
| `egress_denied: <host>` | the host was blocked by the egress proxy |
|
|
339
349
|
| `egress_allowed: <host>` | the host was allowed through |
|
|
340
350
|
| `no_mcp_error: true` | no MCP round-trip failed (`RunResult.mcpErrors` is empty — no unhandled server, no handler throw) — live-only: MCP round-trips are harness-computed, not in the SDK stdout stream, so evidence-unavailable on replay (never a vacuous pass). **Only `true` is valid** |
|
|
341
351
|
| `max_peak_rss_bytes: <N>` | peak sampled RSS of the agent sandbox ≤ N bytes (`RunResult.resources.peakRssBytes`) — live-only: replay never spawns a sandbox to sample, so evidence-unavailable on replay/protocol (never a vacuous pass); also evidence-unavailable when sampling captured no RSS value |
|
|
342
|
-
| `semantic_matches: {rubric: [...], min_pass?, judge_model?}` | a pinned LLM judge grades each fixed `rubric` claim against the run's answer — the **union of the agent's final result text (`RunResult.finalMessage`), the transcript, and the final on-disk content of any files the agent authored during the run** — so a claim about content the skill led the agent to *write to a file* grades as reliably as one about inlined prose (authored-file evidence is captured on every live sandbox tier including **microvm** — its session tree is snapshotted from the VM into the run dir). When the authored-file evidence backing the judged document is **incomplete** — a file dropped at the capture-size cap, unreadable at read-back, or (on `--resume`) the scratchpad walk skipped — the assert fails evidence-unavailable rather than trusting a judge grade over a partial document; this is separate from the malformed-grade `judgeInvalid` path below. The assert passes iff ≥ `min_pass` claims pass (default: all — avoid for a gating scenario). Results align by claim index and are recorded per-claim in `RunResult.assertions[].semanticClaims` (`[{index, claim, pass}]`, so a consumer can diff the per-claim profile across runs); a rep whose grade can't be parsed (after one retry) is marked `RunResult.assertions[].judgeInvalid` and **never silently dropped** — it is excluded from the pass denominator, and the guard against a misleading score from that exclusion is the gate's minimum-valid-rep floor (`MIN_VALID` ≥ 4) plus this visibility, not a claim that denominator-shrinking inflation is impossible. Within a rep, a grade that's still unparseable after the retry **fails that assert outright** (evidence-unavailable, not a vacuous pass) — a persistently-flaky judge reds the run rather than silently passing. `judge_model` pins the grader (default when neither it nor `COWORK_HARNESS_JUDGE_MODEL` is set: `claude-opus-4-8`; a dated id keeps a before/after comparison reproducible). Live-only: the judge is a live model call, so evidence-unavailable / skipped-loud on replay (never a vacuous pass) |
|
|
352
|
+
| `semantic_matches: {rubric: [...], min_pass?, judge_model?, include_subagent_text?}` | a pinned LLM judge grades each fixed `rubric` claim against the run's answer — the **union of the agent's final result text (`RunResult.finalMessage`), the transcript, and the final on-disk content of any files the agent authored during the run** — so a claim about content the skill led the agent to *write to a file* grades as reliably as one about inlined prose. **"The transcript" is narrower than it reads: top-level `assistant_text` ONLY.** It excludes every `tool_use`/`tool_result` and **all sub-agent-originated text** (including fork-scoped `Skill`/`Agent(fork)` dispatches — the harness attributes their *tool* calls to the main agent, but not their text). ⚠️ **Consequence: a rubric claim about whether a tool was called can NEVER grade true** — the evidence is not in the judged document. Such a claim looks reasonable and silently caps your pass rate; assert tool use with `tool_called` / `present_files_called` / `subagent_dispatched` / `hook_blocked` instead. Sub-agent text is captured in `RunResult.subagents[].reasoning` and reaches the judge only via opt-in `include_subagent_text: true` (`kind:"text"` turns only — sub-agent *thinking* arrives empty with `redacted:true`, so it would pad the document with blanks) (authored-file evidence is captured on every live sandbox tier including **microvm** — its session tree is snapshotted from the VM into the run dir). When the authored-file evidence backing the judged document is **incomplete** — a file dropped at the capture-size cap, unreadable at read-back, or (on `--resume`) the scratchpad walk skipped — the assert fails evidence-unavailable rather than trusting a judge grade over a partial document; this is separate from the malformed-grade `judgeInvalid` path below. The assert passes iff ≥ `min_pass` claims pass (default: all — avoid for a gating scenario). Results align by claim index and are recorded per-claim in `RunResult.assertions[].semanticClaims` (`[{index, claim, pass}]`, so a consumer can diff the per-claim profile across runs); a rep whose grade can't be parsed (after one retry) is marked `RunResult.assertions[].judgeInvalid` and **never silently dropped** — it is excluded from the pass denominator, and the guard against a misleading score from that exclusion is the gate's minimum-valid-rep floor (`MIN_VALID` ≥ 4) plus this visibility, not a claim that denominator-shrinking inflation is impossible. Within a rep, a grade that's still unparseable after the retry **fails that assert outright** (evidence-unavailable, not a vacuous pass) — a persistently-flaky judge reds the run rather than silently passing. `judge_model` pins the grader (default when neither it nor `COWORK_HARNESS_JUDGE_MODEL` is set: `claude-opus-4-8`; a dated id keeps a before/after comparison reproducible). Live-only: the judge is a live model call, so evidence-unavailable / skipped-loud on replay (never a vacuous pass) |
|
|
343
353
|
| `artifact_json: {artifact, path, …}` | assert a JSON artifact's contents — `equals`/`gt`/`in`/`exists`/`absent`/`is_null` over a dotted `path` (`in` = membership in a list, for a stochastic/LLM value; `absent` ≠ `is_null`; an unresolved intermediate fails loud) |
|
|
344
354
|
| `computer_links_resolve: true` | every `computer://` link in the model-visible transcript resolves to an artifact that exists in the run's collected outputs/mounts — a dangling link fails, naming which target was checked (a live host path, the collected work tree, or the replay manifest). **Requires ≥1 link** (zero links fails — use `computer_links_resolve_if_present` for the presence-free variant). **Only `true` is valid** (`false` is rejected by the schema) |
|
|
345
355
|
| `computer_links_resolve_if_present: true` | like `computer_links_resolve` but passes vacuously when the transcript has zero `computer://` links — the presence-free variant. **Only `true` is valid** |
|
|
@@ -369,6 +379,7 @@ codes (`VerdictSignal["code"]` in `src/run/verdict.ts`):
|
|
|
369
379
|
| `transport_error` | fail | The connection dropped mid/after-run |
|
|
370
380
|
| `permissive_auto_allow` | fail | A cowork-parity auto-allow real Cowork would block (opt out: `allow_permissive_auto_allow`) |
|
|
371
381
|
| `outputs_delete` | fail | An unauthorized delete touched `mnt/outputs` (opt out: author `no_delete_in_outputs`) |
|
|
382
|
+
| `mount_delete` | warn | A delete touched a delete-denied mount other than `outputs` (a `rw` connected folder). Production denies `unlink`/`rmdir` there until per-mount approval, so the run diverged. Warn, not fail: the harness detects post-hoc what production enforces. Author `no_delete_in_mounts` to hard-fail, or `allow_delete_in` to waive |
|
|
372
383
|
| `host_path_leak` | fail | A host path leaked into model-visible text (opt out: author `transcript_no_host_path`) |
|
|
373
384
|
| `l0_plugin_divergence` | fail | L0/protocol plugin loading diverged from Cowork (opt out: `allow_l0_plugin_divergence`) |
|
|
374
385
|
| `missing_capability` | fail | A `requires_capabilities` need was unmet, or the skill used a capability the image omits (opt out: `allow_missing_capability`, or `skill --allow-missing-capability` on an open-ended run) |
|
|
@@ -378,7 +389,8 @@ codes (`VerdictSignal["code"]` in `src/run/verdict.ts`):
|
|
|
378
389
|
| `prompt_asset_missing` | warn | The run proceeded with a missing prompt asset (`COWORK_HARNESS_ALLOW_MISSING_PROMPT=1`); fidelity is degraded |
|
|
379
390
|
| `scan_unavailable` | warn | Post-run scan evidence unavailable (`RunResult.scan` undefined) — the host-path and outputs-delete guards did not run this run |
|
|
380
391
|
| `ended_with_question` | warn | Live-lane heuristic: the final answer contains a question and the run wrote no deliverable to `outputs/` — the lenient sibling of `stalled` (covers a mid-message `?`, or tool work after the last gate that still ended asking). Opt out: `allow_stall` |
|
|
381
|
-
| `undelivered_deliverables` | warn | The skill produced file(s) OUTSIDE every user-visible root and never delivered them
|
|
392
|
+
| `undelivered_deliverables` | warn | The skill produced file(s) OUTSIDE every user-visible root and never delivered them, so they stay invisible to the user. Fires on every run without opting in, because the scenarios that most need it are the ones whose author never considered delivery. Silent when the evidence cannot answer the question (no workspace walk, a tier that runs no scratchpad walk, absent delivery telemetry, a resumed turn, or a lane where delivery is unobservable — see `delivery_unobservable`) — never a vacuous clean. **`lane: local` only**: on remote, delivery cannot be measured at all, so that lane reports `delivery_unobservable` instead of guessing. Opt out: `allow_undelivered_deliverables` |
|
|
393
|
+
| `delivery_unobservable` | warn | `lane: remote` only — the run produced file(s), but whether any reached the user CANNOT be verified: nothing is delivered by location on that lane and the harness models no remote delivery tool (production uses the agent-native `SendUserFile`). The honest counterpart to `undelivered_deliverables`, which would otherwise fire on every remote run that writes anything — a signal that always fires carries no information. Mutually exclusive with it; quiet when the run produced nothing to deliver. A harness coverage gap, not a skill defect. Opt out: `allow_undelivered_deliverables` |
|
|
382
394
|
| `exec_infra_error` | warn | Host-loop: one or more container `exec` calls failed for infrastructure reasons, so those tool calls returned an error to the agent. Warns rather than fails because the run's other evidence is intact — unlike `infra_error`, where a dead supervisor contaminates everything. Caveat: if *every* exec failed, the agent ran nothing and this still only warns — check `result.infraErrors` |
|
|
383
395
|
|
|
384
396
|
A **fail**-severity signal does not change `result.result` (still `"success"`), but it DOES fail the
|
|
@@ -390,12 +402,12 @@ overall run verdict and exit code — `assert result: success` alone won't catch
|
|
|
390
402
|
A cassette (`record`/`replay`) has **no filesystem and no network**. `replay` re-evaluates only the
|
|
391
403
|
**content** assertions. The authoritative list is `ALWAYS_CONTENT_KEYS`/`QUESTION_GATE_KEYS`/`MANIFEST_KEYS` (composed) in `src/run/cassette.ts`.
|
|
392
404
|
|
|
393
|
-
**Scenario source — the WHOLE scenario is frozen; only `assert:` can be opted back to
|
|
394
|
-
captures every key (`name`/`prompt`/`session`/`baseline`/`fidelity`/`lane`/`skills
|
|
395
|
-
`requires_capabilities`/`expect_denied`/`assert`), and a plain `replay` evaluates
|
|
396
|
-
frozen copy (byte-deterministic, ignores the working tree); editing
|
|
397
|
-
it — replay only prints a `::notice::` when a sibling's
|
|
398
|
-
load at all. An edited `lane:`/`fidelity:`/`baseline:` reaches a replay ONLY by re-recording. `replay --assert-from
|
|
405
|
+
**Scenario source — the WHOLE scenario is frozen; only `assert:` (+`expect_denied:`) can be opted back to
|
|
406
|
+
disk.** A cassette captures every key (`name`/`prompt`/`session`/`baseline`/`fidelity`/`lane`/`skills`/
|
|
407
|
+
`answers`/`execution`/`requires_capabilities`/`expect_denied`/`assert`), and a plain `replay` evaluates
|
|
408
|
+
**all** of them from that frozen copy (byte-deterministic, ignores the working tree); editing
|
|
409
|
+
`scenarios/<name>.yaml` does not change it — replay only prints a `::notice::` when a sibling's
|
|
410
|
+
`assert:`/`prompt:` differs, or when it fails to load at all. An edited `lane:`/`fidelity:`/`baseline:` reaches a replay ONLY by re-recording. `replay --assert-from
|
|
399
411
|
<scenario.yaml>` / `--reassert` is the opt-in token-free re-check against the on-disk block; it **hard-fails**
|
|
400
412
|
on recording-shaping drift (`prompt`/`answers`/`baseline`/`skills`) or skill-content staleness (it implies
|
|
401
413
|
`--fail-on-skill-drift`). `expect_denied`/filesystem/egress keys are sourced from on-disk but stay live-only —
|
|
@@ -2,7 +2,7 @@
|
|
|
2
2
|
|
|
3
3
|
Each recipe composes facts that live scattered across SKILL.md and the other references into one
|
|
4
4
|
decision path. Every one answers a question a real fleet owner had to work out the hard way.
|
|
5
|
-
Tracks `cowork-harness 1.
|
|
5
|
+
Tracks `cowork-harness 1.18.0` (baseline `desktop-1.25927.0`), same as SKILL.md's front-matter. Recipe 2's `resolved-tier`/`unverifiable-tier` staleness classes and
|
|
6
6
|
Recipe 3's `init-redact` shipped in 0.24.0 and are part of the current feature set — no version gate
|
|
7
7
|
needed if your CLI meets SKILL.md's version floor.
|
|
8
8
|
|
|
@@ -131,7 +131,8 @@ degrade the advice. It is real work to calibrate; these steps are the traps that
|
|
|
131
131
|
1. **Author a suite of Q&A scenarios — one per representative question.** Each installs your skill and
|
|
132
132
|
asserts the answer with a rubric. The judge grades the **union of the agent's final answer, the
|
|
133
133
|
transcript, and any files it wrote**, so a claim about content your skill leads the agent to *write to a
|
|
134
|
-
file* grades as reliably as one about inlined prose — you don't have to force an inline answer
|
|
134
|
+
file* grades as reliably as one about inlined prose — you don't have to force an inline answer.
|
|
135
|
+
**But "the transcript" is narrower than it sounds — see step 2a before writing a claim about tool use:**
|
|
135
136
|
```yaml
|
|
136
137
|
session: ./_session.yaml # plugins.local_plugins + enabled: [<your-skill>@local]
|
|
137
138
|
fidelity: container
|
|
@@ -142,6 +143,20 @@ degrade the advice. It is real work to calibrate; these steps are the traps that
|
|
|
142
143
|
- <one discrete, checkable claim a correct answer MUST make>
|
|
143
144
|
- <another>
|
|
144
145
|
```
|
|
146
|
+
2a. **NEVER write a claim about whether a TOOL was called — the judge cannot see tool calls.** The
|
|
147
|
+
"transcript" the judge receives is **top-level assistant prose only**. It excludes every `tool_use`
|
|
148
|
+
and `tool_result`, and it excludes **all sub-agent text** (including `Skill` and `Agent(fork)`
|
|
149
|
+
dispatches, whose *tool* calls the harness does attribute to the main agent — the text path does not).
|
|
150
|
+
Sub-agent text is captured in `RunResult.subagents[].reasoning` but is **not** sent to the judge
|
|
151
|
+
unless you opt in with `include_subagent_text: true`.
|
|
152
|
+
|
|
153
|
+
So a rubric claim like *"the agent used a tool to surface the file, or said none was available"* has a
|
|
154
|
+
first branch that **can never grade true**, no matter how the skill behaves — the evidence simply
|
|
155
|
+
isn't in the document. Such a claim looks reasonable, survives drafting, and silently caps your pass
|
|
156
|
+
rate. Assert tool use with the structural keys instead (`tool_called`, `present_files_called`,
|
|
157
|
+
`subagent_dispatched`, `hook_blocked`), and reserve `semantic_matches` for what the agent *said* or
|
|
158
|
+
*wrote*. For a fan-out skill whose real work happens in sub-agents, add `include_subagent_text: true`.
|
|
159
|
+
|
|
145
160
|
2. **Write DISCRIMINATING claims, and verify each against ground truth — not memory.** A claim that
|
|
146
161
|
contradicts how the tool actually behaves can *never* pass (the correct skill will contradict it), and
|
|
147
162
|
it silently poisons the gate. Check each claim against the code/docs. Decompose into single,
|
|
@@ -2,8 +2,10 @@
|
|
|
2
2
|
"$comment": "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
|
|
3
3
|
"keys": [
|
|
4
4
|
"all_tasks_completed",
|
|
5
|
+
"allow_delete_in",
|
|
5
6
|
"allow_l0_plugin_divergence",
|
|
6
7
|
"allow_missing_capability",
|
|
8
|
+
"allow_outputs_delete",
|
|
7
9
|
"allow_permissive_auto_allow",
|
|
8
10
|
"allow_stall",
|
|
9
11
|
"allow_undelivered_deliverables",
|
|
@@ -26,6 +28,7 @@
|
|
|
26
28
|
"max_tokens",
|
|
27
29
|
"max_tool_errors",
|
|
28
30
|
"max_turns",
|
|
31
|
+
"no_delete_in_mounts",
|
|
29
32
|
"no_delete_in_outputs",
|
|
30
33
|
"no_hook_blocked",
|
|
31
34
|
"no_lost_write_back",
|
|
@@ -91,10 +94,26 @@
|
|
|
91
94
|
"timeout_ms"
|
|
92
95
|
],
|
|
93
96
|
"verdictModifierKeys": [
|
|
97
|
+
"allow_delete_in",
|
|
94
98
|
"allow_l0_plugin_divergence",
|
|
95
99
|
"allow_missing_capability",
|
|
100
|
+
"allow_outputs_delete",
|
|
96
101
|
"allow_permissive_auto_allow",
|
|
97
102
|
"allow_stall",
|
|
98
103
|
"allow_undelivered_deliverables"
|
|
104
|
+
],
|
|
105
|
+
"servedHookEvents": [
|
|
106
|
+
"PreToolUse"
|
|
107
|
+
],
|
|
108
|
+
"knownHookEvents": [
|
|
109
|
+
"Notification",
|
|
110
|
+
"PostToolUse",
|
|
111
|
+
"PreCompact",
|
|
112
|
+
"PreToolUse",
|
|
113
|
+
"SessionEnd",
|
|
114
|
+
"SessionStart",
|
|
115
|
+
"Stop",
|
|
116
|
+
"SubagentStop",
|
|
117
|
+
"UserPromptSubmit"
|
|
99
118
|
]
|
|
100
119
|
}
|