cowork-harness 0.13.0 → 0.14.0

package/.claude/skills/cowork-harness/scripts/assertion-keys.json

@@ -1,5 +1,5 @@
 {
-  "$comment": "GENERATED from the Zod Assertion schema (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
+  "$comment": "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
   "keys": [
     "allow_l0_plugin_divergence",
     "allow_missing_capability",
@@ -32,6 +32,19 @@
     "transcript_not_matches",
     "user_visible_artifact"
   ],
+  "topLevelKeys": [
+    "answers",
+    "assert",
+    "baseline",
+    "expect_denied",
+    "fidelity",
+    "name",
+    "on_unanswered",
+    "prompt",
+    "requires_capabilities",
+    "session",
+    "skills"
+  ],
   "verdictModifierKeys": [
     "allow_l0_plugin_divergence",
     "allow_missing_capability",

package/.claude/skills/cowork-harness/scripts/scenario.py

@@ -116,8 +116,11 @@ if UNCLASSIFIED_KEYS:
         "— add them to the linter's CONTENT/GATE/MANIFEST/LIVE_ONLY/VERDICT_MODIFIER sets.",
         file=sys.stderr,
     )
-# every valid top-level scenario key
-TOP_LEVEL_KEYS = {
+# Embedded fallback for the top-level scenario keys — kept EQUAL to the generated `topLevelKeys`
+# (test-enforced, like _CLASSIFIED_KEYS for assert keys) so a missing assertion-keys.json can't silently
+# reintroduce key drift. `profile`/`assertions` are NOT here — they are deprecated aliases handled by the
+# special-cases in the unknown-key check below (`k != "assertions" and k != "profile"`).
+_EMBEDDED_TOP_LEVEL_KEYS = {
     "name",
     "baseline",
     "session",
@@ -128,7 +131,32 @@ TOP_LEVEL_KEYS = {
     "expect_denied",
     "assert",
     "skills",  # F-6: opt-in skill-staleness hash scope
+    "requires_capabilities",  # Fix 4b: scenario-level required-capability declaration (pre-flight gate)
 }
+
+
+def _load_top_level_keys():
+    """The authoritative top-level scenario-key set, generated from the Zod ScenarioObject schema into
+    `assertion-keys.json` (so the unknown-key check can't drift and false-flag a valid key). Falls back to
+    the embedded `_EMBEDDED_TOP_LEVEL_KEYS` (kept equal to the generated list) with a loud warning if the
+    file is missing or predates the `topLevelKeys` field."""
+    p = Path(__file__).resolve().parent / "assertion-keys.json"
+    try:
+        keys = json.loads(p.read_text(encoding="utf-8")).get("topLevelKeys")
+        if not keys:
+            raise KeyError("topLevelKeys")
+        return set(keys)
+    except Exception:
+        print(
+            f"::warning:: assertion-keys.json missing or has no topLevelKeys next to scenario.py ({p}) — "
+            "using a built-in top-level-key list that may be stale (run `npm run schema`).",
+            file=sys.stderr,
+        )
+        return set(_EMBEDDED_TOP_LEVEL_KEYS)
+
+
+# every valid top-level scenario key (generated from the zod ScenarioObject schema; see _load_top_level_keys)
+TOP_LEVEL_KEYS = _load_top_level_keys()
 REGEX_KEYS = {
     "transcript_matches",
     "transcript_not_matches",

package/CHANGELOG.md

@@ -6,6 +6,32 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [0.14.0] — 2026-06-25
+
+### Added
+
+- **`replay` surfaces staleness + skipped assertions in JSON.** Each `--output-format json` result now carries
+  `staleness[]` (class-tagged: `baseline` / `skill` / `shared-root` / `format` / `unverifiable-baseline` /
+  `unverifiable-skill`) and `skippedAssertions` (`{full, partial}`), so a token-free CI gate can see a stale
+  cassette or live-only assertions it didn't evaluate WITHOUT the verdict changing — a stale but otherwise
+  passing replay stays `ok:true` by default. Previously these were stderr-only `::warning::` lines invisible
+  to a JSON consumer.
+- **`replay --fail-on-skill-drift`.** A narrower release gate than `--strict`: fails only on skill-source drift
+  (`skill` / `shared-root` / `unverifiable-skill` — "can't verify the skill ⇒ not green"), while baseline /
+  format / environment-level staleness stays a non-failing warning. `--strict` remains the superset (fails on
+  every class).
+
+### Fixed
+
+- **Linter no longer false-flags `requires_capabilities`.** `cowork-harness lint` warned `unknown-top-key` on
+  the valid scenario field `requires_capabilities` (its hand-maintained top-level-key list had drifted from the
+  schema). The list is now generated from the Zod `ScenarioObject` schema (like the assertion-key list), so it
+  can't drift; a parity test guards it.
+- **SKILL.md capability pre-flight wording corrected.** It described `requires_capabilities` as skill-declared
+  and said the harness "warns before the run"; it is a **scenario field** and the harness **aborts before the
+  paid run (exit 3)** when the image omits a required capability (unless `allow_missing_capability: true`). The
+  authoritative `docs/scenario.md` was already correct.
+
 ## [0.13.0] — 2026-06-25
 
 ### Added

package/README.md

@@ -93,7 +93,7 @@ Skill testing is the headline use, but the tool is a general harness over the Co
 | `skill <folder> "<prompt>"` | Run a local skill/plugin folder once against the staged agent | ad-hoc "is the skill alive / does it do X?" — the fast inner loop |
 | `run <scenario.yaml \| dir/>` | Run authored scenarios with `assert:` + a CI-ready exit code | you want a repeatable, **asserted regression test** |
 | `chat <folder> [prompt]` | Interactive multi-turn REPL against a skill (TTY); optional seed prompt is sent as the first turn. `--upload <file>` / `--folder <dir>` (repeatable) attach files/project folders; `--verbose` / `-V` shows thinking blocks + tool inputs; `--fidelity protocol\|container\|hostloop` | debugging a multi-turn flow by hand |
-| `record` / `replay` | `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on a stale one; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
+| `record` / `replay` | `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on any stale one, `--fail-on-skill-drift` only on skill-source drift; staleness is also reported per-result in `staleness[]` for a JSON gate; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
 | `verify-cassettes <file\|dir>` | Token-free CI gate over committed cassettes: a privacy scan (email/currency/domain → exit 1; whole-token allows via `--allow` / class-scoped `--allow-domain` / `--allow-email` / `--allow-file`) + a staleness check (`--skip-privacy` or `--skip-staleness` to run only one half). A dir argument scans `*.cassette.json` in that dir only (**non-recursive**) | gating **committed cassettes** against PII leaks + "edited the skill, forgot to re-record" |
 | `verify-run <run-dir> <scenario.yaml>` | Re-evaluate a scenario's `assert:` (and, when the scenario declares `answers:`, whether they still match the run's actual gates) against an already-kept run dir — **no live agent, no tokens, no Docker** (~1s) | iterating on a wrong assertion or a drifted `answer` without a full live re-record |
 | `trace <run-id>` | Digest a run's `events.jsonl` (`--view tools\|questions\|dispatches`; legacy `--tools` / `--gates` / `--dispatches` aliases still accepted) | "how many sub-agents *actually* dispatched, and which?" |
@@ -208,7 +208,7 @@ This repo ships a **companion skill** (`.claude/skills/cowork-harness/`) that te
 /plugin install cowork-harness@cowork-harness
 ```
 
-The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.13.0` (a version floor that fails loud rather than silently fetching a too-old CLI; Node ≥ 20). Tiers above `protocol` still need Docker/Lima and a Claude Desktop agent binary — see the prerequisites below.
+The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.14.0` (a version floor that fails loud rather than silently fetching a too-old CLI; Node ≥ 20). Tiers above `protocol` still need Docker/Lima and a Claude Desktop agent binary — see the prerequisites below.
 
 It also follows the open [Agent Skills](https://github.com/vercel-labs/skills) spec, so it installs cross-editor (Cursor, Codex, OpenCode, …) by pointing the `npx skills` CLI at `.claude/skills/cowork-harness` in this repo. (Working *inside* this repo, the skill auto-loads as a project skill — no install needed.)
 

package/dist/cli.js

@@ -72,7 +72,8 @@ const HELP = `cowork-harness <command>   (v${"$VERSION"})
       [--decider-llm [--intent "…"]] | [--on-unanswered fail|first]   answer live via a model / auto-pick
                                (a live decider flags the cassette non-deterministic; it still replays deterministically)
   replay <file|dir>            deterministic protocol-replay of a cassette or a dir of them (no token, no Docker)
-      [--strict]               fail (exit 1) on a stale cassette instead of warning
+      [--strict]               fail (exit 1) on ANY stale cassette instead of warning
+      [--fail-on-skill-drift]  fail only on skill-source drift (skill/shared-root); baseline drift stays a warning
       [--output-format json]
   verify-cassettes <file|dir>  CI gate (no token): privacy scan + staleness — exit 1 on finding or drift
       [--skip-privacy|--skip-staleness]  skip one check
@@ -262,7 +263,7 @@ const SUBCOMMAND_USAGE = {
         '       answer gates LIVE: [--decider-dir <dir>] (single scenario only) | [--decider-llm [--intent "<one line>"]] | [--on-unanswered fail|first]\n' +
         "       (a live decider flags the cassette non-deterministic — re-recording may drift; replay stays deterministic. --rerecord-stale rejects these flags.)\n" +
         "       NOTE: --allow-failing only relaxes the post-run VERDICT gate; it does NOT salvage an unanswered gate (that throws before any cassette is written — use --on-unanswered first / a decider).",
-    replay: "usage: replay <file.cassette.json | dir/> [--strict] [--output-format text|json]",
+    replay: "usage: replay <file.cassette.json | dir/> [--strict] [--fail-on-skill-drift] [--output-format text|json]",
     "verify-cassettes": "usage: verify-cassettes <file|dir> [--skip-privacy|--skip-staleness] [--allow <regex>]... [--allow-domain <regex>]... [--allow-email <regex>]... [--allow-file <path>]... [--output-format json]",
     trace: "usage: trace <run-id | run-dir | events.jsonl> [--view tools|questions|dispatches] [--output-format json]\n       --view tools       tool call / result rows\n       --view questions   gate lifecycle (question → answer → delivered)\n       --view dispatches  sub-agent dispatch tree + dispatch_count_max\n       (default: all views)\n       (for what the run PRODUCED — artifacts — use `inspect`)",
     assertions: "usage: assertions --list [--output-format json]",

package/dist/io.js

@@ -1,11 +1,12 @@
 /**
  * Emit a structured warning to stderr with the GitHub-actions `::warning::` annotation prefix — the one
- * place warning formatting/severity lives, so call sites pass only the message body. Accepts a string
- * that already carries the prefix (written as-is) so migrating existing `::warning:: …` writes is a
- * no-op on content. Uses `process.stderr.write` (the mechanism the warnings always used) so test spies
- * on it still observe warnings and the output is byte-identical.
+ * place warning formatting/severity lives, so call sites pass only the message body. A message that ALREADY
+ * carries a GitHub annotation prefix (`::warning::`, `::notice::`, or `::error::`) is written as-is, so a
+ * call site that wants a softer/harder severity (e.g. `::notice:: …`) gets exactly that — NOT a doubled
+ * `::warning:: ::notice:: …`. Uses `process.stderr.write` (the mechanism the warnings always used) so test
+ * spies on it still observe warnings and the output is byte-identical for plain `::warning::` content.
  */
 export function warn(message) {
-    const line = message.startsWith("::warning::") ? message : `::warning:: ${message}`;
+    const line = /^::(warning|notice|error)::/.test(message) ? message : `::warning:: ${message}`;
     process.stderr.write(line.endsWith("\n") ? line : line + "\n");
 }

package/dist/run/cassette.js

@@ -1520,7 +1520,7 @@ export async function cmdReplay(args) {
     try {
         p = parseArgs(args, {
             // --quiet/--verbose accepted for flag consistency but currently no-op in replay (renderer plan is fixed).
-            booleans: ["--strict", "--quiet", "--verbose"],
+            booleans: ["--strict", "--fail-on-skill-drift", "--quiet", "--verbose"],
             values: ["--output-format"],
             enums: { "--output-format": ["text", "json"] },
             aliases: { "-q": "--quiet", "-V": "--verbose" },
@@ -1532,7 +1532,7 @@ export async function cmdReplay(args) {
     }
     const target = p.positionals[0];
     if (!target) {
-        log("usage: replay <file.cassette.json | dir/> [--strict] [--output-format text|json]");
+        log("usage: replay <file.cassette.json | dir/> [--strict] [--fail-on-skill-drift] [--output-format text|json]");
         return process.exit(2);
     }
     if (p.positionals.length > 1) {
@@ -1540,7 +1540,10 @@ export async function cmdReplay(args) {
         return process.exit(2);
     }
     const json = p.options["--output-format"] === "json";
-    const strict = p.flags["--strict"] ?? false; // #1b: escalate staleness warnings to failures (release gate)
+    const strict = p.flags["--strict"] ?? false; // #1b: escalate ALL staleness findings to failures (release gate)
+    const failOnSkillDrift = p.flags["--fail-on-skill-drift"] ?? false; // narrower gate: only skill-source drift fails
+    if (strict && failOnSkillDrift)
+        warn("::notice:: [replay] --strict and --fail-on-skill-drift both passed — --strict is the superset (fails on every class), so --fail-on-skill-drift is redundant here\n");
     const resolved = resolveInputs(target, ".cassette.json");
     if ("error" in resolved) {
         log(`replay: ${resolved.error}`);
@@ -1564,7 +1567,7 @@ export async function cmdReplay(args) {
         // fine" (false-green by abort). Wrap per-file: turn an unexpected throw into a tallied error result.
         let result;
         try {
-            result = await replayCassette(rc.cassette, renderer ? [renderer] : [], { strict, cassetteDir: dirname(f) });
+            result = await replayCassette(rc.cassette, renderer ? [renderer] : [], { strict, failOnSkillDrift, cassetteDir: dirname(f) });
         }
         catch (e) {
             log(`replay: ${f}: ${e?.message ?? String(e)}`);
@@ -1860,7 +1863,10 @@ export function cmdRehash(args) {
 }
 /** Replay a cassette through Run and re-evaluate the content assertions. With a `cassette.artifacts`
  *  manifest (#1), filesystem assertions (file_exists/user_visible_artifact/artifact_json) ALSO run, against
- *  the materialized snapshot. `opts.strict` (#1b) escalates staleness warnings to failing assertions. */
+ *  the materialized snapshot. `opts.strict` (#1b) escalates ALL staleness findings to failing assertions;
+ *  `opts.failOnSkillDrift` escalates only the skill-source classes (`skill`/`shared-root`/`unverifiable-skill`),
+ *  leaving baseline drift a non-failing warning. Either way the findings are always surfaced in
+ *  `RunResult.staleness` for JSON consumers. */
 export async function replayCassette(cassette, hooks = [], opts = {}) {
     // Cassette format version: ABSENT = legacy (0); a FUTURE version means this harness may misread fields
     // it doesn't know about. in strict mode this is a hard failure (future semantics may not be
@@ -1884,6 +1890,11 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
     // unchanged (frozen-structure limit). The skill-hash recompute needs the local skill dirs to be resolvable
     // from the cassette's session path; when they aren't (a moved/committed cassette), we say so rather than
     // silently skipping.
+    // Findings are pushed UNCONDITIONALLY (class-tagged) so they're surfaced in JSON (RunResult.staleness) even
+    // on the default gate — a token-free consumer can then distinguish "verified clean" from "couldn't verify"
+    // (the `unverifiable-*` classes) WITHOUT the verdict changing. The `--strict` / `--fail-on-skill-drift`
+    // gates below are the ONLY place a finding becomes a failing assertion. The single `warn()` loop at the end
+    // is the lone stderr emitter — no per-branch `warn()`, so a non-strict run never double-warns one cause.
     const staleness = [];
     if (cassette.fingerprint) {
         const fp = cassette.fingerprint;
@@ -1894,48 +1905,54 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
         }
         catch {
             baselineLoadFailed = true;
-            if (opts.strict) {
-                staleness.push("baseline could not be loaded — cannot verify staleness (--strict: treating as stale)");
-            }
-            else {
-                warn("cowork-harness: strict staleness check skipped — baseline could not be loaded\n");
-            }
+            staleness.push({
+                class: "unverifiable-baseline",
+                message: "baseline could not be loaded — cannot verify staleness (env/platform, not skill drift)",
+            });
         }
         if (!baselineLoadFailed && liveBaseline && liveBaseline !== fp.baseline)
-            staleness.push(`baseline moved ${fp.baseline} → ${liveBaseline} since record — re-record before trusting this replay`);
+            staleness.push({
+                class: "baseline",
+                message: `baseline moved ${fp.baseline} → ${liveBaseline} since record — re-record before trusting this replay`,
+            });
         if (fp.skillHash) {
             const live = buildFingerprint(cassette.scenario.session, fp.baseline, opts.cassetteDir, cassette.scenario.skills);
             if (live.skillHash === undefined) {
-                // under --strict, unresolvable skill dirs are a failure (can't verify ⇒ not green).
-                // Without --strict, warn but continue (the baseline check still applies).
-                if (opts.strict) {
-                    staleness.push("skill dirs not resolvable from this cassette location — cannot verify skill staleness (--strict: treating as stale)");
-                }
-                else {
-                    warn("::warning:: [replay] skill fingerprint not re-checkable (local skill dirs not resolvable from this cassette location) — baseline check still applies\n");
-                }
+                staleness.push({
+                    class: "unverifiable-skill",
+                    message: "skill dirs not resolvable from this cassette location — cannot verify skill staleness (the baseline check still applies)",
+                });
             }
             else if (live.skillHash !== fp.skillHash) {
                 const recordedVersion = cassette.cassetteVersion ?? 0;
                 if (recordedVersion < CASSETTE_VERSION) {
-                    staleness.push(`recorded under an older hash format (v${recordedVersion} → v${CASSETTE_VERSION}) — re-record once after upgrading`);
+                    staleness.push({
+                        class: "format",
+                        message: `recorded under an older hash format (v${recordedVersion} → v${CASSETTE_VERSION}) — re-record once after upgrading`,
+                    });
                 }
                 else if (fp.sharedHash !== undefined && live.sharedHash !== undefined) {
                     const scope = fp.skillScope?.length ? fp.skillScope.map((s) => `skills/${s}`).join(", ") : "skill";
                     if (live.sharedHash !== fp.sharedHash) {
-                        staleness.push(`shared root changed since record (scope: ${scope}) — re-record before trusting this replay`);
+                        staleness.push({
+                            class: "shared-root",
+                            message: `shared root changed since record (scope: ${scope}) — re-record before trusting this replay`,
+                        });
                     }
                     else {
-                        staleness.push(`${scope} changed since record — re-record before trusting this replay`);
+                        staleness.push({ class: "skill", message: `${scope} changed since record — re-record before trusting this replay` });
                     }
                 }
                 else {
-                    staleness.push("local skill/plugin dir contents changed since record — re-record before trusting this replay");
+                    staleness.push({
+                        class: "skill",
+                        message: "local skill/plugin dir contents changed since record — re-record before trusting this replay",
+                    });
                 }
             }
         }
         for (const s of staleness)
-            warn(`::warning:: [replay] cassette stale: ${s}\n`);
+            warn(`::warning:: [replay] cassette stale: ${s.message}\n`);
     }
     // §2.5 backward compat: warn loudly when controlOut is absent so the user knows question/gate
     // assertions are being EXCLUDED (not vacuously evaluated) from this run.
@@ -2085,10 +2102,17 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
         toolResultTexts: rec.toolResults.map((r) => r.assertText ?? r.text),
         truncatedPaths: replayTruncatedPaths,
     });
-    // #1b: under --strict, a staleness mismatch is a failing assertion (non-zero exit), not just a warning.
+    // #1b: under --strict, EVERY staleness finding becomes a failing assertion (non-zero exit), not just a
+    // warning. --fail-on-skill-drift is the narrower gate: only the skill-source classes fail (incl.
+    // `unverifiable-skill` — can't verify skill staleness ⇒ not green), while baseline / format / env-level
+    // findings stay non-failing. `else if` makes --strict the superset when both are passed.
+    const SKILL_DRIFT_CLASSES = new Set(["skill", "shared-root", "unverifiable-skill"]);
     if (opts.strict)
         for (const s of staleness)
-            assertions.push({ assertion: {}, pass: false, message: `cassette stale (--strict): ${s}` });
+            assertions.push({ assertion: {}, pass: false, message: `cassette stale (--strict): ${s.message}` });
+    else if (opts.failOnSkillDrift)
+        for (const s of staleness.filter((s) => SKILL_DRIFT_CLASSES.has(s.class)))
+            assertions.push({ assertion: {}, pass: false, message: `skill-source drift (--fail-on-skill-drift): ${s.message}` });
     // future cassette version — hard failure under --strict (forward semantics may not be
     // correctly interpreted here, so a green replay would be a false-green).
     if (futureVersionMsg && opts.strict)
@@ -2175,6 +2199,10 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
         subagents: rec.subagents,
         unanswered: rec.unanswered,
         outDir: "(replay)",
+        // Class-tagged staleness + skip counts, surfaced to JSON callers (the gate decision already happened
+        // above via failing assertions; these fields are pure data so a green stays green by default).
+        staleness: staleness.length ? staleness : undefined,
+        skippedAssertions: { full: fullSkipCount, partial: partialSkipCount },
         // A cassette freezes the answer path: the replay itself is deterministic regardless of how the
         // original run was answered. Always explicit (never undefined) so renderer.ts:146 treats it
         // correctly — undefined would silently render as "deterministic" (#47 C1 [review-2]).

package/docs/cassette.md

@@ -211,8 +211,9 @@ with `record --max-artifact-bytes <n>` (or `COWORK_HARNESS_MAX_ARTIFACT_BYTES`)
 stays replay-checkable, and `record` fails fast if an `artifact_json` targets an artifact it had to truncate (that
 would pass at record but fail at replay). Without a manifest (older
 cassettes), these are skipped. A green replay re-confirms *record-time* artifacts, **not** that the current
-skill still produces them — `replay --strict` fails the run when the `fingerprint` shows the skill/baseline
-drifted.
+skill still produces them — `replay --strict` fails the run when the `fingerprint` shows ANY skill/baseline
+drift, or `replay --fail-on-skill-drift` fails only on skill-source drift (leaving baseline drift a warning).
+Either way, every replay result also reports the drift in `staleness[]` (class-tagged) for a JSON gate to read.
 
 ### Still skipped on replay (no filesystem/network in a cassette)
 

package/docs/scenario.md

@@ -340,7 +340,9 @@ small JSON bodies) into the cassette, and `replay` materializes that snapshot to
 `artifact_json` needs the JSON body inlined (small files); a hash-only (oversized) entry still satisfies
 `file_exists` but not `artifact_json`. Without a manifest (older cassettes), they are **skipped** (loud).
 A green `replay` re-confirms *record-time* artifacts, **not** that the current skill still produces them —
-that needs a live `run` (the cassette's `--strict` staleness fingerprint warns when the skill/baseline drifted).
+that needs a live `run` (the cassette's staleness fingerprint warns when the skill/baseline drifted; `replay
+--strict` fails on any drift, `--fail-on-skill-drift` on skill-source drift only, and every result reports it
+in `staleness[]` for a JSON gate).
 
 **Egress + other filesystem** assertions (`no_delete_in_outputs`, `self_heal_ran`,
 `transcript_no_host_path`, `egress_*`/`expect_denied`) are still **skipped** on `replay` — they only run on

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cowork-harness",
-  "version": "0.13.0",
+  "version": "0.14.0",
   "description": "Scriptable, CI-friendly harness for Claude Cowork's runtime contract for testing skills across scenarios — same agent, mounts, egress allowlist, permission protocol, and sandbox limitations.",
   "license": "MIT",
   "type": "module",

package/scripts/gen-schema.ts

@@ -19,14 +19,21 @@ export const SCHEMA_DIR = join(REPO_ROOT, "schema");
  *  is. Writer + the drift-guard test both reference this one constant. */
 export const ASSERTION_KEYS_PATH = join(REPO_ROOT, ".claude/skills/cowork-harness/scripts/assertion-keys.json");
 
-/** The authoritative assertion-key list, derived from the Zod `Assertion` schema (the same source
- *  `assert --list` reads). Generating it keeps `scenario.py`'s unknown-key check from drifting. */
+/** The authoritative key lists `scenario.py` reads — derived from the Zod schemas (the same source
+ *  `assert --list` reads). Generating them keeps the linter's unknown-key checks from drifting: `keys` is
+ *  the `assert:` catalog, `topLevelKeys` the scenario top-level catalog (an earlier hand-maintained copy
+ *  drifted and false-flagged the valid `requires_capabilities`). `profile`/`assertions` are NOT here — they
+ *  are deprecated aliases handled by scenario.py's special-cases (the Zod `Scenario` preprocess remaps
+ *  `profile`; `assertions` is a hard error), so they are intentionally absent from the schema shape. */
 export function buildAssertionKeys(): string {
   return (
     JSON.stringify(
       {
-        $comment: "GENERATED from the Zod Assertion schema (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
+        $comment: "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
         keys: Object.keys(Assertion.shape).sort(),
+        // Every valid top-level scenario key, from the ScenarioObject strictObject shape (NOT the `Scenario`
+        // preprocess wrapper). scenario.py keeps an embedded fallback parity-tested against this.
+        topLevelKeys: Object.keys(ScenarioObject.shape).sort(),
         // The verdict-modifier subset (no-op assertions that suppress a default-fail). scenario.py keeps a
         // hardcoded copy parity-tested against this; see VERDICT_MODIFIER_KEYS in src/types.ts.
         verdictModifierKeys: [...VERDICT_MODIFIER_KEYS].sort(),