cowork-harness 0.13.0 → 0.15.0

package/.claude/skills/cowork-harness/scripts/assertion-keys.json

@@ -1,5 +1,5 @@
 {
-  "$comment": "GENERATED from the Zod Assertion schema (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
+  "$comment": "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
   "keys": [
     "allow_l0_plugin_divergence",
     "allow_missing_capability",
@@ -32,6 +32,19 @@
     "transcript_not_matches",
     "user_visible_artifact"
   ],
+  "topLevelKeys": [
+    "answers",
+    "assert",
+    "baseline",
+    "expect_denied",
+    "fidelity",
+    "name",
+    "on_unanswered",
+    "prompt",
+    "requires_capabilities",
+    "session",
+    "skills"
+  ],
   "verdictModifierKeys": [
     "allow_l0_plugin_divergence",
     "allow_missing_capability",

package/.claude/skills/cowork-harness/scripts/scenario.py

@@ -116,8 +116,11 @@ if UNCLASSIFIED_KEYS:
         "— add them to the linter's CONTENT/GATE/MANIFEST/LIVE_ONLY/VERDICT_MODIFIER sets.",
         file=sys.stderr,
     )
-# every valid top-level scenario key
-TOP_LEVEL_KEYS = {
+# Embedded fallback for the top-level scenario keys — kept EQUAL to the generated `topLevelKeys`
+# (test-enforced, like _CLASSIFIED_KEYS for assert keys) so a missing assertion-keys.json can't silently
+# reintroduce key drift. `profile`/`assertions` are NOT here — they are deprecated aliases handled by the
+# special-cases in the unknown-key check below (`k != "assertions" and k != "profile"`).
+_EMBEDDED_TOP_LEVEL_KEYS = {
     "name",
     "baseline",
     "session",
@@ -128,7 +131,32 @@ TOP_LEVEL_KEYS = {
     "expect_denied",
     "assert",
     "skills",  # F-6: opt-in skill-staleness hash scope
+    "requires_capabilities",  # Fix 4b: scenario-level required-capability declaration (pre-flight gate)
 }
+
+
+def _load_top_level_keys():
+    """The authoritative top-level scenario-key set, generated from the Zod ScenarioObject schema into
+    `assertion-keys.json` (so the unknown-key check can't drift and false-flag a valid key). Falls back to
+    the embedded `_EMBEDDED_TOP_LEVEL_KEYS` (kept equal to the generated list) with a loud warning if the
+    file is missing or predates the `topLevelKeys` field."""
+    p = Path(__file__).resolve().parent / "assertion-keys.json"
+    try:
+        keys = json.loads(p.read_text(encoding="utf-8")).get("topLevelKeys")
+        if not keys:
+            raise KeyError("topLevelKeys")
+        return set(keys)
+    except Exception:
+        print(
+            f"::warning:: assertion-keys.json missing or has no topLevelKeys next to scenario.py ({p}) — "
+            "using a built-in top-level-key list that may be stale (run `npm run schema`).",
+            file=sys.stderr,
+        )
+        return set(_EMBEDDED_TOP_LEVEL_KEYS)
+
+
+# every valid top-level scenario key (generated from the zod ScenarioObject schema; see _load_top_level_keys)
+TOP_LEVEL_KEYS = _load_top_level_keys()
 REGEX_KEYS = {
     "transcript_matches",
     "transcript_not_matches",

package/CHANGELOG.md

@@ -6,6 +6,54 @@ All notable changes to this project are documented here. The format is based on
 
 ## [Unreleased]
 
+## [0.15.0] — 2026-06-25
+
+### Fixed
+
+- **Staleness no longer masks a scoped skill's own drift behind a co-occurring shared change.** For a
+  `skills:`-scoped cassette, when BOTH the shared roots AND the scoped skill's own files changed, the
+  diagnosis previously reported only `shared root changed` and never the skill — the two buckets were
+  mutually exclusive (shared tested first). It now attributes drift per-bucket by the actual changed paths and
+  emits BOTH a `shared-root` and a `skill` finding when both moved, each naming its own files. The same
+  diagnosis (per-file `[N changed (…)]` detail and the `COWORK_HARNESS_DEBUG_SKILLHASH` hook) now also runs on
+  the `replay` lane, which previously had a separate, less-detailed copy. With `COWORK_HARNESS_AGENT_SCOPE=skill`
+  a changed `agents/<skill>.md` is attributed to that skill, matching the hash boundary.
+
+### Changed
+
+- **`replay --fail-on-skill-drift` no longer fails on a `COWORK_HARNESS_GITSET` / `COWORK_HARNESS_AGENT_SCOPE`
+  flip.** A record-vs-verify mismatch in either setting is now classed `format` ("re-record under the same
+  mode") rather than misattributed to skill/shared drift, so it is a non-failing warning under
+  `--fail-on-skill-drift` (which targets skill-source drift only). It still fails under `--strict` and still
+  reds `verify-cassettes`. Previously the `replay` lane mislabeled such a flip as `shared-root`/`skill` and
+  failed the skill-drift gate.
+
+## [0.14.0] — 2026-06-25
+
+### Added
+
+- **`replay` surfaces staleness + skipped assertions in JSON.** Each `--output-format json` result now carries
+  `staleness[]` (class-tagged: `baseline` / `skill` / `shared-root` / `format` / `unverifiable-baseline` /
+  `unverifiable-skill`) and `skippedAssertions` (`{full, partial}`), so a token-free CI gate can see a stale
+  cassette or live-only assertions it didn't evaluate WITHOUT the verdict changing — a stale but otherwise
+  passing replay stays `ok:true` by default. Previously these were stderr-only `::warning::` lines invisible
+  to a JSON consumer.
+- **`replay --fail-on-skill-drift`.** A narrower release gate than `--strict`: fails only on skill-source drift
+  (`skill` / `shared-root` / `unverifiable-skill` — "can't verify the skill ⇒ not green"), while baseline /
+  format / environment-level staleness stays a non-failing warning. `--strict` remains the superset (fails on
+  every class).
+
+### Fixed
+
+- **Linter no longer false-flags `requires_capabilities`.** `cowork-harness lint` warned `unknown-top-key` on
+  the valid scenario field `requires_capabilities` (its hand-maintained top-level-key list had drifted from the
+  schema). The list is now generated from the Zod `ScenarioObject` schema (like the assertion-key list), so it
+  can't drift; a parity test guards it.
+- **SKILL.md capability pre-flight wording corrected.** It described `requires_capabilities` as skill-declared
+  and said the harness "warns before the run"; it is a **scenario field** and the harness **aborts before the
+  paid run (exit 3)** when the image omits a required capability (unless `allow_missing_capability: true`). The
+  authoritative `docs/scenario.md` was already correct.
+
 ## [0.13.0] — 2026-06-25
 
 ### Added

package/README.md

@@ -93,7 +93,7 @@ Skill testing is the headline use, but the tool is a general harness over the Co
 | `skill <folder> "<prompt>"` | Run a local skill/plugin folder once against the staged agent | ad-hoc "is the skill alive / does it do X?" — the fast inner loop |
 | `run <scenario.yaml \| dir/>` | Run authored scenarios with `assert:` + a CI-ready exit code | you want a repeatable, **asserted regression test** |
 | `chat <folder> [prompt]` | Interactive multi-turn REPL against a skill (TTY); optional seed prompt is sent as the first turn. `--upload <file>` / `--folder <dir>` (repeatable) attach files/project folders; `--verbose` / `-V` shows thinking blocks + tool inputs; `--fidelity protocol\|container\|hostloop` | debugging a multi-turn flow by hand |
-| `record` / `replay` | `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on a stale one; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
+| `record` / `replay` | `record` saves a control-protocol cassette (one scenario, or batch a `dir/`; `--rerecord-stale` refreshes only drifted ones; `--concurrency <N>` records a batch N at a time — runs are fully isolated, so it just bounds Docker/API pressure; `--max-artifact-bytes` raises the 64 KiB inline-body cap; `--dry-run` previews the scenarios + token/binary checks without recording; answer gates **live during the recording** with `--decider-dir`/`--decider-llm`/`--on-unanswered first` instead of pre-scripting them — a live decider flags the cassette non-deterministic but it still replays deterministically); `replay <file.cassette.json \| dir/>` runs a cassette **file or a `dir/` of `*.cassette.json`** deterministically (`--strict` fails on any stale one, `--fail-on-skill-drift` only on skill-source drift; staleness is also reported per-result in `staleness[]` for a JSON gate; exits on the worst verdict) | **token-free, Docker-free CI** from a once-recorded run |
 | `verify-cassettes <file\|dir>` | Token-free CI gate over committed cassettes: a privacy scan (email/currency/domain → exit 1; whole-token allows via `--allow` / class-scoped `--allow-domain` / `--allow-email` / `--allow-file`) + a staleness check (`--skip-privacy` or `--skip-staleness` to run only one half). A dir argument scans `*.cassette.json` in that dir only (**non-recursive**) | gating **committed cassettes** against PII leaks + "edited the skill, forgot to re-record" |
 | `verify-run <run-dir> <scenario.yaml>` | Re-evaluate a scenario's `assert:` (and, when the scenario declares `answers:`, whether they still match the run's actual gates) against an already-kept run dir — **no live agent, no tokens, no Docker** (~1s) | iterating on a wrong assertion or a drifted `answer` without a full live re-record |
 | `trace <run-id>` | Digest a run's `events.jsonl` (`--view tools\|questions\|dispatches`; legacy `--tools` / `--gates` / `--dispatches` aliases still accepted) | "how many sub-agents *actually* dispatched, and which?" |
@@ -208,7 +208,7 @@ This repo ships a **companion skill** (`.claude/skills/cowork-harness/`) that te
 /plugin install cowork-harness@cowork-harness
 ```
 
-The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.13.0` (a version floor that fails loud rather than silently fetching a too-old CLI; Node ≥ 20). Tiers above `protocol` still need Docker/Lima and a Claude Desktop agent binary — see the prerequisites below.
+The skill **self-bootstraps the CLI**: if `cowork-harness` isn't on your PATH it falls back to `npx cowork-harness@>=0.15.0` (a version floor that fails loud rather than silently fetching a too-old CLI; Node ≥ 20). Tiers above `protocol` still need Docker/Lima and a Claude Desktop agent binary — see the prerequisites below.
 
 It also follows the open [Agent Skills](https://github.com/vercel-labs/skills) spec, so it installs cross-editor (Cursor, Codex, OpenCode, …) by pointing the `npx skills` CLI at `.claude/skills/cowork-harness` in this repo. (Working *inside* this repo, the skill auto-loads as a project skill — no install needed.)
 

package/dist/cli.js

@@ -72,7 +72,8 @@ const HELP = `cowork-harness <command>   (v${"$VERSION"})
       [--decider-llm [--intent "…"]] | [--on-unanswered fail|first]   answer live via a model / auto-pick
                                (a live decider flags the cassette non-deterministic; it still replays deterministically)
   replay <file|dir>            deterministic protocol-replay of a cassette or a dir of them (no token, no Docker)
-      [--strict]               fail (exit 1) on a stale cassette instead of warning
+      [--strict]               fail (exit 1) on ANY stale cassette instead of warning
+      [--fail-on-skill-drift]  fail only on skill-source drift (skill/shared-root); baseline drift stays a warning
       [--output-format json]
   verify-cassettes <file|dir>  CI gate (no token): privacy scan + staleness — exit 1 on finding or drift
       [--skip-privacy|--skip-staleness]  skip one check
@@ -262,7 +263,7 @@ const SUBCOMMAND_USAGE = {
         '       answer gates LIVE: [--decider-dir <dir>] (single scenario only) | [--decider-llm [--intent "<one line>"]] | [--on-unanswered fail|first]\n' +
         "       (a live decider flags the cassette non-deterministic — re-recording may drift; replay stays deterministic. --rerecord-stale rejects these flags.)\n" +
         "       NOTE: --allow-failing only relaxes the post-run VERDICT gate; it does NOT salvage an unanswered gate (that throws before any cassette is written — use --on-unanswered first / a decider).",
-    replay: "usage: replay <file.cassette.json | dir/> [--strict] [--output-format text|json]",
+    replay: "usage: replay <file.cassette.json | dir/> [--strict] [--fail-on-skill-drift] [--output-format text|json]",
     "verify-cassettes": "usage: verify-cassettes <file|dir> [--skip-privacy|--skip-staleness] [--allow <regex>]... [--allow-domain <regex>]... [--allow-email <regex>]... [--allow-file <path>]... [--output-format json]",
     trace: "usage: trace <run-id | run-dir | events.jsonl> [--view tools|questions|dispatches] [--output-format json]\n       --view tools       tool call / result rows\n       --view questions   gate lifecycle (question → answer → delivered)\n       --view dispatches  sub-agent dispatch tree + dispatch_count_max\n       (default: all views)\n       (for what the run PRODUCED — artifacts — use `inspect`)",
     assertions: "usage: assertions --list [--output-format json]",

package/dist/io.js

@@ -1,11 +1,12 @@
 /**
  * Emit a structured warning to stderr with the GitHub-actions `::warning::` annotation prefix — the one
- * place warning formatting/severity lives, so call sites pass only the message body. Accepts a string
- * that already carries the prefix (written as-is) so migrating existing `::warning:: …` writes is a
- * no-op on content. Uses `process.stderr.write` (the mechanism the warnings always used) so test spies
- * on it still observe warnings and the output is byte-identical.
+ * place warning formatting/severity lives, so call sites pass only the message body. A message that ALREADY
+ * carries a GitHub annotation prefix (`::warning::`, `::notice::`, or `::error::`) is written as-is, so a
+ * call site that wants a softer/harder severity (e.g. `::notice:: …`) gets exactly that — NOT a doubled
+ * `::warning:: ::notice:: …`. Uses `process.stderr.write` (the mechanism the warnings always used) so test
+ * spies on it still observe warnings and the output is byte-identical for plain `::warning::` content.
  */
 export function warn(message) {
-    const line = message.startsWith("::warning::") ? message : `::warning:: ${message}`;
+    const line = /^::(warning|notice|error)::/.test(message) ? message : `::warning:: ${message}`;
     process.stderr.write(line.endsWith("\n") ? line : line + "\n");
 }

package/dist/run/cassette.js

@@ -22,7 +22,7 @@ import { jsonEnvelope } from "./envelope.js";
 import { parseArgs } from "../cli-args.js";
 import { resolveInputs } from "./inputs.js";
 import { realProbe } from "./doctor.js";
-import { hashSkillDirs, hashSharedOnly, computeContentSig, skillHashEntries, OS_JUNK_PATTERN } from "./skill-hash.js";
+import { hashSkillDirs, hashSharedOnly, computeContentSig, skillHashEntries, OS_JUNK_PATTERN, agentSkillName } from "./skill-hash.js";
 import { computeVerdict } from "./verdict.js";
 import { redactJsonLine, redactText, redactStructural, loadRedactionPolicy } from "../redact.js";
 import { collectSecrets, scrubField } from "../secrets.js";
@@ -424,12 +424,8 @@ function debugSkillHashMismatch(cassette, cassetteDir, fp, live) {
         `contentSig ${sigVerdict}${junk ? ` · ${junk} OS-junk file(s) flagged above` : ""}. ` +
         `Note: this lists the CURRENT tree; a true per-file diff vs record needs the record-time set (re-record after excluding junk, then compare).\n`);
 }
-/** v5: diff two per-file manifests (recorded vs live) into an exact "what changed" SUMMARY — the actionable
- *  upgrade over the bucket-level "something changed". Returns just the summary (no prefix/suffix), so the
- *  caller can append it to the bucket/generic message (preserving the G-4 shared-vs-scoped semantic). Samples
- *  up to 3 paths per category. Null when the manifests are equal (hashes differ but files don't — caller
- *  falls back to its bucket message). */
-function diffFileSigs(recorded, live) {
+/** v5: diff two per-file manifests (recorded vs live) into the exact changed/added/removed path lists. */
+function diffFileSigsPaths(recorded, live) {
     const rec = new Map(recorded);
     const liv = new Map(live);
     const added = [];
@@ -444,26 +440,67 @@ function diffFileSigs(recorded, live) {
     for (const [p] of rec)
         if (!liv.has(p))
             removed.push(p);
-    if (!added.length && !removed.length && !changed.length)
+    return { added, removed, changed };
+}
+/** Format a {@link FileSigDiff} into an actionable summary (samples up to 3 paths per category). Null when
+ *  the diff is empty (hashes differ but files don't — a structure-only change; caller falls back to its
+ *  bucket message). */
+function summarizeFileSigDiff(diff) {
+    if (!diff.added.length && !diff.removed.length && !diff.changed.length)
         return null;
     const sample = (a) => `${a.slice(0, 3).join(", ")}${a.length > 3 ? `, +${a.length - 3} more` : ""}`;
     const parts = [];
-    if (changed.length)
-        parts.push(`${changed.length} changed (${sample(changed)})`);
-    if (added.length)
-        parts.push(`${added.length} added (${sample(added)})`);
-    if (removed.length)
-        parts.push(`${removed.length} removed (${sample(removed)})`);
+    if (diff.changed.length)
+        parts.push(`${diff.changed.length} changed (${sample(diff.changed)})`);
+    if (diff.added.length)
+        parts.push(`${diff.added.length} added (${sample(diff.added)})`);
+    if (diff.removed.length)
+        parts.push(`${diff.removed.length} removed (${sample(diff.removed)})`);
     return parts.join("; ");
 }
-/** B3 staleness GATE: recompute the fingerprint and report drift. Unlike `replayCassette` (which WARNS),
- *  the gate treats an unresolvable skillHash as a failure — can't verify ⇒ not green. No fingerprint → nothing
- *  to check (legacy cassette). */
-export function checkStaleness(cassette, cassetteDir) {
+/** Convenience: diff + summarize in one call (the non-scoped/whole-tree caller wants only the summary). */
+function diffFileSigs(recorded, live) {
+    return summarizeFileSigDiff(diffFileSigsPaths(recorded, live));
+}
+/** Finding 1 (G-4, path-accurate): partition a manifest diff into the shared-root bucket vs the
+ *  skill-private bucket, EXACTLY mirroring `scopedAccept`/`sharedOnlyAccept` in skill-hash.ts so attribution
+ *  matches the hash boundary. A path under `skills/<name>/` is skill-private when `<name>` is in scope; with
+ *  agent-scoping ON, a skill-named `agents/<name>.md` is also skill-private. Everything else is shared. This
+ *  lets `computeStaleness` emit a `skill` finding AND a `shared-root` finding when BOTH buckets drift, so a
+ *  co-occurring shared change can no longer mask the skill's own drift (the original bug). */
+function partitionDriftBuckets(diff, scope, scopeAgents) {
+    const shared = { added: [], removed: [], changed: [] };
+    const skill = { added: [], removed: [], changed: [] };
+    const isSkillPrivate = (relPath) => {
+        const parts = relPath.split("/");
+        if (parts[0] === "skills" && parts.length >= 2)
+            return scope.has(parts[1]);
+        if (scopeAgents) {
+            const an = agentSkillName(parts);
+            if (an !== null && scope.has(an))
+                return true;
+        }
+        return false;
+    };
+    for (const cat of ["added", "removed", "changed"])
+        for (const p of diff[cat])
+            (isSkillPrivate(p) ? skill : shared)[cat].push(p);
+    return { shared, skill };
+}
+/** The SINGLE staleness diagnosis (unifies what used to be two divergent copies: `checkStaleness` and the
+ *  inline block in `replayCassette`). Recompute the fingerprint and report drift as class-tagged findings;
+ *  each CALLER applies its own gate-vs-warn policy:
+ *   - `verify-cassettes` / the re-record work-list go through the `checkStaleness` string adapter and treat
+ *     ANY finding as "stale ⇒ re-record" (so `unverifiable-*` stays a hard fail there — can't verify ⇒ not
+ *     green). The adapter MUST be class-blind (forward every finding) or that gate false-greens.
+ *   - `replayCassette` consumes the findings directly: warn by default, `--strict` fails on all,
+ *     `--fail-on-skill-drift` fails only on `SKILL_DRIFT_CLASSES`.
+ *  No fingerprint → nothing to check (legacy cassette). */
+export function computeStaleness(cassette, cassetteDir) {
     const fp = cassette.fingerprint;
     if (!fp)
         return [];
-    const msgs = [];
+    const findings = [];
     let liveBaseline;
     try {
         liveBaseline = loadBaseline("latest").appVersion;
@@ -471,54 +508,114 @@ export function checkStaleness(cassette, cassetteDir) {
     catch {
         /* baseline not loadable */
     }
-    // Gate mode: can't verify ⇒ not green. The cassette carries a baseline-of-record but we can't load the
-    // current one to compare — a fail, not a silent skip (baselines ship with the package, so this is rare).
+    // The cassette carries a baseline-of-record but we can't load the current one to compare. Surfaced as
+    // `unverifiable-baseline` (env/platform, not skill drift): a non-failing warning on the default replay gate,
+    // but a hard fail for `verify-cassettes`/the work-list via the class-blind string adapter (can't verify ⇒
+    // not green). baselines ship with the package, so this is rare.
     if (liveBaseline === undefined)
-        msgs.push("cannot load the latest baseline to verify staleness — run `cowork-harness sync` or ship baselines/ (can't verify ⇒ not green)");
+        findings.push({
+            class: "unverifiable-baseline",
+            message: "cannot load the latest baseline to verify staleness — run `cowork-harness sync` or ship baselines/ (env/platform, not skill drift; can't verify ⇒ not green)",
+        });
     else if (liveBaseline !== fp.baseline)
-        msgs.push(`baseline moved ${fp.baseline} → ${liveBaseline} since record — re-record`);
+        findings.push({ class: "baseline", message: `baseline moved ${fp.baseline} → ${liveBaseline} since record — re-record` });
     if (fp.skillHash) {
         const live = buildFingerprint(cassette.scenario.session, fp.baseline, cassetteDir, cassette.scenario.skills);
         const recMode = fp.mode ?? "raw";
         const liveMode = live.mode ?? "raw";
         if (live.skillHash === undefined)
-            msgs.push("skill dirs not resolvable from the cassette location — cannot verify staleness (gate fails: can't verify ⇒ not green)");
+            findings.push({
+                class: "unverifiable-skill",
+                message: "skill dirs not resolvable from the cassette location — cannot verify skill staleness (can't verify ⇒ not green)",
+            });
         else if (recMode !== liveMode)
-            // Phase C: a hash from a different boundary mode is not comparable — don't emit a misleading content diff.
-            msgs.push(`recorded in '${recMode}' file-set mode, verifying in '${liveMode}' (COWORK_HARNESS_GITSET) — re-record under the same mode`);
+            // Phase C: a hash from a different boundary mode is not comparable — re-record, don't emit a misleading
+            // content diff. Classed `format` (not skill drift): a mode flip is an env/config mismatch, not source drift.
+            findings.push({
+                class: "format",
+                message: `recorded in '${recMode}' file-set mode, verifying in '${liveMode}' (COWORK_HARNESS_GITSET) — re-record under the same mode`,
+            });
         else if ((fp.agentScope ?? "off") !== (live.agentScope ?? "off"))
             // Agent-scoping flip (COWORK_HARNESS_AGENT_SCOPE): the scoped hash covers a different file set, so it's
             // not comparable — re-record under the same setting (mirrors the GITSET mode flip above).
-            msgs.push(`recorded with agent-scope '${fp.agentScope ?? "off"}', verifying with '${live.agentScope ?? "off"}' (COWORK_HARNESS_AGENT_SCOPE) — re-record under the same setting`);
+            findings.push({
+                class: "format",
+                message: `recorded with agent-scope '${fp.agentScope ?? "off"}', verifying with '${live.agentScope ?? "off"}' (COWORK_HARNESS_AGENT_SCOPE) — re-record under the same setting`,
+            });
         else if (live.skillHash !== fp.skillHash) {
-            debugSkillHashMismatch(cassette, cassetteDir, fp, live); // H9: surface WHICH files drifted (or a hint to enable it)
+            debugSkillHashMismatch(cassette, cassetteDir ?? "", fp, live); // H9 (D2): surface WHICH files drifted
             const recordedVersion = cassette.cassetteVersion ?? 0;
-            // v5: name the EXACT changed/added/removed file(s) from the per-file manifest, APPENDED to the
-            // bucket/generic message so the G-4 shared-vs-scoped semantic is preserved AND the file is named.
-            const summary = fp.fileSigs && live.fileSigs ? diffFileSigs(fp.fileSigs, live.fileSigs) : null;
-            const detail = summary ? ` [${summary}]` : "";
             if (recordedVersion < CASSETTE_VERSION) {
-                msgs.push(`recorded under an older hash format (v${recordedVersion} → v${CASSETTE_VERSION}) — re-record once after upgrading`);
+                findings.push({
+                    class: "format",
+                    message: `recorded under an older hash format (v${recordedVersion} → v${CASSETTE_VERSION}) — re-record once after upgrading`,
+                });
             }
             else if (fp.sharedHash !== undefined && live.sharedHash !== undefined) {
-                // G-4: bucket-level diagnosis — which component of the scoped hash changed?
-                const scope = fp.skillScope.map((s) => `skills/${s}`).join(", ");
-                if (live.sharedHash !== fp.sharedHash) {
-                    msgs.push(`shared root changed since record (scope: ${scope})${detail} — re-record`);
+                // G-4 + Finding 1: attribute drift to the shared and/or skill bucket(s). D6: `skillScope` is always
+                // non-empty when `sharedHash` is set (single assignment site under the same guard in buildFingerprint);
+                // the `?? []` is a defensive guard only — the on-disk cassette shape is not schema-validated.
+                const scopeArr = fp.skillScope ?? [];
+                const scopeLabel = scopeArr.map((s) => `skills/${s}`).join(", ") || "skill";
+                const scopeSet = new Set(scopeArr);
+                const scopeAgents = (live.agentScope ?? "off") === "skill";
+                if (fp.fileSigs && live.fileSigs) {
+                    // Path-accurate: emit a finding per bucket that ACTUALLY changed, so a co-occurring shared change can
+                    // never mask the skill's own drift (the original bug) and vice-versa.
+                    const { shared, skill } = partitionDriftBuckets(diffFileSigsPaths(fp.fileSigs, live.fileSigs), scopeSet, scopeAgents);
+                    const sharedSummary = summarizeFileSigDiff(shared);
+                    const skillSummary = summarizeFileSigDiff(skill);
+                    if (sharedSummary)
+                        findings.push({
+                            class: "shared-root",
+                            message: `shared root changed since record (scope: ${scopeLabel}) [${sharedSummary}] — re-record`,
+                        });
+                    if (skillSummary)
+                        findings.push({ class: "skill", message: `${scopeLabel} changed since record [${skillSummary}] — re-record` });
+                    if (!sharedSummary && !skillSummary) {
+                        // Hashes differ but the per-file manifest shows no path change (structure-only: an empty dir or a
+                        // symlink re-point). Fall back to the hash buckets; emit BOTH classes if the shared hash moved so
+                        // neither bucket is masked.
+                        if (live.sharedHash !== fp.sharedHash)
+                            findings.push({ class: "shared-root", message: `shared root changed since record (scope: ${scopeLabel}) — re-record` });
+                        findings.push({ class: "skill", message: `${scopeLabel} changed since record — re-record` });
+                    }
                 }
                 else {
-                    msgs.push(`${scope} changed since record${detail} — re-record`);
+                    // Pre-detail cassette (no per-file manifest, e.g. > MANIFEST_MAX_FILES): can't isolate the bucket.
+                    // Emit BOTH classes when the shared hash moved so the skill's own drift is never masked (over-warns,
+                    // but the gate is already red and both classes are in the fail set).
+                    if (live.sharedHash !== fp.sharedHash) {
+                        findings.push({ class: "shared-root", message: `shared root changed since record (scope: ${scopeLabel}) — re-record` });
+                        findings.push({
+                            class: "skill",
+                            message: `${scopeLabel} may also have changed since record (no per-file manifest to isolate) — re-record`,
+                        });
+                    }
+                    else {
+                        findings.push({ class: "skill", message: `${scopeLabel} changed since record — re-record` });
+                    }
                 }
             }
-            else if (summary) {
-                msgs.push(`skill files changed since record — ${summary} — re-record`);
-            }
             else {
-                msgs.push("local skill/plugin dir contents changed since record — re-record");
+                // Non-scoped (whole-tree) cassette: name the changed files when the per-file manifest is present (D5),
+                // else the generic fallback.
+                const summary = fp.fileSigs && live.fileSigs ? diffFileSigs(fp.fileSigs, live.fileSigs) : null;
+                if (summary)
+                    findings.push({ class: "skill", message: `skill files changed since record — ${summary} — re-record` });
+                else
+                    findings.push({ class: "skill", message: "local skill/plugin dir contents changed since record — re-record" });
             }
         }
     }
-    return msgs;
+    return findings;
+}
+/** B3 staleness GATE adapter for the string consumers (`verify-cassettes`, the re-record work-list). Returns
+ *  the unified findings as plain reason strings. MUST stay class-blind (forward EVERY finding) so an
+ *  `unverifiable-baseline` / `unverifiable-skill` still reds those gates — filtering a class here would
+ *  false-green `verify-cassettes` on a cassette it can't verify. */
+export function checkStaleness(cassette, cassetteDir) {
+    return computeStaleness(cassette, cassetteDir).map((f) => f.message);
 }
 /** A minimal RunRecord for a truncated-cassette replay — empty collections so downstream evaluate()/the
  *  mismatch loops don't NPE; result:"error" because the cassette could not be driven to completion. */
@@ -1520,7 +1617,7 @@ export async function cmdReplay(args) {
     try {
         p = parseArgs(args, {
             // --quiet/--verbose accepted for flag consistency but currently no-op in replay (renderer plan is fixed).
-            booleans: ["--strict", "--quiet", "--verbose"],
+            booleans: ["--strict", "--fail-on-skill-drift", "--quiet", "--verbose"],
             values: ["--output-format"],
             enums: { "--output-format": ["text", "json"] },
             aliases: { "-q": "--quiet", "-V": "--verbose" },
@@ -1532,7 +1629,7 @@ export async function cmdReplay(args) {
     }
     const target = p.positionals[0];
     if (!target) {
-        log("usage: replay <file.cassette.json | dir/> [--strict] [--output-format text|json]");
+        log("usage: replay <file.cassette.json | dir/> [--strict] [--fail-on-skill-drift] [--output-format text|json]");
         return process.exit(2);
     }
     if (p.positionals.length > 1) {
@@ -1540,7 +1637,10 @@ export async function cmdReplay(args) {
         return process.exit(2);
     }
     const json = p.options["--output-format"] === "json";
-    const strict = p.flags["--strict"] ?? false; // #1b: escalate staleness warnings to failures (release gate)
+    const strict = p.flags["--strict"] ?? false; // #1b: escalate ALL staleness findings to failures (release gate)
+    const failOnSkillDrift = p.flags["--fail-on-skill-drift"] ?? false; // narrower gate: only skill-source drift fails
+    if (strict && failOnSkillDrift)
+        warn("::notice:: [replay] --strict and --fail-on-skill-drift both passed — --strict is the superset (fails on every class), so --fail-on-skill-drift is redundant here\n");
     const resolved = resolveInputs(target, ".cassette.json");
     if ("error" in resolved) {
         log(`replay: ${resolved.error}`);
@@ -1564,7 +1664,7 @@ export async function cmdReplay(args) {
         // fine" (false-green by abort). Wrap per-file: turn an unexpected throw into a tallied error result.
         let result;
         try {
-            result = await replayCassette(rc.cassette, renderer ? [renderer] : [], { strict, cassetteDir: dirname(f) });
+            result = await replayCassette(rc.cassette, renderer ? [renderer] : [], { strict, failOnSkillDrift, cassetteDir: dirname(f) });
         }
         catch (e) {
             log(`replay: ${f}: ${e?.message ?? String(e)}`);
@@ -1860,7 +1960,10 @@ export function cmdRehash(args) {
 }
 /** Replay a cassette through Run and re-evaluate the content assertions. With a `cassette.artifacts`
  *  manifest (#1), filesystem assertions (file_exists/user_visible_artifact/artifact_json) ALSO run, against
- *  the materialized snapshot. `opts.strict` (#1b) escalates staleness warnings to failing assertions. */
+ *  the materialized snapshot. `opts.strict` (#1b) escalates ALL staleness findings to failing assertions;
+ *  `opts.failOnSkillDrift` escalates only the skill-source classes (`skill`/`shared-root`/`unverifiable-skill`),
+ *  leaving baseline drift a non-failing warning. Either way the findings are always surfaced in
+ *  `RunResult.staleness` for JSON consumers. */
 export async function replayCassette(cassette, hooks = [], opts = {}) {
     // Cassette format version: ABSENT = legacy (0); a FUTURE version means this harness may misread fields
     // it doesn't know about. in strict mode this is a hard failure (future semantics may not be
@@ -1884,59 +1987,16 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
     // unchanged (frozen-structure limit). The skill-hash recompute needs the local skill dirs to be resolvable
     // from the cassette's session path; when they aren't (a moved/committed cassette), we say so rather than
     // silently skipping.
-    const staleness = [];
-    if (cassette.fingerprint) {
-        const fp = cassette.fingerprint;
-        let liveBaseline;
-        let baselineLoadFailed = false;
-        try {
-            liveBaseline = loadBaseline("latest").appVersion;
-        }
-        catch {
-            baselineLoadFailed = true;
-            if (opts.strict) {
-                staleness.push("baseline could not be loaded — cannot verify staleness (--strict: treating as stale)");
-            }
-            else {
-                warn("cowork-harness: strict staleness check skipped — baseline could not be loaded\n");
-            }
-        }
-        if (!baselineLoadFailed && liveBaseline && liveBaseline !== fp.baseline)
-            staleness.push(`baseline moved ${fp.baseline} → ${liveBaseline} since record — re-record before trusting this replay`);
-        if (fp.skillHash) {
-            const live = buildFingerprint(cassette.scenario.session, fp.baseline, opts.cassetteDir, cassette.scenario.skills);
-            if (live.skillHash === undefined) {
-                // under --strict, unresolvable skill dirs are a failure (can't verify ⇒ not green).
-                // Without --strict, warn but continue (the baseline check still applies).
-                if (opts.strict) {
-                    staleness.push("skill dirs not resolvable from this cassette location — cannot verify skill staleness (--strict: treating as stale)");
-                }
-                else {
-                    warn("::warning:: [replay] skill fingerprint not re-checkable (local skill dirs not resolvable from this cassette location) — baseline check still applies\n");
-                }
-            }
-            else if (live.skillHash !== fp.skillHash) {
-                const recordedVersion = cassette.cassetteVersion ?? 0;
-                if (recordedVersion < CASSETTE_VERSION) {
-                    staleness.push(`recorded under an older hash format (v${recordedVersion} → v${CASSETTE_VERSION}) — re-record once after upgrading`);
-                }
-                else if (fp.sharedHash !== undefined && live.sharedHash !== undefined) {
-                    const scope = fp.skillScope?.length ? fp.skillScope.map((s) => `skills/${s}`).join(", ") : "skill";
-                    if (live.sharedHash !== fp.sharedHash) {
-                        staleness.push(`shared root changed since record (scope: ${scope}) — re-record before trusting this replay`);
-                    }
-                    else {
-                        staleness.push(`${scope} changed since record — re-record before trusting this replay`);
-                    }
-                }
-                else {
-                    staleness.push("local skill/plugin dir contents changed since record — re-record before trusting this replay");
-                }
-            }
-        }
-        for (const s of staleness)
-            warn(`::warning:: [replay] cassette stale: ${s}\n`);
-    }
+    // Findings are surfaced UNCONDITIONALLY (class-tagged) in JSON (RunResult.staleness) even on the default
+    // gate — a token-free consumer can distinguish "verified clean" from "couldn't verify" (the `unverifiable-*`
+    // classes) WITHOUT the verdict changing. The `--strict` / `--fail-on-skill-drift` gates below are the ONLY
+    // place a finding becomes a failing assertion. The single `warn()` loop is the lone stderr emitter — no
+    // per-branch `warn()`, so a non-strict run never double-warns one cause. Uses the SHARED `computeStaleness`
+    // (no longer a forked copy), so it inherits the per-file detail, the `debugSkillHashMismatch` hook, the
+    // GITSET/agent-scope flip buckets, and the both-buckets attribution fix for free.
+    const staleness = computeStaleness(cassette, opts.cassetteDir);
+    for (const s of staleness)
+        warn(`::warning:: [replay] cassette stale: ${s.message}\n`);
     // §2.5 backward compat: warn loudly when controlOut is absent so the user knows question/gate
     // assertions are being EXCLUDED (not vacuously evaluated) from this run.
     if (!session.hasControlOut) {
@@ -2085,10 +2145,17 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
         toolResultTexts: rec.toolResults.map((r) => r.assertText ?? r.text),
         truncatedPaths: replayTruncatedPaths,
     });
-    // #1b: under --strict, a staleness mismatch is a failing assertion (non-zero exit), not just a warning.
+    // #1b: under --strict, EVERY staleness finding becomes a failing assertion (non-zero exit), not just a
+    // warning. --fail-on-skill-drift is the narrower gate: only the skill-source classes fail (incl.
+    // `unverifiable-skill` — can't verify skill staleness ⇒ not green), while baseline / format / env-level
+    // findings stay non-failing. `else if` makes --strict the superset when both are passed.
+    const SKILL_DRIFT_CLASSES = new Set(["skill", "shared-root", "unverifiable-skill"]);
     if (opts.strict)
         for (const s of staleness)
-            assertions.push({ assertion: {}, pass: false, message: `cassette stale (--strict): ${s}` });
+            assertions.push({ assertion: {}, pass: false, message: `cassette stale (--strict): ${s.message}` });
+    else if (opts.failOnSkillDrift)
+        for (const s of staleness.filter((s) => SKILL_DRIFT_CLASSES.has(s.class)))
+            assertions.push({ assertion: {}, pass: false, message: `skill-source drift (--fail-on-skill-drift): ${s.message}` });
     // future cassette version — hard failure under --strict (forward semantics may not be
     // correctly interpreted here, so a green replay would be a false-green).
     if (futureVersionMsg && opts.strict)
@@ -2175,6 +2242,10 @@ export async function replayCassette(cassette, hooks = [], opts = {}) {
         subagents: rec.subagents,
         unanswered: rec.unanswered,
         outDir: "(replay)",
+        // Class-tagged staleness + skip counts, surfaced to JSON callers (the gate decision already happened
+        // above via failing assertions; these fields are pure data so a green stays green by default).
+        staleness: staleness.length ? staleness : undefined,
+        skippedAssertions: { full: fullSkipCount, partial: partialSkipCount },
         // A cassette freezes the answer path: the replay itself is deterministic regardless of how the
         // original run was answered. Always explicit (never undefined) so renderer.ts:146 treats it
         // correctly — undefined would silently render as "deterministic" (#47 C1 [review-2]).

package/dist/run/skill-hash.js

@@ -211,8 +211,9 @@ function agentScopeEnabled() {
 }
 /** Map the first path segment under `agents/` to the skill name it would belong to: a FILE strips its
  *  extension (`agents/cap-table.md` → `cap-table`), a DIR is used as-is (`agents/cap-table/x.md` → `cap-table`).
- *  Returns null for non-`agents/` paths or a bare `agents` marker. */
-function agentSkillName(parts) {
+ *  Returns null for non-`agents/` paths or a bare `agents` marker. Exported so staleness attribution can
+ *  classify a changed `agents/<name>.md` exactly as the hash walk does (agent-scoped → skill-private). */
+export function agentSkillName(parts) {
     if (parts[0] !== "agents" || parts.length < 2)
         return null;
     return parts.length === 2 ? parts[1].replace(/\.[^.]+$/, "") : parts[1];

package/docs/cassette.md

@@ -211,8 +211,9 @@ with `record --max-artifact-bytes <n>` (or `COWORK_HARNESS_MAX_ARTIFACT_BYTES`)
 stays replay-checkable, and `record` fails fast if an `artifact_json` targets an artifact it had to truncate (that
 would pass at record but fail at replay). Without a manifest (older
 cassettes), these are skipped. A green replay re-confirms *record-time* artifacts, **not** that the current
-skill still produces them — `replay --strict` fails the run when the `fingerprint` shows the skill/baseline
-drifted.
+skill still produces them — `replay --strict` fails the run when the `fingerprint` shows ANY skill/baseline
+drift, or `replay --fail-on-skill-drift` fails only on skill-source drift (leaving baseline drift a warning).
+Either way, every replay result also reports the drift in `staleness[]` (class-tagged) for a JSON gate to read.
 
 ### Still skipped on replay (no filesystem/network in a cassette)
 
@@ -320,9 +321,12 @@ possible.
 - **`recorded under an older hash format (vN → vM)`** — format upgrade; re-record once and the message
   goes away. (Cassettes recorded before format **v6** all need one re-record — see the boundary note below.)
 - **`skill files changed since record — N changed (path, …)`** — the **exact** changed/added/removed file(s),
-  from the per-file manifest (`fileSigs`). For a scoped cassette it is prefixed with the bucket diagnosis
-  (`shared root changed (scope: skills/x) [N changed (…)]`) so you still see whether the change was in your
-  scoped skill or a shared dependency.
+  from the per-file manifest (`fileSigs`). For a scoped cassette the drift is attributed **per bucket** by the
+  actual changed paths: a `shared root changed (scope: skills/x) [N changed (…)]` message for shared-dependency
+  changes and a `skills/x changed since record [N changed (…)]` message for the scoped skill's own files. When
+  **both** buckets change you get **both** messages — a co-occurring shared change no longer masks the skill's
+  own drift. (With `COWORK_HARNESS_AGENT_SCOPE=skill`, a changed `agents/<x>.md` is attributed to skill `x`,
+  matching the hash boundary.)
 - **`recorded in '<mode>' file-set mode, verifying in '<mode>'`** — the staleness boundary differs between
   record and verify (e.g. recorded in a git work tree but verified from a non-repo copy); the hashes are not
   comparable, so re-record under the same mode.

package/docs/scenario.md

@@ -340,7 +340,9 @@ small JSON bodies) into the cassette, and `replay` materializes that snapshot to
 `artifact_json` needs the JSON body inlined (small files); a hash-only (oversized) entry still satisfies
 `file_exists` but not `artifact_json`. Without a manifest (older cassettes), they are **skipped** (loud).
 A green `replay` re-confirms *record-time* artifacts, **not** that the current skill still produces them —
-that needs a live `run` (the cassette's `--strict` staleness fingerprint warns when the skill/baseline drifted).
+that needs a live `run` (the cassette's staleness fingerprint warns when the skill/baseline drifted; `replay
+--strict` fails on any drift, `--fail-on-skill-drift` on skill-source drift only, and every result reports it
+in `staleness[]` for a JSON gate).
 
 **Egress + other filesystem** assertions (`no_delete_in_outputs`, `self_heal_ran`,
 `transcript_no_host_path`, `egress_*`/`expect_denied`) are still **skipped** on `replay` — they only run on

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "cowork-harness",
-  "version": "0.13.0",
+  "version": "0.15.0",
   "description": "Scriptable, CI-friendly harness for Claude Cowork's runtime contract for testing skills across scenarios — same agent, mounts, egress allowlist, permission protocol, and sandbox limitations.",
   "license": "MIT",
   "type": "module",

package/scripts/gen-schema.ts

@@ -19,14 +19,21 @@ export const SCHEMA_DIR = join(REPO_ROOT, "schema");
  *  is. Writer + the drift-guard test both reference this one constant. */
 export const ASSERTION_KEYS_PATH = join(REPO_ROOT, ".claude/skills/cowork-harness/scripts/assertion-keys.json");
 
-/** The authoritative assertion-key list, derived from the Zod `Assertion` schema (the same source
- *  `assert --list` reads). Generating it keeps `scenario.py`'s unknown-key check from drifting. */
+/** The authoritative key lists `scenario.py` reads — derived from the Zod schemas (the same source
+ *  `assert --list` reads). Generating them keeps the linter's unknown-key checks from drifting: `keys` is
+ *  the `assert:` catalog, `topLevelKeys` the scenario top-level catalog (an earlier hand-maintained copy
+ *  drifted and false-flagged the valid `requires_capabilities`). `profile`/`assertions` are NOT here — they
+ *  are deprecated aliases handled by scenario.py's special-cases (the Zod `Scenario` preprocess remaps
+ *  `profile`; `assertions` is a hard error), so they are intentionally absent from the schema shape. */
 export function buildAssertionKeys(): string {
   return (
     JSON.stringify(
       {
-        $comment: "GENERATED from the Zod Assertion schema (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
+        $comment: "GENERATED from the Zod schemas (src/types.ts) by scripts/gen-schema.ts — do not edit; run `npm run schema`.",
         keys: Object.keys(Assertion.shape).sort(),
+        // Every valid top-level scenario key, from the ScenarioObject strictObject shape (NOT the `Scenario`
+        // preprocess wrapper). scenario.py keeps an embedded fallback parity-tested against this.
+        topLevelKeys: Object.keys(ScenarioObject.shape).sort(),
         // The verdict-modifier subset (no-op assertions that suppress a default-fail). scenario.py keeps a
         // hardcoded copy parity-tested against this; see VERDICT_MODIFIER_KEYS in src/types.ts.
         verdictModifierKeys: [...VERDICT_MODIFIER_KEYS].sort(),