cowork-harness 2.2.0 → 2.3.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude/skills/cowork-harness/SKILL.md +31 -12
- package/.claude/skills/cowork-harness/references/ci-recipe.md +15 -13
- package/.claude/skills/cowork-harness/references/critique.md +1 -1
- package/.claude/skills/cowork-harness/references/fidelity-and-answers.md +1 -1
- package/.claude/skills/cowork-harness/references/scenario-schema.md +14 -12
- package/.claude/skills/cowork-harness/references/task-recipes.md +2 -2
- package/.claude/skills/cowork-harness/scripts/assertion-keys.json +1 -0
- package/.claude/skills/cowork-harness/scripts/scenario.py +8 -1
- package/CHANGELOG.md +236 -0
- package/DESIGN.md +3 -3
- package/README.md +41 -11
- package/baselines/desktop-1.37937.1.json +816 -0
- package/baselines/prompts/cowork-system-prompt-fingerprints.json +27 -0
- package/dist/assert.js +77 -0
- package/dist/cli.js +15 -4
- package/dist/run/cassette.js +255 -77
- package/dist/run/execute.js +59 -0
- package/dist/run/trace-view.js +39 -1
- package/dist/sync/cowork-sync.js +207 -8
- package/dist/types.js +39 -9
- package/docs/README.md +3 -0
- package/docs/cassette.md +36 -11
- package/docs/debugging.md +25 -0
- package/docs/fidelity-gaps.md +5 -1
- package/docs/maintenance.md +15 -1
- package/docs/scenario.md +13 -10
- package/examples/replays/README.md +1 -1
- package/examples/replays/example-multiselect-gate.cassette.json +41 -46
- package/examples/replays/example-pdf-skill.cassette.json +90 -90
- package/examples/replays/hostloop-computer-links.cassette.json +62 -68
- package/llms.txt +3 -1
- package/package.json +1 -1
- package/schema/scenario.schema.json +28 -9
package/dist/run/trace-view.js
CHANGED
|
@@ -362,6 +362,13 @@ export function buildGateTrace(file) {
|
|
|
362
362
|
rows.push({
|
|
363
363
|
question: req.questions.map((q) => q.question ?? q.header ?? "").join(" / "),
|
|
364
364
|
requestId: req.id,
|
|
365
|
+
// The ask-time options ride along unmodified — no truncation here, only at render time, so a JSON
|
|
366
|
+
// consumer (`--output-format json`) gets the full offered text and the text view stays readable.
|
|
367
|
+
subQuestions: req.questions.map((q) => ({
|
|
368
|
+
question: q.question ?? q.header ?? "",
|
|
369
|
+
options: q.options ?? [],
|
|
370
|
+
...(q.multiSelect !== undefined ? { multiSelect: q.multiSelect } : {}),
|
|
371
|
+
})),
|
|
365
372
|
subQuestionCount: req.questions.length,
|
|
366
373
|
injectedAnswer: answers.get(req.id),
|
|
367
374
|
delivered: tr ? (tr.isError ? "error" : "ok") : "unobserved",
|
|
@@ -423,6 +430,36 @@ export function buildGateTrace(file) {
|
|
|
423
430
|
}
|
|
424
431
|
return rows;
|
|
425
432
|
}
|
|
433
|
+
/** Per-option description cap in the TEXT view only. Long enough for the producer-authored sentence a
|
|
434
|
+
* founder is deciding on; short enough that a gate offering four options stays one screen. The JSON
|
|
435
|
+
* envelope carries the untruncated text, so nothing is lost — only wrapped. */
|
|
436
|
+
const GATE_OPTION_DESC_CAP = 240;
|
|
437
|
+
/** The offered options of one gate, indented under it. Rendered for EVERY gate that recorded options —
|
|
438
|
+
* not only bundled ones — because the whole point is that this text was previously invisible. A gate
|
|
439
|
+
* whose payload carried no options at all renders nothing extra rather than an empty `offered:` header. */
|
|
440
|
+
function formatGateOptions(subs) {
|
|
441
|
+
if (!subs.some((s) => s.options.length))
|
|
442
|
+
return "";
|
|
443
|
+
const multi = subs.length > 1;
|
|
444
|
+
const out = [];
|
|
445
|
+
for (const s of subs) {
|
|
446
|
+
if (!s.options.length)
|
|
447
|
+
continue;
|
|
448
|
+
// The sub-question label is repeated only on a bundled gate; on a single-sub-question gate it is
|
|
449
|
+
// already the row's headline and repeating it is noise.
|
|
450
|
+
if (multi)
|
|
451
|
+
out.push(` ${s.question}${s.multiSelect ? " (multi-select)" : ""}:`);
|
|
452
|
+
else if (s.multiSelect)
|
|
453
|
+
out.push(` (multi-select)`);
|
|
454
|
+
for (const o of s.options) {
|
|
455
|
+
const desc = o.description === undefined || o.description === ""
|
|
456
|
+
? ""
|
|
457
|
+
: ` — ${o.description.length > GATE_OPTION_DESC_CAP ? o.description.slice(0, GATE_OPTION_DESC_CAP) + "…" : o.description}`;
|
|
458
|
+
out.push(` ${multi ? " " : ""}• ${o.label}${desc}`);
|
|
459
|
+
}
|
|
460
|
+
}
|
|
461
|
+
return out.length ? `\n offered:\n${out.join("\n")}` : "";
|
|
462
|
+
}
|
|
426
463
|
export function formatGateTrace(rows) {
|
|
427
464
|
if (!rows.length)
|
|
428
465
|
return "(no AskUserQuestion gates in this run)";
|
|
@@ -432,7 +469,8 @@ export function formatGateTrace(rows) {
|
|
|
432
469
|
// sub-question count is shown only when the gate bundled more than one — reconciles this row
|
|
433
470
|
// against questions_count_max, which counts sub-questions, not gates.
|
|
434
471
|
const subCount = r.subQuestionCount > 1 ? ` (${r.subQuestionCount} sub-questions)` : "";
|
|
435
|
-
|
|
472
|
+
const offered = formatGateOptions(r.subQuestions ?? []);
|
|
473
|
+
return `${mark[r.delivered]} gate "${r.question}"${subCount}${offered}\n answered: ${r.injectedAnswer ?? "(none)"}\n delivered: ${r.delivered}${r.error ? ` — ${r.error}` : ""}${prov}`;
|
|
436
474
|
});
|
|
437
475
|
const totalSubQuestions = rows.reduce((sum, r) => sum + r.subQuestionCount, 0);
|
|
438
476
|
lines.push(`\n${rows.length} gate(s), ${totalSubQuestions} sub-question(s) total — questions_count_max counts sub-questions (assert with \`questions_count_max: ${totalSubQuestions}\`)`);
|
package/dist/sync/cowork-sync.js
CHANGED
|
@@ -1179,10 +1179,23 @@ export function checkCodeTripwires(bundle) {
|
|
|
1179
1179
|
flags.push(`NOTE: code tripwire: io.modelcontextprotocol/skills capability appears ${skillsExt}x (was 1) — the MCP-skills capability surface changed; re-verify whether MCP servers can now contribute skills`);
|
|
1180
1180
|
return flags;
|
|
1181
1181
|
}
|
|
1182
|
+
/** Sites building the delete-deny resolver on the newest baseline (Desktop 1.37937.1): the VM-loop
|
|
1183
|
+
* mount-set builder and host-loop `computeBashMounts`. A FLOOR, not an equality — see its use site. */
|
|
1184
|
+
const MOUNT_DELETE_DENY_MIN_SITES = 2;
|
|
1182
1185
|
export function checkMountModeFacts(bundle) {
|
|
1183
1186
|
const flags = [];
|
|
1184
|
-
|
|
1185
|
-
|
|
1187
|
+
// The delete-deny resolver. A bare `.test()` was the same single-anchor hole the per-mount checks below
|
|
1188
|
+
// just closed: the resolver is now built on BOTH lanes (1 site in Desktop 1.34493.1, 2 from 1.37937.0),
|
|
1189
|
+
// so once one lane has it, `.test()` cannot see the other lane losing it. Guard a FLOOR rather than an
|
|
1190
|
+
// exact count — a lane gaining the resolver is benign and must not red a sync, a lane losing it is the
|
|
1191
|
+
// containment change. The floor is the count observed on the newest baseline; raise it deliberately
|
|
1192
|
+
// when a release adds a lane. Running `sync` against a Desktop OLDER than the floor's release will
|
|
1193
|
+
// flag, which is correct: that install does not build what the pinned baseline describes.
|
|
1194
|
+
const denySites = (bundle.match(/\?"rwd":"rw"/g) ?? []).length;
|
|
1195
|
+
if (denySites < MOUNT_DELETE_DENY_MIN_SITES)
|
|
1196
|
+
flags.push(`mountLayout: the delete-deny resolver (IX \`…?"rwd":"rw"\`) is built at ${denySites} site(s), below the pinned floor of ` +
|
|
1197
|
+
`${MOUNT_DELETE_DENY_MIN_SITES} — an execution lane lost delete-deny resolution, so outputs/projects default mode may have ` +
|
|
1198
|
+
"changed on that lane; re-derive mountLayout.mounts[].mode per lane (see baselines $comment_modes)");
|
|
1186
1199
|
// Every mount whose mode is HARDCODED at the mount-set builder, rather than resolved through
|
|
1187
1200
|
// NOTE the lane difference, because "spawn-time" is wrong for half of it: the VM-loop builder runs
|
|
1188
1201
|
// once at spawn, but the host-loop one is wired as `computeBashMounts` and RECOMPUTES PER BASH CALL
|
|
@@ -1192,18 +1205,30 @@ export function checkMountModeFacts(bundle) {
|
|
|
1192
1205
|
// outputs and each connected folder go through the resolver (`rw`, or `rwd` once approved) while these
|
|
1193
1206
|
// are pinned `"ro"`. Worth pinning individually because a mount silently moving from `ro` to a
|
|
1194
1207
|
// writable mode is a containment change we would otherwise model wrongly with nothing failing.
|
|
1208
|
+
//
|
|
1209
|
+
// EVERY SITE must carry the mode, not merely one of them. Each of these names is built at TWO sites —
|
|
1210
|
+
// the VM-loop mount-set builder and host-loop `computeBashMounts` — and an `re.test(bundle)` is
|
|
1211
|
+
// satisfied by either, so a one-lane `ro`→`rw` flip (a real containment change on exactly one
|
|
1212
|
+
// execution tier) passed green. Comparing the site count to the with-mode count closes that: the
|
|
1213
|
+
// `site` pattern locates the mount irrespective of its mode, `ro` requires the mode too.
|
|
1195
1214
|
const hardcodedRo = [
|
|
1196
|
-
["uploads", /\("uploads"\)\][^}]{0,90}mode:\s*"ro"/],
|
|
1197
|
-
[".claude/skills", /\("\.claude\/skills"\)\][^}]{0,120}mode:\s*"ro"/],
|
|
1198
|
-
[".claude/projects", /\("\.claude\/projects"\)\][^}]{0,120}mode:\s*"ro"/],
|
|
1215
|
+
["uploads", /\("uploads"\)\]/g, /\("uploads"\)\][^}]{0,90}mode:\s*"ro"/g],
|
|
1216
|
+
[".claude/skills", /\("\.claude\/skills"\)\]/g, /\("\.claude\/skills"\)\][^}]{0,120}mode:\s*"ro"/g],
|
|
1217
|
+
[".claude/projects", /\("\.claude\/projects"\)\]/g, /\("\.claude\/projects"\)\][^}]{0,120}mode:\s*"ro"/g],
|
|
1199
1218
|
// Project ATTACHMENTS (`userSelectedProjectUuids`) — one mount per uuid, read-only. This is the fact
|
|
1200
1219
|
// that settles whether a project mount belongs in the delete-denied set: it does not, because it is
|
|
1201
1220
|
// not writable at all.
|
|
1202
|
-
[".projects/<uuid>", /\(`\.projects\/\$\{[^}]+\}`\)\][^}]{0,90}mode:\s*"ro"/],
|
|
1221
|
+
[".projects/<uuid>", /\(`\.projects\/\$\{[^}]+\}`\)\]/g, /\(`\.projects\/\$\{[^}]+\}`\)\][^}]{0,90}mode:\s*"ro"/g],
|
|
1203
1222
|
];
|
|
1204
|
-
for (const [name,
|
|
1205
|
-
|
|
1223
|
+
for (const [name, siteRe, roRe] of hardcodedRo) {
|
|
1224
|
+
const sites = (bundle.match(siteRe) ?? []).length;
|
|
1225
|
+
const ro = (bundle.match(roRe) ?? []).length;
|
|
1226
|
+
if (sites === 0)
|
|
1206
1227
|
flags.push(`mountLayout: the read-only ("ro") mount for ${name} is gone from the asar — its mode may have changed; re-derive mountLayout.mounts[].mode (see baselines $comment_modes)`);
|
|
1228
|
+
else if (ro !== sites)
|
|
1229
|
+
flags.push(`mountLayout: ${name} is built at ${sites} site(s) but only ${ro} carry mode:"ro" — one execution lane's mount became writable; ` +
|
|
1230
|
+
`re-derive mountLayout.mounts[].mode per lane (see baselines $comment_modes)`);
|
|
1231
|
+
}
|
|
1207
1232
|
return flags;
|
|
1208
1233
|
}
|
|
1209
1234
|
/**
|
|
@@ -2108,7 +2133,22 @@ const SPAWN_PIN_KEYS = [
|
|
|
2108
2133
|
// what is new is the Cowork spawn setting it outright.
|
|
2109
2134
|
"CLAUDE_PREVIEW_CLASSIFIER_FLOOR",
|
|
2110
2135
|
"DISABLE_AUTOUPDATER",
|
|
2136
|
+
// Desktop 1.37937.0. MCP_TOOL_TIMEOUT is the first key that is BOTH 1p-pinned and 3p-constructed:
|
|
2137
|
+
// W1 builds it unconditionally (`String(<getter>())` → the `??18e4` default), and the 3p-only branch
|
|
2138
|
+
// in W3 gained a second, settings-conditional site (`...i!==void 0&&{MCP_TOOL_TIMEOUT:String(i)}`,
|
|
2139
|
+
// `i` a chunk-local `let`). It stays PINNED here, on its W1 site. Allowlisting it instead would be a
|
|
2140
|
+
// silent contract loss: resolveInto checks SPAWN_ENV_ALLOWLIST BEFORE this list, so the key would
|
|
2141
|
+
// vanish from the generated env entirely — and it is not in REQUIRED_SPAWN_KEYS, so nothing would
|
|
2142
|
+
// hard-fail. The 3p site is handled per-SITE by the branch rule in applyWindow, not per-key here.
|
|
2111
2143
|
"MCP_TOOL_TIMEOUT",
|
|
2144
|
+
// Desktop 1.37937.0. Both UNCONDITIONAL in W1 (no gate, no session/deployment branch), constructed as
|
|
2145
|
+
// plain string literals between ENABLE_APPEND_SUBAGENT_PROMPT and ENABLE_PROMPT_CACHING_1H — which is
|
|
2146
|
+
// still set, so these are ADDITIVE rather than its replacement. Same call as
|
|
2147
|
+
// CLAUDE_PREVIEW_CLASSIFIER_FLOOR above: unconditional in W1 ⇒ every first-party Cowork session
|
|
2148
|
+
// receives them ⇒ pin, so a later gate or value change is a --diff line, not a silent shift. They read
|
|
2149
|
+
// 0 times in agent 2.1.241 and 6 times each in 2.1.246, so this is a live contract, not a dormant one.
|
|
2150
|
+
"CLAUDE_CODE_PROMPT_CACHE_TTL",
|
|
2151
|
+
"CLAUDE_CODE_SUBAGENT_PROMPT_CACHE_TTL",
|
|
2112
2152
|
"USE_LOCAL_OAUTH",
|
|
2113
2153
|
"USE_STAGING_OAUTH",
|
|
2114
2154
|
];
|
|
@@ -2307,6 +2347,101 @@ function braceScanWindow(bundle, anchor) {
|
|
|
2307
2347
|
}
|
|
2308
2348
|
return null;
|
|
2309
2349
|
}
|
|
2350
|
+
/**
|
|
2351
|
+
* The literal that marks the 3p-only deployment branch. Anchoring on CONTENT rather than on the
|
|
2352
|
+
* predicate identifier is deliberate: the branch reads `...n&&{…}` where `n=<deployment>().type==="3p"`
|
|
2353
|
+
* is bound in the enclosing FUNCTION HEADER, which sits before every window's start anchor — so `n`
|
|
2354
|
+
* cannot be resolved from inside the window, and hard-coding `n` would be exactly the minified-name
|
|
2355
|
+
* anchoring that has broken sentinels three releases running (S6c, Cg, S14b). DISABLE_GROWTHBOOK has
|
|
2356
|
+
* been the first key of that literal in every build on record, and SPAWN_ENV_ALLOWLIST already treats
|
|
2357
|
+
* it as the branch's canonical marker in prose.
|
|
2358
|
+
*/
|
|
2359
|
+
const THIRD_PARTY_BRANCH_MARKER = "DISABLE_GROWTHBOOK:";
|
|
2360
|
+
/**
|
|
2361
|
+
* The `mcp__` membership of the spawn's `allowedTools[]`, pinned as a SET (S10b). Read from Desktop
|
|
2362
|
+
* 1.37937.1; `mcp__plugins__search_connectors` is the entry that was new in 1.37937.0 and that S9/S10
|
|
2363
|
+
* could not see. Note this array is the pre-approval list only — a name here does not mean the tool is
|
|
2364
|
+
* OFFERED: `search_connectors` is declared solely on the 3p deployment
|
|
2365
|
+
* (`getDynamicTools:()=>…type==="3p"?[…]:[]`, and its handler branch re-checks the same predicate), so
|
|
2366
|
+
* the first-party inventory the harness serves is unaffected. Re-classify, then re-pin, on any delta.
|
|
2367
|
+
*/
|
|
2368
|
+
const SPAWN_ALLOWED_MCP_TOOLS = [
|
|
2369
|
+
"mcp__mcp-registry__search_mcp_registry",
|
|
2370
|
+
"mcp__mcp-registry__suggest_connectors",
|
|
2371
|
+
"mcp__mcp-registry__list_connectors",
|
|
2372
|
+
"mcp__plugins__search_plugins",
|
|
2373
|
+
"mcp__plugins__search_connectors",
|
|
2374
|
+
"mcp__plugins__suggest_plugin_install",
|
|
2375
|
+
"mcp__plugins__list_plugins",
|
|
2376
|
+
"mcp__skills__list_skills",
|
|
2377
|
+
"mcp__skills__suggest_skills",
|
|
2378
|
+
"mcp__scheduled-tasks__list_scheduled_tasks",
|
|
2379
|
+
"mcp__computer-use",
|
|
2380
|
+
];
|
|
2381
|
+
/**
|
|
2382
|
+
* Slice the SPAWN's balanced `allowedTools:[…]` array text (bracket-aware, so a nested `[]` cannot end
|
|
2383
|
+
* it early). Anchored on the built-in head S9 pins, NOT on the bare `allowedTools:[` — the joined bundle
|
|
2384
|
+
* holds several unrelated `allowedTools` arrays (settings schemas, the bundled CLI) and the first one is
|
|
2385
|
+
* not the spawn's. A first-match slice reported all 11 pinned tools as removed at once, which is what a
|
|
2386
|
+
* mis-anchored window looks like: a total, implausible delta rather than a plausible one.
|
|
2387
|
+
*/
|
|
2388
|
+
function sliceAllowedToolsArray(bundle) {
|
|
2389
|
+
const a = bundle.search(/allowedTools:\["Task","Bash","Glob","Grep","Read","Edit","Write","NotebookEdit","WebFetch"/);
|
|
2390
|
+
if (a < 0)
|
|
2391
|
+
return null;
|
|
2392
|
+
const open = bundle.indexOf("[", a);
|
|
2393
|
+
let depth = 0;
|
|
2394
|
+
for (let i = open; i < bundle.length; i++) {
|
|
2395
|
+
const c = bundle[i];
|
|
2396
|
+
if (c === "[")
|
|
2397
|
+
depth++;
|
|
2398
|
+
else if (c === "]") {
|
|
2399
|
+
depth--;
|
|
2400
|
+
if (depth === 0)
|
|
2401
|
+
return bundle.slice(open, i + 1);
|
|
2402
|
+
}
|
|
2403
|
+
}
|
|
2404
|
+
return null;
|
|
2405
|
+
}
|
|
2406
|
+
/**
|
|
2407
|
+
* Slice the `...<pred>&&{…}` spread whose balanced body contains the 3p marker, or null when the window
|
|
2408
|
+
* carries no such branch (W1/W2 today). Returns the WHOLE spread text (`...<pred>&&{…}`) so the caller
|
|
2409
|
+
* can blank it wholesale. Brace matching is string-aware, since the body holds nested spreads and
|
|
2410
|
+
* quoted values.
|
|
2411
|
+
*/
|
|
2412
|
+
function sliceThirdPartyBranch(text) {
|
|
2413
|
+
for (const m of text.matchAll(/\.\.\.[^{}]{0,80}?&&\{/g)) {
|
|
2414
|
+
const open = m.index + m[0].length - 1;
|
|
2415
|
+
let depth = 0;
|
|
2416
|
+
let q = null;
|
|
2417
|
+
for (let i = open; i < text.length; i++) {
|
|
2418
|
+
const c = text[i];
|
|
2419
|
+
if (q) {
|
|
2420
|
+
if (c === "\\")
|
|
2421
|
+
i++;
|
|
2422
|
+
else if (c === q)
|
|
2423
|
+
q = null;
|
|
2424
|
+
continue;
|
|
2425
|
+
}
|
|
2426
|
+
if (c === '"' || c === "'" || c === "`") {
|
|
2427
|
+
q = c;
|
|
2428
|
+
continue;
|
|
2429
|
+
}
|
|
2430
|
+
if (c === "{")
|
|
2431
|
+
depth++;
|
|
2432
|
+
else if (c === "}") {
|
|
2433
|
+
depth--;
|
|
2434
|
+
if (depth === 0) {
|
|
2435
|
+
const body = text.slice(open, i + 1);
|
|
2436
|
+
if (body.includes(THIRD_PARTY_BRANCH_MARKER))
|
|
2437
|
+
return text.slice(m.index, i + 1);
|
|
2438
|
+
break;
|
|
2439
|
+
}
|
|
2440
|
+
}
|
|
2441
|
+
}
|
|
2442
|
+
}
|
|
2443
|
+
return null;
|
|
2444
|
+
}
|
|
2310
2445
|
/** Parse `K:"v"`-style inner pairs of a `{…}` object body (used for gate-conditional spread inners). */
|
|
2311
2446
|
function enumSpawnKeys(text) {
|
|
2312
2447
|
const out = [];
|
|
@@ -2403,6 +2538,27 @@ export function deriveSpawnEnv(bundle, gates, files) {
|
|
|
2403
2538
|
hardFail = true;
|
|
2404
2539
|
}
|
|
2405
2540
|
};
|
|
2541
|
+
// Inner key of the 3p-only branch (`...<deploymentType==="3p">&&{…}` — located by CONTENT, see
|
|
2542
|
+
// THIRD_PARTY_BRANCH_MARKER). Classify by NAME ONLY: the branch is the classifier, and the harness
|
|
2543
|
+
// models a FIRST-PARTY session, so no value inside it can ever reach the modeled env.
|
|
2544
|
+
//
|
|
2545
|
+
// - allowlisted → fine (this is what every "3p-provider-only branch" entry already means);
|
|
2546
|
+
// - PINNED → fine, and deliberately NOT resolved. A pinned key earns its value from its 1p site in
|
|
2547
|
+
// W1/W2; a 3p-only expression says nothing about it. Resolving here would (a) flag on shapes that
|
|
2548
|
+
// are irrelevant to the modeled session — Desktop 1.37937.0's `String(i)`, where `i` is a
|
|
2549
|
+
// chunk-local `let` — and (b) be actively dangerous if it DID resolve: `resolveConst` hops a
|
|
2550
|
+
// 1-char name against a whole 2 MB chunk, so a build whose first `i=` happened to be numeric would
|
|
2551
|
+
// silently write a 3p value. W1 is applied last and would win either way, so nothing is lost.
|
|
2552
|
+
// - unknown → still a hard-fail. A brand-new key must be classified even here, which is the whole
|
|
2553
|
+
// point of keeping the branch enumerated rather than discarding it.
|
|
2554
|
+
const classifyThirdPartyInner = (rawKey) => {
|
|
2555
|
+
if (SPAWN_ENV_ALLOWLIST[rawKey] !== undefined)
|
|
2556
|
+
return;
|
|
2557
|
+
if (SPAWN_PIN_KEYS.includes(rawKey))
|
|
2558
|
+
return;
|
|
2559
|
+
flags.push(`spawn.env: unknown key ${rawKey} constructed in the 3p-only deployment branch — ${SPAWN_ADVICE}`);
|
|
2560
|
+
hardFail = true;
|
|
2561
|
+
};
|
|
2406
2562
|
// A window's non-gate-spread keys. Gate-conditional spreads (…<helper>("id")&&{…}) are handled first
|
|
2407
2563
|
// (they must be resolved against gate STATE, not read as plain literals — an off-gate NONBLOCKING:"0"
|
|
2408
2564
|
// must not override W2's "true"), then blanked so the generic pass never sees them. Helper name is
|
|
@@ -2410,6 +2566,31 @@ export function deriveSpawnEnv(bundle, gates, files) {
|
|
|
2410
2566
|
const applyWindow = (text, target, isW1) => {
|
|
2411
2567
|
let work = text;
|
|
2412
2568
|
const scope = chunkFor(text);
|
|
2569
|
+
// The 3p-only branch (W3 today; W2 would be handled the same way). Blanked BEFORE the generic pass so
|
|
2570
|
+
// it never reads the branch's inner keys as unconditional 1p literals.
|
|
2571
|
+
//
|
|
2572
|
+
// NOT applied to W1, deliberately. W1 is the window every modeled 1p key comes from, so a marker
|
|
2573
|
+
// appearing there is either a Desktop restructure or a false positive — and blanking on either would
|
|
2574
|
+
// silently delete real pinned keys from the derived env, an all-or-nothing contract violated
|
|
2575
|
+
// quietly. Flag and let the generic pass run instead: worst case the branch's keys read as
|
|
2576
|
+
// unconditional (loud, diff-visible), never a silent deletion.
|
|
2577
|
+
if (isW1) {
|
|
2578
|
+
if (sliceThirdPartyBranch(work) !== null) {
|
|
2579
|
+
flags.push(`spawn.env: the 3p-only deployment branch (marker \`${THIRD_PARTY_BRANCH_MARKER}\`) now appears in W1, the window the ` +
|
|
2580
|
+
`modeled first-party env is derived from — re-derive which window owns the deployment split before trusting this env; ${SPAWN_NO_BYPASS}`);
|
|
2581
|
+
hardFail = true;
|
|
2582
|
+
}
|
|
2583
|
+
}
|
|
2584
|
+
else {
|
|
2585
|
+
const tp = sliceThirdPartyBranch(work);
|
|
2586
|
+
if (tp) {
|
|
2587
|
+
for (const k of enumSpawnKeys(tp)) {
|
|
2588
|
+
enumerated.add(k.key);
|
|
2589
|
+
classifyThirdPartyInner(k.key);
|
|
2590
|
+
}
|
|
2591
|
+
work = work.replace(tp, "");
|
|
2592
|
+
}
|
|
2593
|
+
}
|
|
2413
2594
|
if (isW1) {
|
|
2414
2595
|
// B6: the gate helper acquired an `o.`-style receiver here too (`...o.isFeatureEnabled("id")&&{…}`);
|
|
2415
2596
|
// without this widening the block is never blanked and the generic pass below reads its inner keys
|
|
@@ -2718,6 +2899,24 @@ export function checkSpawnContractFacts(bundle, files) {
|
|
|
2718
2899
|
miss("S9 allowedTools head", "the allowedTools[] head list moved (AskUserQuestion is tools-only)");
|
|
2719
2900
|
if (!has(/allowedTools:\[[^\]]{0,400}"ToolSearch","mcp__/))
|
|
2720
2901
|
miss("S10 allowedTools tail-guard", "the built-in→mcp__ boundary moved");
|
|
2902
|
+
// S10b (Desktop 1.37937.0): S9 pins the built-in HEAD and S10 only the built-in→mcp__ BOUNDARY, so the
|
|
2903
|
+
// `mcp__` region between the boundary and the closing bracket was unguarded — and a real addition
|
|
2904
|
+
// (`mcp__plugins__search_connectors`) shipped through both of them green. Pin the membership itself.
|
|
2905
|
+
// A set comparison, not a literal regex, so the flag NAMES the delta instead of just saying "moved".
|
|
2906
|
+
{
|
|
2907
|
+
const arr = sliceAllowedToolsArray(bundle);
|
|
2908
|
+
if (arr == null)
|
|
2909
|
+
miss("S10b allowedTools mcp__ membership", "the allowedTools[] array could not be sliced");
|
|
2910
|
+
else {
|
|
2911
|
+
const seen = new Set((arr.match(/"mcp__[^"]*"/g) ?? []).map((s) => s.slice(1, -1)));
|
|
2912
|
+
const pinned = new Set(SPAWN_ALLOWED_MCP_TOOLS);
|
|
2913
|
+
const added = [...seen].filter((t) => !pinned.has(t)).sort();
|
|
2914
|
+
const gone = [...pinned].filter((t) => !seen.has(t)).sort();
|
|
2915
|
+
if (added.length > 0 || gone.length > 0)
|
|
2916
|
+
miss("S10b allowedTools mcp__ membership", `the allowedTools[] mcp__ set changed${added.length ? ` (+${added.join(", ")})` : ""}${gone.length ? ` (-${gone.join(", ")})` : ""}` +
|
|
2917
|
+
" — classify each entry (which server serves it, and whether it is offered on the 1p deployment) before re-pinning SPAWN_ALLOWED_MCP_TOOLS");
|
|
2918
|
+
}
|
|
2919
|
+
}
|
|
2721
2920
|
// S11/S12 scoped to W1; S13 scoped to W2 — the earn-the-pin assertions for local-agent / cron / provider.
|
|
2722
2921
|
if (!w1 || !has(/CLAUDE_CODE_ENTRYPOINT:"local-agent"/, w1))
|
|
2723
2922
|
miss("S11 ENTRYPOINT local-agent", "the W1 local-agent entrypoint literal is gone");
|
package/dist/types.js
CHANGED
|
@@ -222,10 +222,24 @@ const toolGlob = z
|
|
|
222
222
|
// `cowork-harness assertions --list` (which reads `Assertion.shape[k].description`) — the list can never drift
|
|
223
223
|
// from the schema. Keep descriptions one line.
|
|
224
224
|
export const Assertion = z.strictObject({
|
|
225
|
-
transcript_contains: z
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
225
|
+
transcript_contains: z
|
|
226
|
+
.string()
|
|
227
|
+
.min(1)
|
|
228
|
+
.optional()
|
|
229
|
+
.describe("the transcript contains this literal substring. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing"),
|
|
230
|
+
transcript_not_contains: z
|
|
231
|
+
.string()
|
|
232
|
+
.min(1)
|
|
233
|
+
.optional()
|
|
234
|
+
.describe("the transcript does NOT contain this literal substring. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing"),
|
|
235
|
+
transcript_matches: z
|
|
236
|
+
.string()
|
|
237
|
+
.optional()
|
|
238
|
+
.describe("regex (case-insensitive) over the transcript — fuzzy content for stochastic prose. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing"),
|
|
239
|
+
transcript_not_matches: z
|
|
240
|
+
.string()
|
|
241
|
+
.optional()
|
|
242
|
+
.describe("regex (case-insensitive) that must NOT match the transcript. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so text the agent emitted inside a tool call (an AskUserQuestion gate question, an option label or description) can never match at any phrasing"),
|
|
229
243
|
tool_result_contains: z
|
|
230
244
|
.string()
|
|
231
245
|
.min(1)
|
|
@@ -507,12 +521,15 @@ export const Assertion = z.strictObject({
|
|
|
507
521
|
computer_links_resolve: z
|
|
508
522
|
.literal(true)
|
|
509
523
|
.optional()
|
|
510
|
-
.describe("fails if any computer:// link in the model-visible transcript does not resolve to an artifact that exists in the run's collected outputs/mounts — REQUIRES at least one link (zero links FAILS: use computer_links_resolve_if_present for the presence-free variant); only `true` is valid (writing `false` is a rejected footgun — omit to skip)"),
|
|
524
|
+
.describe("fails if any computer:// link in the model-visible transcript does not resolve to an artifact that exists in the run's collected outputs/mounts — REQUIRES at least one link (zero links FAILS: use computer_links_resolve_if_present for the presence-free variant); only `true` is valid (writing `false` is a rejected footgun — omit to skip). Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so a computer:// link that appeared only inside a tool call or its result is invisible here"),
|
|
511
525
|
computer_links_resolve_if_present: z
|
|
512
526
|
.literal(true)
|
|
513
527
|
.optional()
|
|
514
|
-
.describe("like computer_links_resolve, but PASSES VACUOUSLY when the transcript has zero computer:// links — the lenient, presence-free variant; only `true` is valid"),
|
|
515
|
-
question_asked: z
|
|
528
|
+
.describe("like computer_links_resolve, but PASSES VACUOUSLY when the transcript has zero computer:// links — the lenient, presence-free variant; only `true` is valid. Sees top-level assistant_text ONLY — it excludes every tool_use/tool_result, so a computer:// link that appeared only inside a tool call or its result is invisible here"),
|
|
529
|
+
question_asked: z
|
|
530
|
+
.string()
|
|
531
|
+
.optional()
|
|
532
|
+
.describe("a question matching this regex was asked. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose"),
|
|
516
533
|
question_options: z
|
|
517
534
|
.strictObject({
|
|
518
535
|
when_question: z
|
|
@@ -540,7 +557,20 @@ export const Assertion = z.strictObject({
|
|
|
540
557
|
message: "question_options: set exactly one of `equals` or `contains`",
|
|
541
558
|
})
|
|
542
559
|
.optional()
|
|
543
|
-
.describe("assert the option SET and ORDER a gate offered the user
|
|
560
|
+
.describe("assert the option SET and ORDER a gate offered the user, by LABEL (question_asked matches question text only; option DESCRIPTIONS are not compared here — use question_context for those). Exactly one of equals|contains is required. Evidence is captured at ask time, so it covers a gate that was shown and then denied/stalled/unanswered; a run whose gate evidence is absent fails evidence-unavailable, never vacuously. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose"),
|
|
561
|
+
question_context: z
|
|
562
|
+
.strictObject({
|
|
563
|
+
when_question: z
|
|
564
|
+
.string()
|
|
565
|
+
.optional()
|
|
566
|
+
.describe("regex narrowing to sub-questions whose label matches (the same string question_asked matches); omit to search EVERY gate — unlike question_options, omitting it is not ambiguous here, because this key asks whether the text was shown at all, not which gate offered which set"),
|
|
567
|
+
matches: z
|
|
568
|
+
.string()
|
|
569
|
+
.min(1)
|
|
570
|
+
.describe("regex that must match somewhere in the selected gate(s)' founder-visible payload (case-insensitive). NON-EMPTY: an empty pattern compiles to //i and matches every field of every gate, so it would green any run that fired one — a required field whose empty value asserts nothing is worse than an absent one"),
|
|
571
|
+
})
|
|
572
|
+
.optional()
|
|
573
|
+
.describe("a regex matched against everything a gate put in front of the user: the question label, every option LABEL, and every option DESCRIPTION. Use this when the skill's own wording may land in any of those fields — question_asked sees only the question text and question_options compares only labels, so a sentence delivered in an option's `description` is invisible to both. Evidence is the ask-time AskUserQuestion payload (never a producer's tool_result, which would grade true whether or not the model surfaced anything). Zero gates recorded FAILS; a lane that cannot read the gate payload fails evidence-unavailable, never vacuously. This text is model-composed and is reworded run to run — pin a producer-authored constant, not model prose"),
|
|
544
574
|
questions_count_max: z
|
|
545
575
|
.number()
|
|
546
576
|
.int()
|
|
@@ -645,7 +675,7 @@ export const Assertion = z.strictObject({
|
|
|
645
675
|
.describe("default false: also send each sub-agent's TEXT turns (RunResult.subagents[].reasoning, kind:'text' only) to the judge. Opt-in because it enlarges the judged document, which can re-grade an existing rubric. Use for a fan-out skill whose real work happens in sub-agents — their text is otherwise invisible to the judge. Sub-agent THINKING is excluded: it arrives empty with redacted:true, so including it would pad the document with blanks"),
|
|
646
676
|
})
|
|
647
677
|
.optional()
|
|
648
|
-
.describe("LIVE-ONLY: a pinned LLM judge grades the rubric against the run's answer; skipped-loud on replay (like egress_*). The judged document is finalMessage + transcript + authored files. NOTE the transcript is TOP-LEVEL assistant_text ONLY —
|
|
678
|
+
.describe("LIVE-ONLY: a pinned LLM judge grades the rubric against the run's answer; skipped-loud on replay (like egress_*). The judged document is finalMessage + transcript + authored files. NOTE the transcript is TOP-LEVEL assistant_text ONLY — it excludes every tool_use/tool_result, and no sub-agent text (even fork-scoped) unless include_subagent_text is set. A rubric claim about whether a TOOL was called can therefore never grade true; use tool_called/present_files_called/subagent_dispatched for that"),
|
|
649
679
|
});
|
|
650
680
|
/** Verdict modifiers: assertions that verify nothing themselves — each opts into (suppresses) one
|
|
651
681
|
* default-fail in `computeVerdict`. They are pure no-op `ok()` passes in `assert.ts` and are kept on
|
package/docs/README.md
CHANGED
|
@@ -29,6 +29,9 @@ reading order and tables that follow.
|
|
|
29
29
|
| **Test a skill that asks the user questions** | scripted `answers:` in [scenario.md](./scenario.md), or a live decider while you're still discovering what it asks ([decider-dir.md](./decider-dir.md)) |
|
|
30
30
|
| **Prove my skill doesn't reach the network, or leak host paths** | `egress_denied` / `transcript_no_host_path` in [scenario.md](./scenario.md#full-schema); the enforcement model in [boundary.md](./boundary.md) |
|
|
31
31
|
| Compare a skill across a fix (before/after, without averaging over both) | [debugging.md → iterate across fixes](./debugging.md#iterating-a-skill-across-fixes--the-verification-loop) + `stats --group-by skill-hash` ([stats.md](./stats.md)) |
|
|
32
|
+
| **My sub-agent wrote a file and nothing was delivered** | [subagents.md → Canonical outputs addressing](./subagents.md#canonical-outputs-addressing--tier-qualified-there-is-no-single-cross-tier-literal-form) — the outputs path is **tier-qualified**, so a hand-off path that works on one loop silently lands in sandbox scratch on the other; `analyze-skill` catches it statically, before you pay for a run |
|
|
33
|
+
| **Assert on how the run *behaved*, not what it produced** | [README → Assert on the run, not the output](../README.md#assert-on-the-run-not-the-output) — `subagent_tool_absent`, `dispatch_count_max`, `no_delete_in_outputs`: claims no output diff can reach |
|
|
34
|
+
| **See exactly what a gate put in front of the user** (option labels *and* descriptions) | `trace <run> --view questions`; assert it with `question_context` ([scenario.md](./scenario.md#full-schema)). When no view renders the field you need, read `events.jsonl` directly — [debugging.md → investigate the run](./debugging.md#the-skill-misbehaved--investigate-the-run) has the `jq` recipes |
|
|
32
35
|
| Debug a run that no-ops an assertion | [debugging.md](./debugging.md) |
|
|
33
36
|
| Check whether the sandbox really enforces the boundary | [boundary.md](./boundary.md) |
|
|
34
37
|
| Sync after a Claude Desktop update | [maintenance.md](./maintenance.md) |
|
package/docs/cassette.md
CHANGED
|
@@ -463,10 +463,10 @@ the rules and CI-placement rationale (why each category behaves this way), see
|
|
|
463
463
|
|
|
464
464
|
| Assertion key | What it checks |
|
|
465
465
|
|---|---|
|
|
466
|
-
| `transcript_contains` | literal substring in assistant transcript |
|
|
467
|
-
| `transcript_not_contains` | literal absent from transcript |
|
|
468
|
-
| `transcript_matches` | case-insensitive regex matches transcript |
|
|
469
|
-
| `transcript_not_matches` | regex does not match |
|
|
466
|
+
| `transcript_contains` | literal substring in assistant transcript **Sees top-level `assistant_text` only — it excludes every `tool_use`/`tool_result`**, so text the agent emitted only inside a tool call (an `AskUserQuestion` gate question or option, a tool result) can never match at any phrasing; use the gate keys (`question_asked`, `question_context`, `question_options`) or `tool_result_contains` for those. |
|
|
467
|
+
| `transcript_not_contains` | literal absent from transcript **Sees top-level `assistant_text` only — it excludes every `tool_use`/`tool_result`**, so text the agent emitted only inside a tool call (an `AskUserQuestion` gate question or option, a tool result) can never match at any phrasing; use the gate keys (`question_asked`, `question_context`, `question_options`) or `tool_result_contains` for those. |
|
|
468
|
+
| `transcript_matches` | case-insensitive regex matches transcript **Sees top-level `assistant_text` only — it excludes every `tool_use`/`tool_result`**, so text the agent emitted only inside a tool call (an `AskUserQuestion` gate question or option, a tool result) can never match at any phrasing; use the gate keys (`question_asked`, `question_context`, `question_options`) or `tool_result_contains` for those. |
|
|
469
|
+
| `transcript_not_matches` | regex does not match **Sees top-level `assistant_text` only — it excludes every `tool_use`/`tool_result`**, so text the agent emitted only inside a tool call (an `AskUserQuestion` gate question or option, a tool result) can never match at any phrasing; use the gate keys (`question_asked`, `question_context`, `question_options`) or `tool_result_contains` for those. |
|
|
470
470
|
| `tool_called` | agent invoked the named tool |
|
|
471
471
|
| `tool_not_called` | agent never invoked it |
|
|
472
472
|
| `tool_result_contains` | literal substring in a tool result |
|
|
@@ -501,6 +501,7 @@ the rules and CI-placement rationale (why each category behaves this way), see
|
|
|
501
501
|
| `task_count_min` | at least N tasks were created (`RunResult.tasks.length >= N`) — presence companion for task assertions; evidence-unavailable when `tasks` telemetry is absent |
|
|
502
502
|
| `task_status` | a task whose `subject` OR `id` matches the `match` regex reached the given `status` — evidence-unavailable when `tasks` telemetry is absent |
|
|
503
503
|
| `question_asked` | agent asked an AskUserQuestion matching the regex |
|
|
504
|
+
| `question_context` | a regex matched the gate's founder-visible payload (question label + option labels + option descriptions) |
|
|
504
505
|
| `question_options` | the option set + order that gate offered the user |
|
|
505
506
|
| `questions_count_max` | at most N **sub-questions** asked — a bundled `AskUserQuestion` with K sub-questions counts as K, not 1; `trace --view questions`'s footer total uses the same definition |
|
|
506
507
|
| `gate_answers_delivered: true` | answered gates' answers reached the model — **zero gates fired passes vacuously** (gate firing is model-dependent); pair with `gate_answer_count_min: >= 1` to also require a gate, or drop it and declare `questions_count_max: 0` in a gate-clean scenario |
|
|
@@ -522,7 +523,7 @@ the rules and CI-placement rationale (why each category behaves this way), see
|
|
|
522
523
|
| `allow_delete_in` | verdict modifier — kept on replay → no-op pass (the live per-mount delete scan it waives is zeroed on replay, same as its outputs-scoped sibling) |
|
|
523
524
|
| `allow_undelivered_deliverables` | verdict modifier — kept on replay → no-op pass (suppresses the `undelivered_deliverables` WARN; a replay runs no scratchpad walk of its own, so the signal is evidence-unavailable there regardless) |
|
|
524
525
|
|
|
525
|
-
**`question_asked`, `question_options`, `questions_count_max`, `gate_answers_delivered`, `gate_answer_count_min`, `hook_blocked`,
|
|
526
|
+
**`question_asked`, `question_options`, `question_context`, `questions_count_max`, `gate_answers_delivered`, `gate_answer_count_min`, `hook_blocked`,
|
|
526
527
|
`no_hook_blocked`, `vm_path_denied`, `path_denied`, `no_path_denied` require `controlOut`** (full-fidelity
|
|
527
528
|
replay). On an old cassette without `controlOut` these keys are excluded from evaluation — not vacuously
|
|
528
529
|
passed — and a loud warning fires (see §Backward compatibility). The hook and path-denial keys need
|
|
@@ -645,7 +646,7 @@ When the cassette carries `controlOut`, replay consumes **both** recorded direct
|
|
|
645
646
|
On replay, the replay decider (built by `buildReplayDecider()`) indexes `controlOut` by `request_id` and serves the recorded response to
|
|
646
647
|
the decision pipeline instead of consulting a live decider or asking the user. This makes the full
|
|
647
648
|
`Run.handleDecision` path execute on replay, which populates `rec.questions`, `rec.gateAnswers`, and
|
|
648
|
-
`rec.gateDeliveries` — exactly as in a live run. Consequence: `question_asked`, `question_options`, `questions_count_max`,
|
|
649
|
+
`rec.gateDeliveries` — exactly as in a live run. Consequence: `question_asked`, `question_options`, `question_context`, `questions_count_max`,
|
|
649
650
|
`gate_answers_delivered`, and `gate_answer_count_min` are now genuinely evaluated, not silently skipped
|
|
650
651
|
or vacuously passed.
|
|
651
652
|
|
|
@@ -729,7 +730,7 @@ regressing to the prior false-green behavior:
|
|
|
729
730
|
::warning:: [replay] cassette has no controlOut (pre-full-fidelity) — question/gate assertions
|
|
730
731
|
are NOT checked; re-record to enable them
|
|
731
732
|
```
|
|
732
|
-
2. **`question_asked`, `question_options`, `questions_count_max`, `gate_answers_delivered`, `gate_answer_count_min` are
|
|
733
|
+
2. **`question_asked`, `question_options`, `question_context`, `questions_count_max`, `gate_answers_delivered`, `gate_answer_count_min` are
|
|
733
734
|
excluded** from the evaluated assertion set for that run — not vacuously passed, absent.
|
|
734
735
|
3. All other content assertions (transcript, tool, subagent, result) evaluate normally.
|
|
735
736
|
|
|
@@ -895,6 +896,8 @@ real filesystem selection work `--dry-run` doesn't do, so the combination is rej
|
|
|
895
896
|
(`cowork-harness record scenarios/ --dry-run`): a superset of what actually gets re-recorded (every
|
|
896
897
|
committed cassette's source scenario, not just the ones whose fingerprint drifted), so it's conservative in
|
|
897
898
|
the right direction.
|
|
899
|
+
>
|
|
900
|
+
> **The path-dependent advisory notes in that preview do NOT describe `--rerecord-stale`.** The preview computes each scenario's DEFAULT cassette path, while `--rerecord-stale` writes to the committed cassette it found — a different path, so a different host-inventory/portability verdict. (A `--rerecord-stale` target always exists, so `hostInventoryPreflight` warns there rather than refusing.) The scenario LIST is still a conservative superset; only the notes are not transferable.
|
|
898
901
|
|
|
899
902
|
Each cassette is written **atomically** — to a same-directory temp file, then `rename`d over the target
|
|
900
903
|
(atomic on POSIX). An interrupted, failed, or OOM-killed batch therefore never leaves a partial or corrupt
|
|
@@ -1029,10 +1032,16 @@ counts). Uploads and `mode:r` connected folders are hash-only, and a file over t
|
|
|
1029
1032
|
**`record` carries its own preflight for the same risk.** Recording at a host-inheriting tier
|
|
1030
1033
|
(`protocol`, `hostloop`, or `cowork` resolving to hostloop) into a repo-visible cassette path is
|
|
1031
1034
|
refused by default — that recording would freeze this machine's MCP servers, agents, and account
|
|
1032
|
-
metadata into a committed fixture. `--allow-host-inventory-fixture`
|
|
1033
|
-
|
|
1034
|
-
|
|
1035
|
-
|
|
1035
|
+
metadata into a committed fixture. `--allow-host-inventory-fixture` bypasses **that pre-flight only**.
|
|
1036
|
+
It does NOT consent to a leak: the finished recording is still scanned (below), and a real finding still
|
|
1037
|
+
refuses the write and quarantines the recording. So you do not have to audit the session by hand to pass
|
|
1038
|
+
it — the scan is the actual gate. Writing a recording the scan DID flag needs the separate, louder
|
|
1039
|
+
`--allow-host-inventory-findings`. Both are distinct from `verify-cassettes`' `--allow-host-inventory
|
|
1040
|
+
<regex>` above (a per-finding suppressor on an already-committed cassette, not a record-time flag).
|
|
1041
|
+
|
|
1042
|
+
**Until 2.2.0 one flag did both jobs**, so an operator who passed it to get past a precondition they
|
|
1043
|
+
could not check also switched off the measured scan that would have caught a real leak — the escape
|
|
1044
|
+
hatch defeated the guard that was working.
|
|
1036
1045
|
**The preflight is a PREDICTION; `record` now also checks the EVIDENCE.** The preflight reads the tier
|
|
1037
1046
|
and the destination path before the paid spawn, which is the right place for it — but it never looks at
|
|
1038
1047
|
the resulting bytes, so it can be wrong in both directions. After redaction and before the write, the
|
|
@@ -1047,6 +1056,22 @@ counts). Uploads and `mode:r` connected folders are hash-only, and a file over t
|
|
|
1047
1056
|
inside a working tree, quarantine falls back to the OS temp dir and says so — moving a leak into another
|
|
1048
1057
|
committable location would be theatre.
|
|
1049
1058
|
|
|
1059
|
+
**The pre-flight IS previewable, and is previewed.** `record --dry-run` on a SINGLE scenario file runs
|
|
1060
|
+
the same refusal the real record does, from the same `--out` path and the same flags, and refuses
|
|
1061
|
+
identically — it is a pure function of tier and destination path, so it costs nothing to answer.
|
|
1062
|
+
On a **directory** the same verdicts are reported as `⚠ would-refuse (advisory)` notes and do **not**
|
|
1063
|
+
affect the exit code: a directory target takes no `--out`, so the preview must guess the destination, and
|
|
1064
|
+
a guess must not gate a batch. Dry-run the single file with the real flags to get a binding answer.
|
|
1065
|
+
|
|
1066
|
+
**What is deliberately NOT previewed is what the scan would FIND.** It is the obvious ask —
|
|
1067
|
+
"tell me the inventory before I pay" — and it cannot be answered honestly before the run. The pre-flight
|
|
1068
|
+
is a *prediction* from the tier and the destination path; the inventory itself does not exist until the
|
|
1069
|
+
agent has run and the `system/init` frame has been received. A preview would either re-implement the
|
|
1070
|
+
scanner against a hypothetical (a second oracle, free to disagree with the real one — the failure mode
|
|
1071
|
+
this whole layer exists to avoid) or require the spend it was meant to avoid. The scan on the finished
|
|
1072
|
+
bytes is the answer, and it is why you do not have to audit the session by hand to pass
|
|
1073
|
+
`--allow-host-inventory-fixture`.
|
|
1074
|
+
|
|
1050
1075
|
**The right way out is usually `fidelity: container`**, which is sealed (`HOME=/tmp`) and has nothing
|
|
1051
1076
|
to leak. Redirecting the *output* elsewhere is not equivalent: a cassette recorded outside the repo
|
|
1052
1077
|
and moved in afterwards is [unverifiable for staleness from its own location](#where-a-cassette-lives-and-why-it-cant-move-afterwards) — recoverable only by passing `--session <file>` on every invocation —
|
package/docs/debugging.md
CHANGED
|
@@ -71,6 +71,31 @@ already happened.
|
|
|
71
71
|
2. **`trace` — what did it actually do?** Digests the run's `events.jsonl` into the tools it called, the
|
|
72
72
|
questions (gates) it was asked and how they were answered, and the sub-agent dispatch tree. This is how
|
|
73
73
|
you answer "how many sub-agents *really* dispatched?" or "which gate fired, with what offered labels?"
|
|
74
|
+
— `--view questions` prints each gate's full ask-time payload, every option's **label and
|
|
75
|
+
`description`**, which is where a skill routinely puts the sentence the user is actually deciding on.
|
|
76
|
+
|
|
77
|
+
**When `trace` doesn't render the field you need, read `events.jsonl` directly.** The views are a
|
|
78
|
+
digest, not the record: the run dir's *evidence* surface is wider than any view's *observation*
|
|
79
|
+
surface, and wider still than the assertion catalog. Nothing in the pipeline discards an event, so a
|
|
80
|
+
field no command prints is still on disk — and concluding "it never happened" from a view that
|
|
81
|
+
doesn't render it is a false negative, not a finding. The file is one JSON object per line:
|
|
82
|
+
|
|
83
|
+
```bash
|
|
84
|
+
RUN=~/.cowork-harness/runs/<scenario>/<session> # `--keep` printed this path
|
|
85
|
+
# every AskUserQuestion payload, exactly as the model emitted it — options, descriptions and all
|
|
86
|
+
jq -c 'select(.type=="control_request" and .request.tool_name=="AskUserQuestion") | .request.input' "$RUN/events.jsonl"
|
|
87
|
+
# what the harness answered, keyed by request_id (the other half of the gate). Note the DOUBLE
|
|
88
|
+
# `.response.response` — the outer is the control envelope, the inner is the tool decision.
|
|
89
|
+
jq -c 'select(.response.response.updatedInput.answers)
|
|
90
|
+
| {request_id: .response.request_id, answers: .response.response.updatedInput.answers}' "$RUN/control-out.jsonl"
|
|
91
|
+
# the raw line behind a `trace` row you want to see unabridged
|
|
92
|
+
sed -n '204p' "$RUN/events.jsonl" | jq .
|
|
93
|
+
```
|
|
94
|
+
|
|
95
|
+
If what you find there is worth *gating* on rather than just reading, promote it to an assertion:
|
|
96
|
+
[`question_context`](./scenario.md#full-schema) matches a regex against everything a gate put in front
|
|
97
|
+
of the user (question label, option labels, option descriptions), and `tool_result_contains` /
|
|
98
|
+
`artifact_json` cover most of the rest. An assertion is the version of the finding that survives you.
|
|
74
99
|
3. **`verify-run` — re-assert cheaply.** If the run itself was fine and only an *assertion* is wrong,
|
|
75
100
|
re-check the scenario's assertions against the kept run dir with no live re-record (~1s). When the
|
|
76
101
|
scenario scripts answers, it also re-checks they still match the run's actual gates — so a reworded gate
|
package/docs/fidelity-gaps.md
CHANGED
|
@@ -898,7 +898,11 @@ The remote lane also has an internal MCP server the harness does not model at al
|
|
|
898
898
|
`internal__remote-devices__<tool>` (note the `internal__` prefix, not `mcp__`). Agent-facing tools include
|
|
899
899
|
`device_bash`, `device_list_dir`, `device_stage_files`, `device_commit_files`,
|
|
900
900
|
`device_request_folder_access`, `get_device_info`, and device-artifact tools; `device_commit_files` is the
|
|
901
|
-
remote lane's write-to-disk leg and consumes a `file_uuid` from a prior `SendUserFile`. Desktop
|
|
901
|
+
remote lane's write-to-disk leg and consumes a `file_uuid` from a prior `SendUserFile`. Desktop 1.37937.0
|
|
902
|
+
added `device_fs` and `device_request_delete_permission` to that set, plus a folder-access announce mode
|
|
903
|
+
(`card` / `dialog` / `off`) — the Desktop half of the six `cowork_*` risk categories that had appeared in
|
|
904
|
+
agent 2.1.237's auto-mode permission rubric. **This list is illustrative, not an inventory** (see the
|
|
905
|
+
feature-gating note below), so it is not kept exhaustive; it is updated when a release moves the shape. Desktop advertises
|
|
902
906
|
these *outward* to a cloud session over a device-OAuth bridge; every handler logs
|
|
903
907
|
`session_type: "cowork-remote"`, and no `internal__*` name appears in the local spawn's tool list.
|
|
904
908
|
|
package/docs/maintenance.md
CHANGED
|
@@ -77,6 +77,20 @@ git commit -m "parity: sync to Desktop <new>"
|
|
|
77
77
|
cowork-harness run examples/scenarios/ # regression — drift shows as test diffs (this repo's scenarios live under examples/)
|
|
78
78
|
```
|
|
79
79
|
|
|
80
|
+
> **The live-asar tripwires are macOS-local ONLY — CI cannot run them, so a green CI is not evidence the
|
|
81
|
+
> baseline still matches the installed Desktop.** Every oracle that reads the real bundle gates on
|
|
82
|
+
> `process.platform !== "darwin"` (`test/baseline.test.ts`), and `ci.yml` has **no macOS runner** — all
|
|
83
|
+
> its jobs are `ubuntu-latest` / `ubuntu-24.04-arm`. So the golden spawn-env oracle, the
|
|
84
|
+
> `checkSpawnContractFacts` real-asar regression and every mutation case built on the live bundle
|
|
85
|
+
> **skip in CI, silently**. Measured on this release: Desktop 1.37937.0 added a construction site the
|
|
86
|
+
> resolver could not parse, and both `ci.yml` and `release.yml` would have gone green with that drift
|
|
87
|
+
> unfixed — only running `sync` on a Mac surfaced it.
|
|
88
|
+
>
|
|
89
|
+
> Practical consequence: **the per-release runbook above is the only thing that catches parity drift.**
|
|
90
|
+
> Do not defer it on the strength of a green pipeline. A committed-asar fixture tier (so the oracles have
|
|
91
|
+
> something to run against off-macOS), or at minimum a loud skip in the CI job summary rather than a
|
|
92
|
+
> `console.warn` nobody reads, is open work.
|
|
93
|
+
|
|
80
94
|
If the agent version bumped, there is no image rebuild: the agent ELF is bind-mounted at runtime from the staged Desktop install (`resolveAgentBinary`, `src/baseline.ts`), not baked into the container image. A bumped `agentVersion` only updates `agentBinary.stagedPath` in the baseline (`src/cli.ts`); the container picks up the new binary from that path.
|
|
81
95
|
|
|
82
96
|
### Agent-binary provenance (`sha256`)
|
|
@@ -97,7 +111,7 @@ Another runtime knob in the same family: `COWORK_HARNESS_RESOURCE_INTERVAL_MS` s
|
|
|
97
111
|
Old staged binaries are re-downloadable from Anthropic's own release channel. For the **container/microvm** tiers the harness needs the **Linux/arm64 ELF**, so download it directly and point the resolver at it:
|
|
98
112
|
|
|
99
113
|
```bash
|
|
100
|
-
V=2.1.
|
|
114
|
+
V=2.1.246 # your baseline's agentVersion (read it from baselines/desktop-<latest>.json)
|
|
101
115
|
curl -fSL "https://downloads.claude.ai/claude-code-releases/$V/linux-arm64/claude" -o "claude-$V"
|
|
102
116
|
# verify against the committed baseline sha256 (== manifest platforms["linux-arm64"].checksum):
|
|
103
117
|
shasum -a 256 "claude-$V"
|