cocheck 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/dist/agent-transcripts.js +15 -12
- package/dist/attribution/evidence.js +2 -1
- package/dist/attribution/reconstruct.js +19 -0
- package/dist/attribution/tag.js +3 -3
- package/dist/build-info.json +1 -1
- package/dist/cli.js +31 -11
- package/dist/connector/export-view.js +1 -1
- package/dist/coverage.js +5 -3
- package/dist/credit-guard.js +59 -54
- package/dist/harness-cmd/create.js +76 -0
- package/dist/harness-cmd/index.js +19 -0
- package/dist/harness-cmd/record.js +327 -0
- package/dist/harness-cmd/scrub.js +25 -0
- package/dist/harness-cmd/test.js +318 -0
- package/dist/harnesses/README.md +5 -0
- package/dist/harnesses/claude/cocheck-harness.json +344 -0
- package/dist/harnesses/codex/cocheck-harness.json +212 -0
- package/dist/harnesses/copilot/cocheck-harness.json +296 -0
- package/dist/{copilot-helpers.js → helper-linker.js} +43 -32
- package/dist/hook-adapter.js +287 -63
- package/dist/hook-cli.js +73 -36
- package/dist/hook-journal.js +40 -12
- package/dist/hosts/claude.js +23 -159
- package/dist/hosts/codex.js +41 -81
- package/dist/hosts/compile.js +275 -0
- package/dist/hosts/copilot.js +7 -162
- package/dist/hosts/extras.js +55 -0
- package/dist/hosts/harnesses.js +66 -0
- package/dist/hosts/index.js +176 -8
- package/dist/hosts/install-block.js +332 -0
- package/dist/hosts/manifest.js +372 -0
- package/dist/hosts/shared.js +21 -0
- package/dist/hosts/tool-classes.js +6 -0
- package/dist/hosts/types.js +1 -0
- package/dist/install.js +69 -22
- package/dist/ledger.js +11 -0
- package/dist/normalize.js +33 -9
- package/dist/open-pick.js +5 -3
- package/dist/patch-secrets.js +11 -11
- package/dist/review-server.js +2 -2
- package/dist/timeline/attribution.js +2 -2
- package/dist/timeline/describe.js +20 -17
- package/dist/timeline/load.js +2 -2
- package/dist/timeline/rail/assets/review.js +1 -1
- package/dist/timeline/rail/model.js +8 -8
- package/dist/timeline/read-signal.js +4 -4
- package/dist/timeline/render/view-v1.js +2 -13
- package/dist/timeline/runs.js +9 -9
- package/dist/timeline/timeline.js +8 -8
- package/dist/timeline/write-signal.js +4 -3
- package/package.json +2 -2
- package/public-kit/harness/GUIDE.md +81 -0
- package/public-kit/harness/README.md +10 -0
- package/public-kit/harness/cases/claude/shell-writes-file.json +120 -0
- package/public-kit/harness/cases/codex/apply-patch-writes-files.json +177 -0
- package/public-kit/harness/cases/codex/helper-writes-file.json +160 -0
- package/public-kit/harness/cases/codex/shell-writes-and-fails.json +110 -0
- package/public-kit/harness/cases/copilot/camel-case-patch-and-failure.json +126 -0
- package/public-kit/harness/cases/copilot/helper-writes-file.json +209 -0
- package/public-kit/harness/cases/copilot/vscode-helper-writes-file.json +236 -0
- package/public-kit/harness/examples/toyagent/cases/write-notes.json +142 -0
- package/public-kit/harness/examples/toyagent/cocheck-harness.json +47 -0
- package/public-kit/harness/examples/toyagent/recordings/payloads.jsonl +8 -0
- package/public-kit/harness/harnesses/antigravity/README.md +32 -0
- package/public-kit/harness/harnesses/antigravity/cases/failed-edit.json +99 -0
- package/public-kit/harness/harnesses/antigravity/cases/helper-writes-file.json +270 -0
- package/public-kit/harness/harnesses/antigravity/cases/shell-ok-fail-and-denied.json +211 -0
- package/public-kit/harness/harnesses/antigravity/cases/write-then-edit.json +191 -0
- package/public-kit/harness/harnesses/antigravity/cocheck-harness.json +62 -0
- package/public-kit/harness/harnesses/antigravity/recordings/payloads.jsonl +63 -0
- package/public-kit/harness/harnesses/cursor/README.md +40 -0
- package/public-kit/harness/harnesses/cursor/cases/editor-write-edit-shell-helper.json +359 -0
- package/public-kit/harness/harnesses/cursor/cases/read-then-write-one-call-id.json +159 -0
- package/public-kit/harness/harnesses/cursor/cocheck-harness.json +67 -0
- package/public-kit/harness/harnesses/cursor/recordings/payloads.jsonl +28 -0
- package/public-kit/harness/scaffold/README.md +42 -0
- package/public-kit/harness/scaffold/cocheck-harness.json +72 -0
- package/public-kit/harness/scaffold/recordings/.gitkeep +0 -0
- package/public-kit/harness/spec/v1.md +286 -0
- package/scripts/hook-launch.sh +1 -1
|
@@ -0,0 +1,318 @@
|
|
|
1
|
+
// `cocheck harness test --cases <dir> [--manifest <path>]`: replays each case's recorded hook calls through the real hook path in a temporary
|
|
2
|
+
// data home and folder, then compares what the hook made of them (the normalized events and the credit of each file) with what the case expects.
|
|
3
|
+
// Nothing of the person's own data home, hook configs or folders is read or written: HOME, the data home and the agent homes all point into the temp tree.
|
|
4
|
+
import { cpSync, existsSync, mkdirSync, mkdtempSync, readdirSync, readFileSync, realpathSync, statSync, writeFileSync } from "node:fs";
|
|
5
|
+
import { rm } from "node:fs/promises";
|
|
6
|
+
import { DatabaseSync } from "node:sqlite";
|
|
7
|
+
import { tmpdir } from "node:os";
|
|
8
|
+
import path from "node:path";
|
|
9
|
+
import { runHook } from "../hook-cli.js";
|
|
10
|
+
import { normalize } from "../normalize.js";
|
|
11
|
+
import { allProfiles, isHostKey, loadOutsiderHarnesses } from "../hosts/index.js";
|
|
12
|
+
import { MANIFEST_FILE } from "../hosts/manifest.js";
|
|
13
|
+
import { setTracking } from "../tracking.js";
|
|
14
|
+
import { memoryKeyProvider, setKeyProvider } from "../seal.js";
|
|
15
|
+
import { stopWarmConnectors } from "../connector/diff.js";
|
|
16
|
+
import { stopAllConnectors } from "../connector/runner.js";
|
|
17
|
+
export const CASE_SCHEMA = "cocheck-harness-case/1";
|
|
18
|
+
export const LEVELS = ["L0", "L1", "L2", "L3"];
|
|
19
|
+
const isObject = (v) => v !== null && typeof v === "object" && !Array.isArray(v);
|
|
20
|
+
const isStrings = (v) => Array.isArray(v) && v.every((s) => typeof s === "string");
|
|
21
|
+
/** A relative path that stays inside the folder it is joined to. */
|
|
22
|
+
const inside = (p) => p !== "" && !path.isAbsolute(p) && !p.split(/[\\/]/).includes("..") && !/^[A-Za-z]:/.test(p);
|
|
23
|
+
/** Checks a parsed case file; returns the case or a plain sentence saying what is wrong. */
|
|
24
|
+
export function checkCaseShape(v) {
|
|
25
|
+
if (!isObject(v))
|
|
26
|
+
return "the case is not a JSON object";
|
|
27
|
+
if (v.schema !== CASE_SCHEMA)
|
|
28
|
+
return `schema: expected "${CASE_SCHEMA}"`;
|
|
29
|
+
if (typeof v.harness !== "string" || v.harness === "")
|
|
30
|
+
return "harness: expected the harness key";
|
|
31
|
+
if (typeof v.description !== "string" || v.description === "")
|
|
32
|
+
return "description: expected a sentence";
|
|
33
|
+
if (!LEVELS.includes(v.level))
|
|
34
|
+
return `level: expected one of ${LEVELS.join(", ")}`;
|
|
35
|
+
if (!Array.isArray(v.recording) || v.recording.length === 0)
|
|
36
|
+
return "recording: expected a list of recorded hook calls";
|
|
37
|
+
for (const [i, c] of v.recording.entries()) {
|
|
38
|
+
if (!isObject(c) || typeof c.event !== "string" || typeof c.stdin !== "string" || !isStrings(c.argv))
|
|
39
|
+
return `recording[${i}]: expected {event, stdin (text), argv (list of words)}`;
|
|
40
|
+
}
|
|
41
|
+
const files = v.files ?? {};
|
|
42
|
+
if (!isObject(files))
|
|
43
|
+
return "files: expected an object";
|
|
44
|
+
const before = files.before ?? {};
|
|
45
|
+
if (!isObject(before) || Object.entries(before).some(([p, c]) => !inside(p) || typeof c !== "string"))
|
|
46
|
+
return "files.before: expected file names inside the folder, each with its text";
|
|
47
|
+
const writes = files.writes ?? [];
|
|
48
|
+
if (!Array.isArray(writes))
|
|
49
|
+
return "files.writes: expected a list";
|
|
50
|
+
for (const [i, w] of writes.entries()) {
|
|
51
|
+
if (!isObject(w) || !Number.isInteger(w.afterEvent) || w.afterEvent < 1 || w.afterEvent > v.recording.length)
|
|
52
|
+
return `files.writes[${i}].afterEvent: expected the number (from 1) of a recorded call`;
|
|
53
|
+
if (typeof w.path !== "string" || !inside(w.path))
|
|
54
|
+
return `files.writes[${i}].path: expected a file name inside the folder`;
|
|
55
|
+
if (typeof w.content !== "string")
|
|
56
|
+
return `files.writes[${i}].content: expected text`;
|
|
57
|
+
}
|
|
58
|
+
const expect = v.expect;
|
|
59
|
+
if (!isObject(expect))
|
|
60
|
+
return "expect: expected an object";
|
|
61
|
+
if (!Array.isArray(expect.normalized) || expect.normalized.length !== v.recording.length || expect.normalized.some((e) => !isObject(e)))
|
|
62
|
+
return "expect.normalized: expected one object per recorded call";
|
|
63
|
+
if (!isObject(expect.credit))
|
|
64
|
+
return "expect.credit: expected an object";
|
|
65
|
+
for (const [f, c] of Object.entries(expect.credit)) {
|
|
66
|
+
if (!inside(f))
|
|
67
|
+
return `expect.credit: ${JSON.stringify(f)} is not a file name inside the folder`;
|
|
68
|
+
if (!isObject(c) || typeof c.creditedTo !== "string" || typeof c.basis !== "string")
|
|
69
|
+
return `expect.credit.${f}: expected {creditedTo, basis}`;
|
|
70
|
+
}
|
|
71
|
+
const credited = Object.values(expect.credit).filter((c) => c.creditedTo.startsWith("call:"));
|
|
72
|
+
if (credited.length === 0 && !/\bno credit\b/i.test(v.description))
|
|
73
|
+
return 'expect.credit holds no credited file; a case with none says why in its description (use the words "no credit")';
|
|
74
|
+
return v;
|
|
75
|
+
}
|
|
76
|
+
const jsonText = (s) => JSON.stringify(s).slice(1, -1);
|
|
77
|
+
const SAVED_ENV = ["HOME", "USERPROFILE", "COCHECK_HARNESS_DIR", "COCHECK_HOME"];
|
|
78
|
+
/** Replays `c` in a temporary tree. `manifest`: the harness's manifest file, for a harness that is not built in. */
|
|
79
|
+
export async function replayCase(c, manifest) {
|
|
80
|
+
const root = realpathSync(mkdtempSync(path.join(tmpdir(), "cocheck-harness-")));
|
|
81
|
+
const home = path.join(root, "home");
|
|
82
|
+
const folder = path.join(root, "ws");
|
|
83
|
+
mkdirSync(home);
|
|
84
|
+
mkdirSync(folder);
|
|
85
|
+
const saved = Object.fromEntries(SAVED_ENV.map((k) => [k, process.env[k]]));
|
|
86
|
+
process.env.HOME = root;
|
|
87
|
+
process.env.USERPROFILE = root;
|
|
88
|
+
process.env.COCHECK_HOME = home;
|
|
89
|
+
delete process.env.COCHECK_HARNESS_DIR;
|
|
90
|
+
// Every harness's own folders (named by its profile) move into the temp tree: a variable a profile names is saved, pointed at an empty folder there, and put back.
|
|
91
|
+
const pointHomes = () => {
|
|
92
|
+
for (const name of allProfiles().flatMap((p) => p.homeEnv ?? [])) {
|
|
93
|
+
if (!(name in saved))
|
|
94
|
+
saved[name] = process.env[name];
|
|
95
|
+
const dir = path.join(root, "agent-homes", name);
|
|
96
|
+
mkdirSync(dir, { recursive: true });
|
|
97
|
+
process.env[name] = dir;
|
|
98
|
+
}
|
|
99
|
+
};
|
|
100
|
+
const out = { normalized: [], credit: {} };
|
|
101
|
+
// The key that seals stored file copies lives in the login Keychain per data home; a temp home gets a key held in memory, so the Keychain is never touched.
|
|
102
|
+
setKeyProvider(memoryKeyProvider());
|
|
103
|
+
try {
|
|
104
|
+
loadOutsiderHarnesses(undefined);
|
|
105
|
+
pointHomes();
|
|
106
|
+
if (manifest !== undefined && isHostKey(c.harness))
|
|
107
|
+
out.warning = `${c.harness} is a built-in harness: its built-in profile is used, not the manifest at ${manifest}`;
|
|
108
|
+
if (!isHostKey(c.harness)) {
|
|
109
|
+
if (manifest === undefined)
|
|
110
|
+
return { ...out, fatal: `${c.harness} is not a built-in harness: pass --manifest <path to its ${MANIFEST_FILE}>` };
|
|
111
|
+
const target = path.join(home, "harnesses", c.harness);
|
|
112
|
+
mkdirSync(target, { recursive: true });
|
|
113
|
+
cpSync(manifest, path.join(target, MANIFEST_FILE));
|
|
114
|
+
const skipped = loadOutsiderHarnesses(path.join(home, "harnesses"));
|
|
115
|
+
if (skipped.length > 0)
|
|
116
|
+
return { ...out, fatal: `the manifest did not load: ${skipped.join("; ")}` };
|
|
117
|
+
pointHomes();
|
|
118
|
+
if (!isHostKey(c.harness))
|
|
119
|
+
return { ...out, fatal: `the manifest's key is not ${JSON.stringify(c.harness)}` };
|
|
120
|
+
}
|
|
121
|
+
for (const [file, text] of Object.entries(c.files.before ?? {})) {
|
|
122
|
+
mkdirSync(path.dirname(path.join(folder, file)), { recursive: true });
|
|
123
|
+
writeFileSync(path.join(folder, file), text);
|
|
124
|
+
}
|
|
125
|
+
const start = Date.UTC(2026, 10, 21, 12, 0, 0);
|
|
126
|
+
let clock = start;
|
|
127
|
+
for (const [i, call] of c.recording.entries()) {
|
|
128
|
+
const text = call.stdin.split("<folder>").join(jsonText(folder)).split("<root>").join(jsonText(root));
|
|
129
|
+
const eventIdx = call.argv.indexOf("--event");
|
|
130
|
+
const eventArg = eventIdx >= 0 ? call.argv[eventIdx + 1] : undefined;
|
|
131
|
+
let ev;
|
|
132
|
+
try {
|
|
133
|
+
ev = normalize(c.harness, JSON.parse(text), eventArg);
|
|
134
|
+
}
|
|
135
|
+
catch {
|
|
136
|
+
out.normalized.push(null);
|
|
137
|
+
}
|
|
138
|
+
if (ev !== undefined) {
|
|
139
|
+
const scrubbed = JSON.parse(JSON.stringify({ ...ev, callId: ev.toolUseId }).split(jsonText(folder)).join("<folder>").split(jsonText(root)).join("<root>"));
|
|
140
|
+
out.normalized.push(scrubbed);
|
|
141
|
+
if (typeof ev.session === "string" && ev.session !== "")
|
|
142
|
+
setTracking(home, c.harness, ev.session, true, undefined);
|
|
143
|
+
}
|
|
144
|
+
const argv = ["node", "hook.ts", "--host", c.harness, ...call.argv, "--folder", folder];
|
|
145
|
+
const res = await runHook({ argv, env: { ...process.env, COCHECK_HOME: home }, now: () => (clock += 1000), journalNow: () => start + i * 1000, stdin: (async function* () { yield Buffer.from(text); })() });
|
|
146
|
+
if (res.exitCode !== 0)
|
|
147
|
+
return { ...out, fatal: `call ${i + 1} (${call.event}): the hook exited ${res.exitCode}: ${res.stderr.trim()}` };
|
|
148
|
+
for (const w of c.files.writes ?? []) {
|
|
149
|
+
if (w.afterEvent !== i + 1)
|
|
150
|
+
continue;
|
|
151
|
+
mkdirSync(path.dirname(path.join(folder, w.path)), { recursive: true });
|
|
152
|
+
writeFileSync(path.join(folder, w.path), w.content);
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
out.credit = readCredit(home, folder);
|
|
156
|
+
return out;
|
|
157
|
+
}
|
|
158
|
+
finally {
|
|
159
|
+
setKeyProvider(null);
|
|
160
|
+
loadOutsiderHarnesses(undefined);
|
|
161
|
+
await stopWarmConnectors();
|
|
162
|
+
await stopAllConnectors();
|
|
163
|
+
for (const [k, v] of Object.entries(saved)) {
|
|
164
|
+
if (v === undefined)
|
|
165
|
+
delete process.env[k];
|
|
166
|
+
else
|
|
167
|
+
process.env[k] = v;
|
|
168
|
+
}
|
|
169
|
+
await rm(root, { recursive: true, force: true, maxRetries: 3, retryDelay: 100 });
|
|
170
|
+
}
|
|
171
|
+
}
|
|
172
|
+
/** The credit each file ends with: the last state recorded after a call (Post) for it, read from the temp ledger. */
|
|
173
|
+
function readCredit(home, folder) {
|
|
174
|
+
const dbFile = path.join(home, "ledger.sqlite");
|
|
175
|
+
if (!existsSync(dbFile))
|
|
176
|
+
return {};
|
|
177
|
+
const db = new DatabaseSync(dbFile, { readOnly: true });
|
|
178
|
+
try {
|
|
179
|
+
const rows = db.prepare("SELECT id, file_path, credited_json, credit_basis FROM file_states WHERE phase = 'Post' ORDER BY id").all();
|
|
180
|
+
const proofs = new Set(db.prepare("SELECT file_state_id AS id FROM file_state_proofs").all().map((p) => p.id));
|
|
181
|
+
const credit = {};
|
|
182
|
+
for (const r of rows) {
|
|
183
|
+
const rel = path.relative(folder, r.file_path).split(path.sep).join("/");
|
|
184
|
+
if (rel === "" || rel.startsWith("..") || path.isAbsolute(rel))
|
|
185
|
+
continue;
|
|
186
|
+
const ids = JSON.parse(r.credited_json);
|
|
187
|
+
const basis = proofs.has(r.id) ? "host-reported-edit" : r.credit_basis === "argv-tag" || r.credit_basis === "host-reported-edit" ? r.credit_basis : "inferred";
|
|
188
|
+
credit[rel] = { creditedTo: ids.length === 0 ? "unattributed" : ids.map((id) => `call:${id}`).join(","), basis };
|
|
189
|
+
}
|
|
190
|
+
return credit;
|
|
191
|
+
}
|
|
192
|
+
finally {
|
|
193
|
+
db.close();
|
|
194
|
+
}
|
|
195
|
+
}
|
|
196
|
+
/** The fields of an observed normalized event a generated case records. */
|
|
197
|
+
export function normalizedProjection(ev) {
|
|
198
|
+
if (ev === null)
|
|
199
|
+
return {};
|
|
200
|
+
const out = {};
|
|
201
|
+
for (const k of ["kind", "eventName", "session", "toolName", "callId", "agentId", "agentType", "prompt", "lastAssistantMessage"])
|
|
202
|
+
if (ev[k] !== undefined)
|
|
203
|
+
out[k] = ev[k];
|
|
204
|
+
if (isObject(ev.failure))
|
|
205
|
+
out.failure = { interrupted: ev.failure.interrupted };
|
|
206
|
+
return out;
|
|
207
|
+
}
|
|
208
|
+
/** Every key of `want` holds in `got` (objects compared key by key, anything else exactly). Returns the first difference, in words. */
|
|
209
|
+
function subsetDiff(want, got, at) {
|
|
210
|
+
if (isObject(want)) {
|
|
211
|
+
if (!isObject(got))
|
|
212
|
+
return `${at}: expected an object, got ${JSON.stringify(got)}`;
|
|
213
|
+
for (const [k, w] of Object.entries(want)) {
|
|
214
|
+
const d = subsetDiff(w, got[k], at === "" ? k : `${at}.${k}`);
|
|
215
|
+
if (d !== undefined)
|
|
216
|
+
return d;
|
|
217
|
+
}
|
|
218
|
+
return undefined;
|
|
219
|
+
}
|
|
220
|
+
return JSON.stringify(want) === JSON.stringify(got) ? undefined : `${at}: expected ${JSON.stringify(want)}, got ${got === undefined ? "nothing" : JSON.stringify(got)}`;
|
|
221
|
+
}
|
|
222
|
+
/** The first difference between what a case expects and what the replay saw, in plain words; undefined when they agree. */
|
|
223
|
+
export function firstDifference(c, seen) {
|
|
224
|
+
if (seen.fatal !== undefined)
|
|
225
|
+
return seen.fatal;
|
|
226
|
+
for (const [i, want] of c.expect.normalized.entries()) {
|
|
227
|
+
const got = seen.normalized[i];
|
|
228
|
+
const label = `call ${i + 1} (${c.recording[i].event})`;
|
|
229
|
+
if (got === null || got === undefined) {
|
|
230
|
+
if (Object.keys(want).length > 0)
|
|
231
|
+
return `${label}: the harness's payload could not be read into an event`;
|
|
232
|
+
continue;
|
|
233
|
+
}
|
|
234
|
+
const d = subsetDiff(want, got, "");
|
|
235
|
+
if (d !== undefined)
|
|
236
|
+
return `${label}: normalized event ${d}`;
|
|
237
|
+
}
|
|
238
|
+
for (const [file, want] of Object.entries(c.expect.credit)) {
|
|
239
|
+
const got = seen.credit[file];
|
|
240
|
+
if (got === undefined)
|
|
241
|
+
return `file ${file}: expected credit to ${want.creditedTo}, but no state was recorded for it`;
|
|
242
|
+
if (got.creditedTo !== want.creditedTo)
|
|
243
|
+
return `file ${file}: expected credit to ${want.creditedTo}, got ${got.creditedTo}`;
|
|
244
|
+
if (got.basis !== want.basis)
|
|
245
|
+
return `file ${file}: expected credit basis ${want.basis}, got ${got.basis}`;
|
|
246
|
+
}
|
|
247
|
+
return undefined;
|
|
248
|
+
}
|
|
249
|
+
/** The case files under `target` (a folder of `.json` files, or one file), sorted. */
|
|
250
|
+
function caseFiles(target) {
|
|
251
|
+
if (!existsSync(target))
|
|
252
|
+
return [];
|
|
253
|
+
if (statSync(target).isFile())
|
|
254
|
+
return [target];
|
|
255
|
+
return readdirSync(target).filter((n) => n.endsWith(".json")).sort().map((n) => path.join(target, n));
|
|
256
|
+
}
|
|
257
|
+
const TEST_USAGE = "usage: cocheck harness test --cases <dir> [--manifest <path>]";
|
|
258
|
+
export async function runHarnessTest(args, io) {
|
|
259
|
+
let cases;
|
|
260
|
+
let manifest;
|
|
261
|
+
for (let i = 0; i < args.length; i++) {
|
|
262
|
+
const a = args[i];
|
|
263
|
+
if (a === "--cases" && args[i + 1] !== undefined)
|
|
264
|
+
cases = args[++i];
|
|
265
|
+
else if (a === "--manifest" && args[i + 1] !== undefined)
|
|
266
|
+
manifest = args[++i];
|
|
267
|
+
else {
|
|
268
|
+
io.err(`unexpected ${JSON.stringify(a)}\n${TEST_USAGE}`);
|
|
269
|
+
return 1;
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (cases === undefined) {
|
|
273
|
+
io.err(TEST_USAGE);
|
|
274
|
+
return 1;
|
|
275
|
+
}
|
|
276
|
+
const manifestFile = manifest === undefined ? undefined : path.resolve(existsSync(manifest) && statSync(manifest).isDirectory() ? path.join(manifest, MANIFEST_FILE) : manifest);
|
|
277
|
+
if (manifestFile !== undefined && !existsSync(manifestFile)) {
|
|
278
|
+
io.err(`no manifest at ${manifestFile}`);
|
|
279
|
+
return 1;
|
|
280
|
+
}
|
|
281
|
+
const files = caseFiles(path.resolve(cases));
|
|
282
|
+
if (files.length === 0) {
|
|
283
|
+
io.err(`no case files (*.json) in ${path.resolve(cases)}`);
|
|
284
|
+
return 1;
|
|
285
|
+
}
|
|
286
|
+
let failed = 0;
|
|
287
|
+
const warned = new Set();
|
|
288
|
+
for (const file of files) {
|
|
289
|
+
const name = path.basename(file);
|
|
290
|
+
let problem;
|
|
291
|
+
let level = "";
|
|
292
|
+
try {
|
|
293
|
+
const parsed = checkCaseShape(JSON.parse(readFileSync(file, "utf8")));
|
|
294
|
+
if (typeof parsed === "string")
|
|
295
|
+
problem = parsed;
|
|
296
|
+
else {
|
|
297
|
+
level = ` (${parsed.harness} ${parsed.level})`;
|
|
298
|
+
const seen = await replayCase(parsed, manifestFile);
|
|
299
|
+
if (seen.warning !== undefined && !warned.has(seen.warning)) {
|
|
300
|
+
warned.add(seen.warning);
|
|
301
|
+
io.err(`warning: ${seen.warning}`);
|
|
302
|
+
}
|
|
303
|
+
problem = firstDifference(parsed, seen);
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
catch (error) {
|
|
307
|
+
problem = error instanceof SyntaxError ? "not valid JSON" : `could not run: ${error instanceof Error ? error.message : String(error)}`;
|
|
308
|
+
}
|
|
309
|
+
if (problem === undefined)
|
|
310
|
+
io.out(`pass ${name}${level}`);
|
|
311
|
+
else {
|
|
312
|
+
failed++;
|
|
313
|
+
io.out(`FAIL ${name}${level}: ${problem}`);
|
|
314
|
+
}
|
|
315
|
+
}
|
|
316
|
+
io.out(failed === 0 ? `${files.length} cases passed` : `${failed} of ${files.length} cases failed`);
|
|
317
|
+
return failed === 0 ? 0 : 1;
|
|
318
|
+
}
|
|
@@ -0,0 +1,5 @@
|
|
|
1
|
+
# Built-in harness manifests
|
|
2
|
+
|
|
3
|
+
Each folder here holds one `cocheck-harness.json` (schema `cocheck-harness/1`, see "Harness manifest (v1)" in docs/hooks.md) for a host CoCheck ships support for.
|
|
4
|
+
The folder name is the manifest's `key`. The build copies this folder to `dist/harnesses/`, so it ships in the npm package and the Homebrew install with the rest of `dist/`.
|
|
5
|
+
Adding a built-in harness is a data change here (manifest, plus its cases under `public-kit/harness/cases/<key>/`); nothing under `src/` changes unless the manifest opts into a first-party extra.
|
|
@@ -0,0 +1,344 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "cocheck-harness/1",
|
|
3
|
+
"key": "claude",
|
|
4
|
+
"displayName": "Claude",
|
|
5
|
+
"fullName": "Claude Code",
|
|
6
|
+
"chip": "C",
|
|
7
|
+
"testedVersions": [],
|
|
8
|
+
"events": {
|
|
9
|
+
"PreToolUse": "tool-pre",
|
|
10
|
+
"PostToolUse": "tool-post",
|
|
11
|
+
"PostToolUseFailure": "tool-failure",
|
|
12
|
+
"Stop": "stop",
|
|
13
|
+
"SubagentStop": "subagent-stop",
|
|
14
|
+
"UserPromptSubmit": "prompt"
|
|
15
|
+
},
|
|
16
|
+
"eventFrom": {
|
|
17
|
+
"payload": "hook_event_name"
|
|
18
|
+
},
|
|
19
|
+
"fields": {
|
|
20
|
+
"session": "session_id",
|
|
21
|
+
"eventName": "hook_event_name",
|
|
22
|
+
"cwd": "cwd",
|
|
23
|
+
"toolName": "tool_name",
|
|
24
|
+
"toolInput": "tool_input",
|
|
25
|
+
"toolResponse": "tool_response",
|
|
26
|
+
"agentId": "agent_id",
|
|
27
|
+
"agentType": "agent_type",
|
|
28
|
+
"transcriptPath": "transcript_path",
|
|
29
|
+
"lastAssistantMessage": "last_assistant_message",
|
|
30
|
+
"prompt": "prompt",
|
|
31
|
+
"callReason": "description"
|
|
32
|
+
},
|
|
33
|
+
"callId": {
|
|
34
|
+
"payload": "tool_use_id"
|
|
35
|
+
},
|
|
36
|
+
"extraFields": {
|
|
37
|
+
"turnId": "turn_id",
|
|
38
|
+
"permissionMode": "permission_mode",
|
|
39
|
+
"model": "model",
|
|
40
|
+
"stopHookActive": "stop_hook_active"
|
|
41
|
+
},
|
|
42
|
+
"failure": {
|
|
43
|
+
"tool-failure": {
|
|
44
|
+
"flags": [
|
|
45
|
+
"tool_response.is_error",
|
|
46
|
+
"tool_response.isError"
|
|
47
|
+
],
|
|
48
|
+
"text": [
|
|
49
|
+
"error"
|
|
50
|
+
],
|
|
51
|
+
"flaggedText": [
|
|
52
|
+
{
|
|
53
|
+
"path": "tool_response.content",
|
|
54
|
+
"blocks": true
|
|
55
|
+
},
|
|
56
|
+
"tool_response.error"
|
|
57
|
+
],
|
|
58
|
+
"interrupt": "is_interrupt"
|
|
59
|
+
},
|
|
60
|
+
"tool-post": {
|
|
61
|
+
"flags": [
|
|
62
|
+
"tool_response.is_error",
|
|
63
|
+
"tool_response.isError"
|
|
64
|
+
],
|
|
65
|
+
"text": [
|
|
66
|
+
"error"
|
|
67
|
+
],
|
|
68
|
+
"flaggedText": [
|
|
69
|
+
{
|
|
70
|
+
"path": "tool_response.content",
|
|
71
|
+
"blocks": true
|
|
72
|
+
},
|
|
73
|
+
"tool_response.error"
|
|
74
|
+
],
|
|
75
|
+
"interrupt": "is_interrupt"
|
|
76
|
+
}
|
|
77
|
+
},
|
|
78
|
+
"toolClasses": {
|
|
79
|
+
"Read": {
|
|
80
|
+
"class": "read"
|
|
81
|
+
},
|
|
82
|
+
"Glob": {
|
|
83
|
+
"class": "read-only"
|
|
84
|
+
},
|
|
85
|
+
"Grep": {
|
|
86
|
+
"class": "read-only"
|
|
87
|
+
},
|
|
88
|
+
"LS": {
|
|
89
|
+
"class": "read-only"
|
|
90
|
+
},
|
|
91
|
+
"NotebookRead": {
|
|
92
|
+
"class": "read-only"
|
|
93
|
+
},
|
|
94
|
+
"WebFetch": {
|
|
95
|
+
"class": "never-credited"
|
|
96
|
+
},
|
|
97
|
+
"WebSearch": {
|
|
98
|
+
"class": "never-credited"
|
|
99
|
+
},
|
|
100
|
+
"ToolSearch": {
|
|
101
|
+
"class": "never-credited"
|
|
102
|
+
},
|
|
103
|
+
"Skill": {
|
|
104
|
+
"class": "never-credited"
|
|
105
|
+
},
|
|
106
|
+
"TodoWrite": {
|
|
107
|
+
"class": "never-credited"
|
|
108
|
+
},
|
|
109
|
+
"TodoRead": {
|
|
110
|
+
"class": "never-credited"
|
|
111
|
+
},
|
|
112
|
+
"TaskCreate": {
|
|
113
|
+
"class": "never-credited"
|
|
114
|
+
},
|
|
115
|
+
"TaskUpdate": {
|
|
116
|
+
"class": "never-credited"
|
|
117
|
+
},
|
|
118
|
+
"TaskList": {
|
|
119
|
+
"class": "never-credited"
|
|
120
|
+
},
|
|
121
|
+
"TaskGet": {
|
|
122
|
+
"class": "never-credited"
|
|
123
|
+
},
|
|
124
|
+
"TaskOutput": {
|
|
125
|
+
"class": "never-credited"
|
|
126
|
+
},
|
|
127
|
+
"BashOutput": {
|
|
128
|
+
"class": "never-credited"
|
|
129
|
+
},
|
|
130
|
+
"AskUserQuestion": {
|
|
131
|
+
"class": "never-credited"
|
|
132
|
+
},
|
|
133
|
+
"ExitPlanMode": {
|
|
134
|
+
"class": "never-credited"
|
|
135
|
+
},
|
|
136
|
+
"EnterPlanMode": {
|
|
137
|
+
"class": "never-credited"
|
|
138
|
+
},
|
|
139
|
+
"ListMcpResourcesTool": {
|
|
140
|
+
"class": "never-credited"
|
|
141
|
+
},
|
|
142
|
+
"ReadMcpResourceTool": {
|
|
143
|
+
"class": "never-credited"
|
|
144
|
+
},
|
|
145
|
+
"SendMessage": {
|
|
146
|
+
"class": "never-credited",
|
|
147
|
+
"resumesHelper": true
|
|
148
|
+
},
|
|
149
|
+
"Write": {
|
|
150
|
+
"class": "write",
|
|
151
|
+
"pathFields": [
|
|
152
|
+
"file_path",
|
|
153
|
+
"path"
|
|
154
|
+
],
|
|
155
|
+
"pathChecked": true,
|
|
156
|
+
"fileInput": true,
|
|
157
|
+
"anyCase": true
|
|
158
|
+
},
|
|
159
|
+
"Edit": {
|
|
160
|
+
"class": "edit",
|
|
161
|
+
"pathFields": [
|
|
162
|
+
"file_path",
|
|
163
|
+
"path"
|
|
164
|
+
],
|
|
165
|
+
"pathChecked": true,
|
|
166
|
+
"fileInput": true,
|
|
167
|
+
"anyCase": true
|
|
168
|
+
},
|
|
169
|
+
"MultiEdit": {
|
|
170
|
+
"class": "edit",
|
|
171
|
+
"pathFields": [
|
|
172
|
+
"file_path"
|
|
173
|
+
],
|
|
174
|
+
"pathChecked": true,
|
|
175
|
+
"fileInput": true,
|
|
176
|
+
"anyCase": true
|
|
177
|
+
},
|
|
178
|
+
"NotebookEdit": {
|
|
179
|
+
"class": "notebook-edit",
|
|
180
|
+
"pathFields": [
|
|
181
|
+
"notebook_path"
|
|
182
|
+
],
|
|
183
|
+
"fileInput": true,
|
|
184
|
+
"anyCase": true
|
|
185
|
+
},
|
|
186
|
+
"apply_patch": {
|
|
187
|
+
"class": "patch",
|
|
188
|
+
"anyCase": true
|
|
189
|
+
},
|
|
190
|
+
"patch": {
|
|
191
|
+
"class": "patch",
|
|
192
|
+
"anyCase": true
|
|
193
|
+
},
|
|
194
|
+
"Agent": {
|
|
195
|
+
"class": "launcher",
|
|
196
|
+
"backgroundFlag": true
|
|
197
|
+
},
|
|
198
|
+
"Task": {
|
|
199
|
+
"class": "launcher",
|
|
200
|
+
"backgroundFlag": true
|
|
201
|
+
},
|
|
202
|
+
"Bash": {
|
|
203
|
+
"class": "shell",
|
|
204
|
+
"dialect": "bash",
|
|
205
|
+
"backgroundable": true,
|
|
206
|
+
"anyCase": true
|
|
207
|
+
},
|
|
208
|
+
"PowerShell": {
|
|
209
|
+
"class": "shell",
|
|
210
|
+
"dialect": "powershell",
|
|
211
|
+
"anyCase": true
|
|
212
|
+
},
|
|
213
|
+
"pwsh": {
|
|
214
|
+
"class": "shell",
|
|
215
|
+
"dialect": "powershell",
|
|
216
|
+
"anyCase": true
|
|
217
|
+
}
|
|
218
|
+
},
|
|
219
|
+
"narration": {
|
|
220
|
+
"sources": [
|
|
221
|
+
{
|
|
222
|
+
"kind": "text-before-call",
|
|
223
|
+
"readableAt": "pre"
|
|
224
|
+
},
|
|
225
|
+
{
|
|
226
|
+
"kind": "closing-message",
|
|
227
|
+
"readableAt": "post"
|
|
228
|
+
}
|
|
229
|
+
]
|
|
230
|
+
},
|
|
231
|
+
"sessionFile": {
|
|
232
|
+
"bySessionId": false
|
|
233
|
+
},
|
|
234
|
+
"capabilities": {
|
|
235
|
+
"failureEvent": true,
|
|
236
|
+
"argvTag": true
|
|
237
|
+
},
|
|
238
|
+
"argvTag": {
|
|
239
|
+
"tool": "Bash",
|
|
240
|
+
"preOutput": {
|
|
241
|
+
"hookSpecificOutput": {
|
|
242
|
+
"hookEventName": "{event}",
|
|
243
|
+
"updatedInput": "{input}"
|
|
244
|
+
}
|
|
245
|
+
},
|
|
246
|
+
"envFileVar": "CLAUDE_ENV_FILE",
|
|
247
|
+
"shellEnv": "CLAUDE_CODE_SHELL",
|
|
248
|
+
"sessionStartEvent": "SessionStart"
|
|
249
|
+
},
|
|
250
|
+
"sessionEnv": "CLAUDE_CODE_SESSION_ID",
|
|
251
|
+
"hostOwnedFiles": [
|
|
252
|
+
".claude/settings.local.json"
|
|
253
|
+
],
|
|
254
|
+
"hostOwnedWhat": "the Claude app saving your permission choices",
|
|
255
|
+
"install": {
|
|
256
|
+
"skillRoot": ".claude/skills",
|
|
257
|
+
"inBothAlias": true,
|
|
258
|
+
"timingLabel": "Claude Code",
|
|
259
|
+
"hostFlag": "claude",
|
|
260
|
+
"projectDirEnv": "CLAUDE_PROJECT_DIR",
|
|
261
|
+
"checkPayload": {
|
|
262
|
+
"payload": {
|
|
263
|
+
"session_id": "install-check",
|
|
264
|
+
"cwd": "{folder}",
|
|
265
|
+
"hook_event_name": "PreToolUse",
|
|
266
|
+
"tool_name": "Bash",
|
|
267
|
+
"tool_use_id": "install-check-1",
|
|
268
|
+
"tool_input": {
|
|
269
|
+
"command": "true"
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
},
|
|
273
|
+
"afterInstall": "Restart Claude Code so it reads the new settings.",
|
|
274
|
+
"format": "json",
|
|
275
|
+
"path": "{env:CLAUDE_CONFIG_DIR|{home}/.claude}/settings.json",
|
|
276
|
+
"mode": "merge",
|
|
277
|
+
"rootKey": "hooks",
|
|
278
|
+
"events": [
|
|
279
|
+
"PreToolUse",
|
|
280
|
+
"PostToolUse",
|
|
281
|
+
"PostToolUseFailure",
|
|
282
|
+
"Stop",
|
|
283
|
+
"SubagentStop",
|
|
284
|
+
"UserPromptSubmit"
|
|
285
|
+
],
|
|
286
|
+
"extraEvents": [
|
|
287
|
+
"SessionStart"
|
|
288
|
+
],
|
|
289
|
+
"entryTemplate": {
|
|
290
|
+
"hooks": [
|
|
291
|
+
{
|
|
292
|
+
"type": "command",
|
|
293
|
+
"command": "{command}"
|
|
294
|
+
}
|
|
295
|
+
]
|
|
296
|
+
},
|
|
297
|
+
"matcherEvents": {
|
|
298
|
+
"PreToolUse": "*",
|
|
299
|
+
"PostToolUse": "*",
|
|
300
|
+
"PostToolUseFailure": "*"
|
|
301
|
+
},
|
|
302
|
+
"managedHost": "claude"
|
|
303
|
+
},
|
|
304
|
+
"extras": [
|
|
305
|
+
"claude-transcript-narration",
|
|
306
|
+
"transcript-helpers",
|
|
307
|
+
"transcript-coverage",
|
|
308
|
+
"claude-reveal",
|
|
309
|
+
"claude-session-title",
|
|
310
|
+
"install-default"
|
|
311
|
+
],
|
|
312
|
+
"shape": {
|
|
313
|
+
"PreToolUse": [
|
|
314
|
+
"session_id",
|
|
315
|
+
"tool_name",
|
|
316
|
+
"tool_use_id",
|
|
317
|
+
"tool_input"
|
|
318
|
+
],
|
|
319
|
+
"PostToolUse": [
|
|
320
|
+
"session_id",
|
|
321
|
+
"tool_name",
|
|
322
|
+
"tool_use_id",
|
|
323
|
+
"tool_input",
|
|
324
|
+
"tool_response"
|
|
325
|
+
],
|
|
326
|
+
"PostToolUseFailure": [
|
|
327
|
+
"session_id",
|
|
328
|
+
"tool_name",
|
|
329
|
+
"tool_use_id",
|
|
330
|
+
"error"
|
|
331
|
+
],
|
|
332
|
+
"Stop": [
|
|
333
|
+
"session_id"
|
|
334
|
+
],
|
|
335
|
+
"SubagentStop": [
|
|
336
|
+
"session_id",
|
|
337
|
+
"agent_id"
|
|
338
|
+
],
|
|
339
|
+
"UserPromptSubmit": [
|
|
340
|
+
"session_id",
|
|
341
|
+
"prompt"
|
|
342
|
+
]
|
|
343
|
+
}
|
|
344
|
+
}
|