cocheck 0.4.0 → 0.4.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (81) hide show
  1. package/README.md +17 -3
  2. package/dist/agent-transcripts.js +15 -12
  3. package/dist/attribution/evidence.js +2 -1
  4. package/dist/attribution/reconstruct.js +19 -0
  5. package/dist/attribution/tag.js +3 -3
  6. package/dist/build-info.json +1 -1
  7. package/dist/cli.js +31 -11
  8. package/dist/connector/export-view.js +1 -1
  9. package/dist/coverage.js +5 -3
  10. package/dist/credit-guard.js +59 -54
  11. package/dist/harness-cmd/create.js +76 -0
  12. package/dist/harness-cmd/index.js +19 -0
  13. package/dist/harness-cmd/record.js +327 -0
  14. package/dist/harness-cmd/scrub.js +25 -0
  15. package/dist/harness-cmd/test.js +318 -0
  16. package/dist/harnesses/README.md +5 -0
  17. package/dist/harnesses/claude/cocheck-harness.json +344 -0
  18. package/dist/harnesses/codex/cocheck-harness.json +212 -0
  19. package/dist/harnesses/copilot/cocheck-harness.json +296 -0
  20. package/dist/{copilot-helpers.js → helper-linker.js} +43 -32
  21. package/dist/hook-adapter.js +287 -63
  22. package/dist/hook-cli.js +73 -36
  23. package/dist/hook-journal.js +40 -12
  24. package/dist/hosts/claude.js +23 -159
  25. package/dist/hosts/codex.js +41 -81
  26. package/dist/hosts/compile.js +275 -0
  27. package/dist/hosts/copilot.js +7 -162
  28. package/dist/hosts/extras.js +55 -0
  29. package/dist/hosts/harnesses.js +66 -0
  30. package/dist/hosts/index.js +176 -8
  31. package/dist/hosts/install-block.js +332 -0
  32. package/dist/hosts/manifest.js +372 -0
  33. package/dist/hosts/shared.js +21 -0
  34. package/dist/hosts/tool-classes.js +6 -0
  35. package/dist/hosts/types.js +1 -0
  36. package/dist/install.js +69 -22
  37. package/dist/ledger.js +11 -0
  38. package/dist/normalize.js +33 -9
  39. package/dist/open-pick.js +5 -3
  40. package/dist/patch-secrets.js +11 -11
  41. package/dist/review-server.js +2 -2
  42. package/dist/timeline/attribution.js +2 -2
  43. package/dist/timeline/describe.js +20 -17
  44. package/dist/timeline/load.js +2 -2
  45. package/dist/timeline/rail/assets/review.js +1 -1
  46. package/dist/timeline/rail/model.js +8 -8
  47. package/dist/timeline/read-signal.js +4 -4
  48. package/dist/timeline/render/view-v1.js +2 -13
  49. package/dist/timeline/runs.js +9 -9
  50. package/dist/timeline/timeline.js +8 -8
  51. package/dist/timeline/write-signal.js +4 -3
  52. package/package.json +2 -2
  53. package/public-kit/harness/GUIDE.md +81 -0
  54. package/public-kit/harness/README.md +10 -0
  55. package/public-kit/harness/cases/claude/shell-writes-file.json +120 -0
  56. package/public-kit/harness/cases/codex/apply-patch-writes-files.json +177 -0
  57. package/public-kit/harness/cases/codex/helper-writes-file.json +160 -0
  58. package/public-kit/harness/cases/codex/shell-writes-and-fails.json +110 -0
  59. package/public-kit/harness/cases/copilot/camel-case-patch-and-failure.json +126 -0
  60. package/public-kit/harness/cases/copilot/helper-writes-file.json +209 -0
  61. package/public-kit/harness/cases/copilot/vscode-helper-writes-file.json +236 -0
  62. package/public-kit/harness/examples/toyagent/cases/write-notes.json +142 -0
  63. package/public-kit/harness/examples/toyagent/cocheck-harness.json +47 -0
  64. package/public-kit/harness/examples/toyagent/recordings/payloads.jsonl +8 -0
  65. package/public-kit/harness/harnesses/antigravity/README.md +32 -0
  66. package/public-kit/harness/harnesses/antigravity/cases/failed-edit.json +99 -0
  67. package/public-kit/harness/harnesses/antigravity/cases/helper-writes-file.json +270 -0
  68. package/public-kit/harness/harnesses/antigravity/cases/shell-ok-fail-and-denied.json +211 -0
  69. package/public-kit/harness/harnesses/antigravity/cases/write-then-edit.json +191 -0
  70. package/public-kit/harness/harnesses/antigravity/cocheck-harness.json +62 -0
  71. package/public-kit/harness/harnesses/antigravity/recordings/payloads.jsonl +63 -0
  72. package/public-kit/harness/harnesses/cursor/README.md +40 -0
  73. package/public-kit/harness/harnesses/cursor/cases/editor-write-edit-shell-helper.json +359 -0
  74. package/public-kit/harness/harnesses/cursor/cases/read-then-write-one-call-id.json +159 -0
  75. package/public-kit/harness/harnesses/cursor/cocheck-harness.json +67 -0
  76. package/public-kit/harness/harnesses/cursor/recordings/payloads.jsonl +28 -0
  77. package/public-kit/harness/scaffold/README.md +42 -0
  78. package/public-kit/harness/scaffold/cocheck-harness.json +72 -0
  79. package/public-kit/harness/scaffold/recordings/.gitkeep +0 -0
  80. package/public-kit/harness/spec/v1.md +286 -0
  81. package/scripts/hook-launch.sh +1 -1
@@ -4,6 +4,7 @@
4
4
  // - in script text the call runs (a script file an earlier call wrote or the stored version of it, a heredoc body, a -c/-e script): a string literal outside a comment or
5
5
  // docstring, wherever it sits (`ROOT / "data/x.csv"`, os.path.join, a function argument, open() for reading), unless every use of it is a write target (see `scriptReads`) (inferred).
6
6
  // A file the script only writes (to_csv, save, open in w/a/x), or only mentions in a comment, is never an input.
7
+ import { hasFileInput, isReadTool } from "../credit-guard.js";
7
8
  import { SCRIPT_RUNNERS, callWritesFile, inlineScripts, parseShell, scriptsRun, scriptsWrittenBy, stringsOf } from "./write-signal.js";
8
9
  const WRITE_CALL = /\.\s*(?:save|saveAs|SaveAs|SaveAs2|SaveCopyAs|to_csv|to_excel|to_json|to_pickle|to_parquet|write_text|write_bytes|write|writelines|savefig)\s*\(/g;
9
10
  const OPEN_CALL = /(?<!\w)open\s*\(/g;
@@ -267,7 +268,6 @@ export function scriptReadsUnknown(script) {
267
268
  }
268
269
  return false;
269
270
  }
270
- const WRITE_TOOLS = new Set(["Write", "Edit", "MultiEdit", "NotebookEdit"]);
271
271
  /** Whether the command runs a script file by a plain path: not `-m <module>`, and not a path held in a variable or glob (`$SK/scripts/x.py`), which the task did not write. */
272
272
  function runsFileByPath(words) {
273
273
  const at = words.findIndex((w) => SCRIPT_RUNNERS.test(w.slice(w.lastIndexOf("/") + 1)));
@@ -669,11 +669,11 @@ export function callReads(call, lookup) {
669
669
  if (text.includes(seg.slice(k).join("/")))
670
670
  return true; return false; };
671
671
  const onlyChecked = (f) => checks.some((t) => holds(t, f)) && !holds(rest, f);
672
- if (call.tool === "Read" && typeof call.readPath === "string")
672
+ if (isReadTool(call.tool, call.host) && typeof call.readPath === "string")
673
673
  add(call.readPath, "read");
674
- if (!call.tool || !WRITE_TOOLS.has(call.tool)) {
674
+ if (!call.tool || !hasFileInput(call.tool, call.host)) {
675
675
  for (const f of call.named ?? [])
676
- if (call.tool !== "Read" && !callWritesFile(call, f, lookup) && !onlyChecked(f))
676
+ if (!isReadTool(call.tool, call.host) && !callWritesFile(call, f, lookup) && !onlyChecked(f))
677
677
  add(f, "call");
678
678
  }
679
679
  for (const s of scripts)
@@ -1,13 +1,12 @@
1
1
  // the one adapter from a renderer's View (src/timeline/render/types.ts) to a cocheck.changes/1 level-3 document, at the serve boundary
2
2
  // (src/review-server.ts handleView). The page reads only the document. Envelope allow-list, exactly: engine, file, kind, pending, banner, caveats
3
3
  // (plus the document's own spec, changes, pictures, and `source`, which carries the level). Product-only data (grids, text panes and the page's
4
- // own drawing hints) lives only in `private`: pictures[].private and changes[].private. The words of a change come from the model's own cards and
5
- // row notes (the path that already scrubs paths and secrets), never from the View. No `system`, no document `level`, no `box.side` (a box's side is its picture's).
4
+ // own drawing hints) lives only in `private`: pictures[].private and changes[].private. The words of a change come from the model's own row notes
5
+ // (the path that already scrubs paths and secrets) or the View change's label, never from other View text. No `system`, no document `level`, no `box.side` (a box's side is its picture's).
6
6
  import { checkChanges, CHANGES_SPEC } from "../../connector/changes-v1.js";
7
7
  const SHA = /^[0-9a-f]{64}$/;
8
8
  const CELL = /^[A-Za-z]{1,3}[1-9]\d*(?::[A-Za-z]{1,3}[1-9]\d*)?$/;
9
9
  const NOT_MARKED = "not marked";
10
- const flat = (t) => (t == null ? "" : t.replace(/\s+/g, " ").trim());
11
10
  const num = (v) => typeof v === "number" && Number.isFinite(v);
12
11
  const rectOk = (r) => typeof r === "object" && r !== null && num(r.x) && num(r.y) && num(r.w) && num(r.h);
13
12
  const paneKey = (id) => id.replace(/^(before|after):/, "");
@@ -62,17 +61,7 @@ function anchorOf(byId, idOf, side, rawPane, a) {
62
61
  return a.line !== undefined && Number.isInteger(a.line) && a.line >= 1 ? { picture: pic.id, lines: { from: a.line, to: a.line } } : null;
63
62
  return null;
64
63
  }
65
- const textOf = (s) => (typeof s === "string" && s.trim() !== "" ? { text: s.trim() } : {});
66
- /** A card as one change's words: the full old and new words (the page clips them for the notes column), the verb as the mark, the card's own sentence as `text`. */
67
- function cardWords(c) {
68
- const before = flat(c.full ? c.full.before : c.before), after = flat(c.full ? c.full.after : c.after);
69
- const mark = c.verb === "Added" ? "added" : c.verb === "Removed" ? "removed" : before !== "" && after !== "" ? "changed" : before === "" && after === "" ? "described" : "changed";
70
- return { label: c.place !== "" ? c.place : (c.loc ?? "Change"), mark, ...(before !== "" ? { before } : {}), ...(after !== "" ? { after } : {}), ...textOf(c.text) };
71
- }
72
64
  function wordsFor(vc, w) {
73
- const card = vc.loc === null ? undefined : w.cards.find((c) => c.loc === vc.loc);
74
- if (card !== undefined)
75
- return cardWords(card);
76
65
  const note = vc.loc === null ? undefined : w.rowNotes.find((r) => r.location === vc.loc);
77
66
  if (note !== undefined)
78
67
  return { label: vc.loc, mark: "changed", text: String(note.text) };
@@ -22,7 +22,7 @@
22
22
  // - That holds across turns: the Stop that cut the ledger's turn does not decide it. An agent's calls that run on after a
23
23
  // Stop stay with the request that launched it, not the prompt that wakes the session later.
24
24
  import { pathKit } from "../paths.js";
25
- import { reportsFailureEvents, toolNameOf } from "../hosts/index.js";
25
+ import { reportsFailureEvents, resumesHelper, toolEntryOf, toolNameOf } from "../hosts/index.js";
26
26
  import { isLauncherTool } from "../credit-guard.js";
27
27
  const EVIDENCE_KINDS = new Set(["result", "interrupted", "turn", "blocked", "denied"]);
28
28
  /** An op's `ended` ({at, kind}) when well formed, else null. */
@@ -199,11 +199,11 @@ function markWakes(events, calls) {
199
199
  const launcher = calls.get(w.toolUseId);
200
200
  if (launcher === undefined || launcher.session !== e.session)
201
201
  continue;
202
- if (!(isLauncherTool(launcher.tool) || launcher.tool === "SendMessage" || named.has(launcher.session + "\u0000" + launcher.id)))
202
+ if (!(isLauncherTool(launcher.tool, launcher.host) || resumesHelper(launcher.host, launcher.tool) || named.has(launcher.session + "\u0000" + launcher.id)))
203
203
  continue;
204
204
  const agent = [...calls.values()].filter((c) => c.session === e.session && c.agentId !== null && c.launchId === w.toolUseId).sort((x, y) => (x.startSeq ?? Infinity) - (y.startSeq ?? Infinity) || x.firstOp - y.firstOp)[0];
205
205
  // A SendMessage that resumed an agent is no Agent call, so no agent carries its id; the notification's task id names the agent.
206
- const woken = agent?.agentId ?? (launcher.tool === "SendMessage" ? w.taskId : null);
206
+ const woken = agent?.agentId ?? (resumesHelper(launcher.host, launcher.tool) ? w.taskId : null);
207
207
  wakes.push({ session: e.session, seq: t.seq, at: t.at, toolUseId: w.toolUseId, launcher, agentId: woken, status: w.status, summary: w.summary });
208
208
  }
209
209
  if (wakes.length === 0)
@@ -217,8 +217,8 @@ function markWakes(events, calls) {
217
217
  }
218
218
  // ---- the record, normalised ----
219
219
  /** An Agent or Task call whose input says `run_in_background: true` is a background launch (the meta-file case is read in agent-transcripts.ts). */
220
- function launchedInBackground(tool, input) {
221
- return (tool === "Agent" || tool === "Task") && typeof input === "object" && input !== null && input.run_in_background === true;
220
+ function launchedInBackground(tool, input, host) {
221
+ return toolEntryOf(host, typeof tool === "string" ? tool : null)?.backgroundFlag === true && typeof input === "object" && input !== null && input.run_in_background === true;
222
222
  }
223
223
  export function readRecord(ops, fileStates) {
224
224
  const calls = new Map();
@@ -298,7 +298,7 @@ export function readRecord(ops, fileStates) {
298
298
  closedBy: typeof a.closedBy === "string" ? a.closedBy : null,
299
299
  error: a.closedBy === "failure" && typeof a.error === "string" && a.error !== "" ? a.error : null,
300
300
  errorInferred: a.closedBy === "failure" && a.errorInferred === true,
301
- background: a.background === true || launchedInBackground(tool, a.tool_input ?? a.toolInput ?? a.input),
301
+ background: a.background === true || launchedInBackground(tool, a.tool_input ?? a.toolInput ?? a.input, typeof a.host === "string" ? a.host : null),
302
302
  ...(a.waits === true ? { waits: true } : {}),
303
303
  ended: endedOf(a.ended),
304
304
  narration: a.narration ?? null,
@@ -395,14 +395,14 @@ export function readRecord(ops, fileStates) {
395
395
  resumes.set(key, [...list, send]);
396
396
  };
397
397
  for (const c of calls.values()) {
398
- const to = c.tool === "SendMessage" && typeof c.input === "object" && c.input !== null ? plainId(c.input.to) : null;
398
+ const to = resumesHelper(c.host, c.tool) && typeof c.input === "object" && c.input !== null ? plainId(c.input.to) : null;
399
399
  noteResume(c.session, to, c);
400
400
  }
401
401
  for (const e of events)
402
402
  if (e.type === "prompt")
403
403
  for (const t of e.texts)
404
404
  for (const w of t.wakes ?? [])
405
- if (w.launcher.tool === "SendMessage")
405
+ if (resumesHelper(w.launcher.host, w.launcher.tool))
406
406
  noteResume(w.session, w.agentId, w.launcher);
407
407
  const startedBefore = (a, b) => {
408
408
  if (a.startSeq != null && b.startSeq != null)
@@ -514,7 +514,7 @@ function requestsBySeq(session, segs) {
514
514
  while (j + 1 < wakes.length && wakes[j + 1].seq < at)
515
515
  j++;
516
516
  // Not for a helper's launch or work: those belong to the latest real request before the launching call, never to a notification.
517
- const helperWork = c.agentId !== null || isLauncherTool(c.tool) || c.tool === "SendMessage";
517
+ const helperWork = c.agentId !== null || isLauncherTool(c.tool, c.host) || resumesHelper(c.host, c.tool);
518
518
  if (!helperWork && j >= 0 && wakes[j].seq > (i < 0 ? Number.NEGATIVE_INFINITY : texts[i].t.seq))
519
519
  r = ownerOf.get(wakes[j].launcher) ?? r;
520
520
  r.calls.push(c);
@@ -3,7 +3,7 @@ import { isUriSubject } from "../connector/subject.js";
3
3
  import { subjectDisplay } from "./subject-display.js";
4
4
  import { fromDiffObject } from "../connector/diff-object.js";
5
5
  import { createHash } from "node:crypto";
6
- import { findProfile, profileOf } from "../hosts/index.js";
6
+ import { findProfile, profileOf, resumesHelper } from "../hosts/index.js";
7
7
  import { isSecretPath, secretLine } from "../secret-files.js";
8
8
  import { EMPTY_SHA, secretShasOf } from "../secret-hashes.js";
9
9
  import path from "node:path";
@@ -524,7 +524,7 @@ export function buildTimeline(input) {
524
524
  return { mcp: { server: cut > 0 ? rest.slice(0, cut) : "", tool: cut > 0 ? rest.slice(cut + 2) : rest, inputFields: [...(c.toolLabel?.inputFields ?? [])] } };
525
525
  };
526
526
  const purposeOf = (c) => {
527
- if (!isLauncherTool(c.tool))
527
+ if (!isLauncherTool(c.tool, c.host))
528
528
  return {};
529
529
  const label = c.toolLabel?.fields.description ?? c.toolLabel?.fields.task_name;
530
530
  const raw = typeof label === "string" && label.trim() !== "" ? label : (c.description ?? "");
@@ -533,7 +533,7 @@ export function buildTimeline(input) {
533
533
  };
534
534
  /** A launch's brief from the call's label (the hook scrubbed and cut it); nothing for another tool or a label without one. */
535
535
  const briefOf = (c) => {
536
- if (!isLauncherTool(c.tool) || c.toolLabel === null)
536
+ if (!isLauncherTool(c.tool, c.host) || c.toolLabel === null)
537
537
  return {};
538
538
  const raw = c.toolLabel.fields.prompt ?? c.toolLabel.fields.message;
539
539
  // Encrypted by the host: said so, never shown (records written before the hook dropped it still hold the token in `fields`).
@@ -551,7 +551,7 @@ export function buildTimeline(input) {
551
551
  };
552
552
  /** A SendMessage's recipient, summary and text from the call's label (the hook scrubbed and cut them); nothing for another tool or a label without them. */
553
553
  const sendOf = (c) => {
554
- if (c.tool !== "SendMessage" || c.toolLabel === null)
554
+ if (!resumesHelper(c.host, c.tool) || c.toolLabel === null)
555
555
  return {};
556
556
  const f = c.toolLabel.fields;
557
557
  const one = (k) => (typeof f[k] === "string" && f[k].trim() !== "" ? lineFrom(f[k], { source: "agent-claim", label: SEND_NOTE }).text : null);
@@ -1093,15 +1093,15 @@ export function creditStates(input, R, folder) {
1093
1093
  return false;
1094
1094
  }
1095
1095
  const ev = input.evidence?.get(id) ?? c.ended ?? undefined;
1096
- if (!evidenceMayCredit(ev, { background: runsOnAfterResult(c), closedByOwnEvent: c.closedBy === "post" || c.closedBy === "failure" }, Number.isFinite(at) ? at : null, firstSweepAtOrAfter))
1096
+ if (!evidenceMayCredit(ev, { background: runsOnAfterResult(c, c.host), closedByOwnEvent: c.closedBy === "post" || c.closedBy === "failure" }, Number.isFinite(at) ? at : null, firstSweepAtOrAfter))
1097
1097
  return false;
1098
1098
  // A launch is no suspect at a sweep where a call of its own helper is open
1099
- if (c.waits === true || isWaitTool(c.tool))
1099
+ if (c.waits === true || isWaitTool(c.tool, c.host))
1100
1100
  return false;
1101
- if (!launcherMayCredit(c.tool, s.open.some((o) => R.calls.get(o)?.launchId === id)))
1101
+ if (!launcherMayCredit(c.tool, s.open.some((o) => R.calls.get(o)?.launchId === id), c.host))
1102
1102
  return false;
1103
1103
  const fp = c.toolLabel?.fields.file_path;
1104
- return toolMayCredit(c.tool, typeof fp === "string" ? fp : null, s.file_path, (a, b) => samePath(a, b, false, gKit), gKit);
1104
+ return toolMayCredit(c.tool, typeof fp === "string" ? fp : null, s.file_path, (a, b) => samePath(a, b, false, gKit), gKit, c.host);
1105
1105
  };
1106
1106
  const cannot = input.cannotWrite;
1107
1107
  const guarded = (s, ids) => creditWithEvidence(ids, s.file_path, input.writeEvidence, guard(s), s.step_id ?? null, cannot === undefined ? undefined : (id) => cannot(id, s.file_path, s.step_id ?? null));
@@ -9,13 +9,14 @@
9
9
  // A name inside a read-only command (ls, cat, grep, head, tail, open, file, stat, wc) or in read-only script text is no signal. A path
10
10
  // with a variable in the path text, a glob or `..` in it, or a full path in another folder, is no match. When unsure the answer is "no signal", and the
11
11
  // caller leaves the change joint: a joint change claims no maker, a wrong narrowing claims the wrong one.
12
- const WRITE_TOOLS = new Set(["Write", "Edit", "MultiEdit", "NotebookEdit"]);
13
12
  export const SCRIPT_RUNNERS = /^(python[\d.]*|node|nodejs|ruby|perl|php|deno|bun|Rscript|osascript|tsx|ts-node(?:-esm|-script)?)$/;
14
13
  const SCRIPT_FLAGS = new Set(["-c", "-e", "-E", "-p", "--eval", "--command"]);
15
14
  const DEST_ALL = new Set(["touch"]);
16
15
  const DEST_LAST = new Set(["cp", "mv", "install", "ln", "rsync", "scp"]);
17
16
  /** Whether a path as written in a command or script (`token`) is the file `file` (an absolute path). */
18
17
  import { pathKit } from "../paths.js";
18
+ import { toolEntryOf } from "../hosts/index.js";
19
+ import { hasFileInput } from "../credit-guard.js";
19
20
  function pathIs(token, file) {
20
21
  const kit = pathKit();
21
22
  // A backslash is a separator where the platform says so (Windows), and a quoting character elsewhere.
@@ -329,7 +330,7 @@ const baseOf = (p) => p.slice(p.lastIndexOf("/") + 1);
329
330
  export function scriptsWrittenBy(call) {
330
331
  const out = [];
331
332
  const input = call.input;
332
- if (call.tool === "Write" && input && typeof input === "object" && typeof input.file_path === "string" && typeof input.content === "string")
333
+ if (toolEntryOf(call.host, call.tool)?.class === "write" && hasFileInput(call.tool, call.host) && input && typeof input === "object" && typeof input.file_path === "string" && typeof input.content === "string")
333
334
  out.push({ name: baseOf(input.file_path), text: input.content });
334
335
  const commands = [call.command ?? "", ...(input && typeof input === "object" && typeof input.command === "string" ? [input.command] : [])];
335
336
  for (const text of commands) {
@@ -361,7 +362,7 @@ export function scriptsRun(words) {
361
362
  /** Whether the call's own command or input text shows it writing `file`. `lookup` gives the text of script files the call may run. */
362
363
  export function callWritesFile(call, file, lookup) {
363
364
  const input = call.input;
364
- if (call.tool && WRITE_TOOLS.has(call.tool) && input && typeof input === "object") {
365
+ if (call.tool && hasFileInput(call.tool, call.host) && input && typeof input === "object") {
365
366
  for (const k of ["file_path", "notebook_path", "path"])
366
367
  if (typeof input[k] === "string" && pathIs(input[k], file))
367
368
  return true;
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "cocheck",
3
- "version": "0.4.0",
3
+ "version": "0.4.1",
4
4
  "license": "MIT",
5
5
  "description": "CoCheck shows what AI agents changed in your files, step by step, with a before-and-after review page that says which agent action made each change.",
6
6
  "keywords": [
@@ -57,7 +57,7 @@
57
57
  "prepack": "npm run build"
58
58
  },
59
59
  "dependencies": {
60
- "fast-xml-parser": "^5.11.1",
60
+ "fast-xml-parser": "^5.12.0",
61
61
  "fflate": "^0.8.3"
62
62
  },
63
63
  "devDependencies": {
@@ -0,0 +1,81 @@
1
+ # Add a harness to CoCheck
2
+
3
+ CoCheck shows what an AI agent changed in a folder. It learns that from the hooks a harness runs around each of its tool calls. To add a harness you describe those hooks in a manifest and prove the description with a recorded session. This page takes you from nothing to a harness that passes CoCheck's own test (step 4), tries it on your machine (step 5) and ships it (step 6). You need `cocheck` on your PATH and Node 22.18 or newer. The full reference is `spec/v1.md`.
4
+
5
+ ## 1. Start from the scaffold
6
+
7
+ ```text
8
+ cocheck harness create myagent
9
+ ```
10
+
11
+ This writes a folder `myagent/` with a manifest to fill in, a README and an empty `recordings/` folder. The key (`myagent`) is lowercase letters, digits and dashes, at most 32, and it cannot be a key that already exists.
12
+
13
+ ## 2. Record a real session
14
+
15
+ Run your harness once with CoCheck's recorder in place of the hooks, on a small task that writes one file, for example "read data.csv, then write totals.csv with the column sums".
16
+
17
+ ```text
18
+ cocheck harness record myagent --manifest myagent/cocheck-harness.json --out myagent/recordings -- <the command that starts your harness>
19
+ ```
20
+
21
+ The recorder saves `payloads.jsonl` in `--out`: one line for every hook call, exactly as the harness sent it (`{"event", "argv", "stdin", "ts"}`). Real payloads matter: a manifest written from the harness's documentation alone is wrong about something, and the recording is how you find out what.
22
+
23
+ The command runs in a scratch folder, with the hook config written there only (your own config is never touched), and is stopped after 600 seconds unless you pass `--timeout <seconds>`. The recorder scrubs emails and your home path, and fails rather than write a token-shaped string. It refuses an `--out` that already holds a `payloads.jsonl`.
24
+
25
+ If the harness needs to be run by hand, leave off `-- <command>`: the recorder prints a scratch folder and the environment to export, you run the harness there, then finish with `cocheck harness record --stop <scratch folder>`.
26
+
27
+ If you cannot use the recorder, save each hook call's stdin yourself, one per line, in the same shape.
28
+
29
+ ## 3. Write the manifest
30
+
31
+ Open `myagent/cocheck-harness.json`. Fill in, from your recording:
32
+
33
+ - `events`: each event name the harness sends, mapped to a kind (`prompt`, `tool-pre`, `tool-post`, `tool-failure`, `stop`, `subagent-stop`). And `eventFrom`: where the name is (a payload path, or the hook command's `--event` argument).
34
+ - `fields`: the payload path of the session id, the folder, the tool name, the tool input, the result, the prompt and the closing message. Start with `session`, `toolName` and `toolInput`: that is level 1, and already credits changes to calls.
35
+ - `toolClasses`: every tool the harness reports, and what it does (`write`, `edit`, `shell`, `read`, ...). A tool not listed is unknown: its input is not read, and it is credited only by what changed while its call was open. List a tool that never writes as `never-credited`.
36
+ - `callId`: where the call id is, or `{ "synthesized": true }` when there is none.
37
+ - `failure`: how a failed call looks, to reach level 2.
38
+ - `install`: where the harness's hook config is (`{project}` for a config inside the workspace) and what an entry looks like (`entryTemplates` when some events take another shape).
39
+
40
+ Each step up in level is one more field: see "Levels" and "What degrades without each field" in `spec/v1.md`. A mistake is named by field when CoCheck loads the manifest, for example `myagent/cocheck-harness.json: events.beforeTool: unknown kind "tool-pree"`.
41
+
42
+ ## 4. Write cases and test them
43
+
44
+ A case is a recorded run plus what CoCheck must make of it (`spec/v1.md`, "Cases"). Copy `examples/toyagent/cases/write-notes.json` and replace its recording with lines from your `payloads.jsonl`:
45
+
46
+ - Put each line in `recording` with its `argv` (the `--event` words, if the harness does not name its event in the payload).
47
+ - Write into `files.before` the files that existed before the session, and into `files.writes` each file a call left behind, after which recorded call (`afterEvent`, counting from 1). A recording holds no file contents, so a stand-in is enough.
48
+ - Fill `expect.normalized` with what each event should be (at least `kind` and `session`), and `expect.credit` with each written file, the call it should be credited to, and the basis (`inferred`).
49
+
50
+ Then run:
51
+
52
+ ```text
53
+ cocheck harness test --cases myagent/cases --manifest myagent
54
+ ```
55
+
56
+ CoCheck replays each case through the real hook path in a temporary home (your own data and configs are never touched) and prints one line per case. A pass prints `pass <case>` and the run ends `N cases passed`, exit 0. A failure prints `FAIL <case>: <the first difference>` and exits 1. What you will probably see:
57
+
58
+ | Printed | Meaning and fix |
59
+ | --- | --- |
60
+ | `the manifest did not load: ...` | The manifest has a mistake; the field is named. |
61
+ | `call 3 (afterTool): normalized event kind: expected "tool-post", got "tool-pre"` | `events` maps that name to another kind. |
62
+ | `file notes.txt: expected credit to call:..., but no state was recorded for it` | The file was not seen to change: check `files.writes` and `afterEvent`, and that the writing tool is in `toolClasses`. |
63
+ | `file notes.txt: expected credit to call:a, got call:b` | The credit went to another call: check the call ids (`callId`), or that overlapping calls carry different ids. |
64
+ | `expected credit basis host-reported-edit, got inferred` | A harness in your own folder is held to `inferred` (`spec/v1.md`, "Outsider policy"). |
65
+ | `holds no credited file` | A case must credit a file, or say in its `description` why it has none (use the words "no credit"). |
66
+
67
+ ## 5. Try it from your home folder
68
+
69
+ Copy the folder to `<CoCheck data home>/harnesses/myagent/` (the folder holding `cocheck-harness.json`), then install:
70
+
71
+ ```text
72
+ cocheck install --for myagent
73
+ ```
74
+
75
+ For a harness in your own folder, install prints the config path and the hook entries to add instead of editing your harness's config: add them yourself. Run your harness on a real task, then `cocheck open`. `cocheck status` shows the last event seen from each harness, and names any manifest that was skipped and why.
76
+
77
+ If your harness also runs another harness's hooks (Cursor runs the Claude hooks in `~/.claude/settings.json`), that other harness's CoCheck hook would record your harness's work under its name. Set `capabilities.readsForeignHooks` to `true` and give a `foreignHookFingerprint` that only your harness's payloads match (a key only it sends, such as Cursor's `cursor_version`). The other hook then skips those payloads, but only while your harness's own CoCheck hook is installed for that folder, so nothing goes unrecorded before you add the entries. `cocheck status` prints a line such as `Claude hook skips payloads from Cursor` while it is in force.
78
+
79
+ ## 6. Ship it
80
+
81
+ A harness that works and has its cases can be added to CoCheck by a pull request: the manifest under `harnesses/myagent/` at the repository root (a data folder: no code changes), the cases under `public-kit/harness/cases/myagent/`, and the recording they came from. Once built in, a harness may earn the stronger credit bases an outsider cannot, and its cases run with every release.
@@ -0,0 +1,10 @@
1
+ # CoCheck harness kit
2
+
3
+ This folder is everything you need to add an AI agent harness (a coding agent, a desktop agent, a command-line assistant) to CoCheck. You do not write code: you write one JSON manifest, record a real session, and write cases that say what CoCheck must make of it.
4
+
5
+ - `GUIDE.md` - start here: one page from nothing to a harness that passes `cocheck harness test`, then trying it from your home folder and shipping it by pull request.
6
+ - `spec/v1.md` - the specification (v1): the event CoCheck reads, the levels a harness can join at (L0 to L3), every manifest field, what CoCheck loses when a field is missing, the rules for a harness in your own folder, and the case format.
7
+ - `cases/` - the conformance cases of the harnesses that ship with CoCheck (`claude`, `codex`, `copilot`), built from real recordings and trimmed to what each case needs. `cocheck harness test --cases public-kit/harness/cases/claude` runs one set. Read a case to see what a finished one looks like.
8
+ - `harnesses/` - manifests for harnesses that are not built in, each with its real recording and its cases: `antigravity` (Antigravity CLI), `cursor` (Cursor). Run one with `cocheck harness test --cases public-kit/harness/harnesses/antigravity/cases --manifest public-kit/harness/harnesses/antigravity`.
9
+ - `examples/toyagent/` - a made-up harness whose payloads look like none of the built-in ones: its manifest, its recording and its case. A worked example of an outsider harness; run it with `cocheck harness test --cases public-kit/harness/examples/toyagent/cases --manifest public-kit/harness/examples/toyagent`.
10
+ - `scaffold/` - the starter that `cocheck harness create` copies: a manifest to fill in, a README and an empty `recordings/` folder.
@@ -0,0 +1,120 @@
1
+ {
2
+ "schema": "cocheck-harness-case/1",
3
+ "harness": "claude",
4
+ "description": "Claude Code 2.1.302: a prompt, a shell call that reads data.csv, a shell call whose awk writes totals.csv, then the stop. The whole recording; the session transcript is left out, so no call words are read.",
5
+ "level": "L2",
6
+ "recording": [
7
+ {
8
+ "event": "SessionStart",
9
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"hook_event_name\":\"SessionStart\",\"source\":\"startup\"}",
10
+ "argv": []
11
+ },
12
+ {
13
+ "event": "UserPromptSubmit",
14
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"hook_event_name\":\"UserPromptSubmit\",\"prompt\":\"Before each shell command, write one sentence saying what you are about to do. Step 1: run cat data.csv. Step 2: write totals.csv containing column sums (header a,b,c) using a shell command.\"}",
15
+ "argv": []
16
+ },
17
+ {
18
+ "event": "PreToolUse",
19
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PreToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"cat data.csv\",\"description\":\"Show contents of data.csv\"},\"tool_use_id\":\"toolu_01Dm7osWGJgTmLryibkJFRaM\"}",
20
+ "argv": []
21
+ },
22
+ {
23
+ "event": "PostToolUse",
24
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"cat data.csv\",\"description\":\"Show contents of data.csv\"},\"tool_response\":{\"stdout\":\"a,b,c\\n1,2,3\\n4,5,6\\n7,8,9\",\"stderr\":\"\",\"interrupted\":false,\"isImage\":false,\"noOutputExpected\":false},\"tool_use_id\":\"toolu_01Dm7osWGJgTmLryibkJFRaM\",\"duration_ms\":61}",
25
+ "argv": []
26
+ },
27
+ {
28
+ "event": "PreToolUse",
29
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PreToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"awk -F, 'NR>1{a+=$1;b+=$2;c+=$3} END{print \\\"a,b,c\\\"; print a\\\",\\\"b\\\",\\\"c}' data.csv > totals.csv && cat totals.csv\",\"description\":\"Sum columns into totals.csv and show it\"},\"tool_use_id\":\"toolu_01ETFzqqLkCPkAoHJsF7gBuA\"}",
30
+ "argv": []
31
+ },
32
+ {
33
+ "event": "PostToolUse",
34
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"awk -F, 'NR>1{a+=$1;b+=$2;c+=$3} END{print \\\"a,b,c\\\"; print a\\\",\\\"b\\\",\\\"c}' data.csv > totals.csv && cat totals.csv\",\"description\":\"Sum columns into totals.csv and show it\"},\"tool_response\":{\"stdout\":\"a,b,c\\n12,15,18\",\"stderr\":\"\",\"interrupted\":false,\"isImage\":false,\"noOutputExpected\":false},\"tool_use_id\":\"toolu_01ETFzqqLkCPkAoHJsF7gBuA\",\"duration_ms\":11}",
35
+ "argv": []
36
+ },
37
+ {
38
+ "event": "Stop",
39
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"Stop\",\"stop_hook_active\":false,\"last_assistant_message\":\"I read `data.csv`, which has a header `a,b,c` and three data rows, then wrote `totals.csv` with the sum of each column:\\n\\n```\\na,b,c\\n12,15,18\\n```\",\"background_tasks\":[],\"session_crons\":[]}",
40
+ "argv": []
41
+ },
42
+ {
43
+ "event": "SessionEnd",
44
+ "stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"hook_event_name\":\"SessionEnd\",\"reason\":\"other\"}",
45
+ "argv": []
46
+ }
47
+ ],
48
+ "files": {
49
+ "before": {
50
+ "data.csv": "a,b,c\n1,2,3\n4,5,6\n"
51
+ },
52
+ "writes": [
53
+ {
54
+ "afterEvent": 5,
55
+ "path": "totals.csv",
56
+ "content": "a,b,c\n5,7,9\n"
57
+ }
58
+ ]
59
+ },
60
+ "expect": {
61
+ "normalized": [
62
+ {
63
+ "kind": "other",
64
+ "eventName": "SessionStart",
65
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19"
66
+ },
67
+ {
68
+ "kind": "prompt",
69
+ "eventName": "UserPromptSubmit",
70
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
71
+ "prompt": "Before each shell command, write one sentence saying what you are about to do. Step 1: run cat data.csv. Step 2: write totals.csv containing column sums (header a,b,c) using a shell command."
72
+ },
73
+ {
74
+ "kind": "tool-pre",
75
+ "eventName": "PreToolUse",
76
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
77
+ "toolName": "Bash",
78
+ "callId": "toolu_01Dm7osWGJgTmLryibkJFRaM"
79
+ },
80
+ {
81
+ "kind": "tool-post",
82
+ "eventName": "PostToolUse",
83
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
84
+ "toolName": "Bash",
85
+ "callId": "toolu_01Dm7osWGJgTmLryibkJFRaM"
86
+ },
87
+ {
88
+ "kind": "tool-pre",
89
+ "eventName": "PreToolUse",
90
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
91
+ "toolName": "Bash",
92
+ "callId": "toolu_01ETFzqqLkCPkAoHJsF7gBuA"
93
+ },
94
+ {
95
+ "kind": "tool-post",
96
+ "eventName": "PostToolUse",
97
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
98
+ "toolName": "Bash",
99
+ "callId": "toolu_01ETFzqqLkCPkAoHJsF7gBuA"
100
+ },
101
+ {
102
+ "kind": "stop",
103
+ "eventName": "Stop",
104
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19",
105
+ "lastAssistantMessage": "I read `data.csv`, which has a header `a,b,c` and three data rows, then wrote `totals.csv` with the sum of each column:\n\n```\na,b,c\n12,15,18\n```"
106
+ },
107
+ {
108
+ "kind": "other",
109
+ "eventName": "SessionEnd",
110
+ "session": "b0f5204b-5694-452a-b345-e478329f2f19"
111
+ }
112
+ ],
113
+ "credit": {
114
+ "totals.csv": {
115
+ "creditedTo": "call:toolu_01ETFzqqLkCPkAoHJsF7gBuA",
116
+ "basis": "inferred"
117
+ }
118
+ }
119
+ }
120
+ }