cocheck 0.4.0 → 0.4.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +17 -3
- package/dist/agent-transcripts.js +15 -12
- package/dist/attribution/evidence.js +2 -1
- package/dist/attribution/reconstruct.js +19 -0
- package/dist/attribution/tag.js +3 -3
- package/dist/build-info.json +1 -1
- package/dist/cli.js +31 -11
- package/dist/connector/export-view.js +1 -1
- package/dist/coverage.js +5 -3
- package/dist/credit-guard.js +59 -54
- package/dist/harness-cmd/create.js +76 -0
- package/dist/harness-cmd/index.js +19 -0
- package/dist/harness-cmd/record.js +327 -0
- package/dist/harness-cmd/scrub.js +25 -0
- package/dist/harness-cmd/test.js +318 -0
- package/dist/harnesses/README.md +5 -0
- package/dist/harnesses/claude/cocheck-harness.json +344 -0
- package/dist/harnesses/codex/cocheck-harness.json +212 -0
- package/dist/harnesses/copilot/cocheck-harness.json +296 -0
- package/dist/{copilot-helpers.js → helper-linker.js} +43 -32
- package/dist/hook-adapter.js +287 -63
- package/dist/hook-cli.js +73 -36
- package/dist/hook-journal.js +40 -12
- package/dist/hosts/claude.js +23 -159
- package/dist/hosts/codex.js +41 -81
- package/dist/hosts/compile.js +275 -0
- package/dist/hosts/copilot.js +7 -162
- package/dist/hosts/extras.js +55 -0
- package/dist/hosts/harnesses.js +66 -0
- package/dist/hosts/index.js +176 -8
- package/dist/hosts/install-block.js +332 -0
- package/dist/hosts/manifest.js +372 -0
- package/dist/hosts/shared.js +21 -0
- package/dist/hosts/tool-classes.js +6 -0
- package/dist/hosts/types.js +1 -0
- package/dist/install.js +69 -22
- package/dist/ledger.js +11 -0
- package/dist/normalize.js +33 -9
- package/dist/open-pick.js +5 -3
- package/dist/patch-secrets.js +11 -11
- package/dist/review-server.js +2 -2
- package/dist/timeline/attribution.js +2 -2
- package/dist/timeline/describe.js +20 -17
- package/dist/timeline/load.js +2 -2
- package/dist/timeline/rail/assets/review.js +1 -1
- package/dist/timeline/rail/model.js +8 -8
- package/dist/timeline/read-signal.js +4 -4
- package/dist/timeline/render/view-v1.js +2 -13
- package/dist/timeline/runs.js +9 -9
- package/dist/timeline/timeline.js +8 -8
- package/dist/timeline/write-signal.js +4 -3
- package/package.json +2 -2
- package/public-kit/harness/GUIDE.md +81 -0
- package/public-kit/harness/README.md +10 -0
- package/public-kit/harness/cases/claude/shell-writes-file.json +120 -0
- package/public-kit/harness/cases/codex/apply-patch-writes-files.json +177 -0
- package/public-kit/harness/cases/codex/helper-writes-file.json +160 -0
- package/public-kit/harness/cases/codex/shell-writes-and-fails.json +110 -0
- package/public-kit/harness/cases/copilot/camel-case-patch-and-failure.json +126 -0
- package/public-kit/harness/cases/copilot/helper-writes-file.json +209 -0
- package/public-kit/harness/cases/copilot/vscode-helper-writes-file.json +236 -0
- package/public-kit/harness/examples/toyagent/cases/write-notes.json +142 -0
- package/public-kit/harness/examples/toyagent/cocheck-harness.json +47 -0
- package/public-kit/harness/examples/toyagent/recordings/payloads.jsonl +8 -0
- package/public-kit/harness/harnesses/antigravity/README.md +32 -0
- package/public-kit/harness/harnesses/antigravity/cases/failed-edit.json +99 -0
- package/public-kit/harness/harnesses/antigravity/cases/helper-writes-file.json +270 -0
- package/public-kit/harness/harnesses/antigravity/cases/shell-ok-fail-and-denied.json +211 -0
- package/public-kit/harness/harnesses/antigravity/cases/write-then-edit.json +191 -0
- package/public-kit/harness/harnesses/antigravity/cocheck-harness.json +62 -0
- package/public-kit/harness/harnesses/antigravity/recordings/payloads.jsonl +63 -0
- package/public-kit/harness/harnesses/cursor/README.md +40 -0
- package/public-kit/harness/harnesses/cursor/cases/editor-write-edit-shell-helper.json +359 -0
- package/public-kit/harness/harnesses/cursor/cases/read-then-write-one-call-id.json +159 -0
- package/public-kit/harness/harnesses/cursor/cocheck-harness.json +67 -0
- package/public-kit/harness/harnesses/cursor/recordings/payloads.jsonl +28 -0
- package/public-kit/harness/scaffold/README.md +42 -0
- package/public-kit/harness/scaffold/cocheck-harness.json +72 -0
- package/public-kit/harness/scaffold/recordings/.gitkeep +0 -0
- package/public-kit/harness/spec/v1.md +286 -0
- package/scripts/hook-launch.sh +1 -1
|
@@ -4,6 +4,7 @@
|
|
|
4
4
|
// - in script text the call runs (a script file an earlier call wrote or the stored version of it, a heredoc body, a -c/-e script): a string literal outside a comment or
|
|
5
5
|
// docstring, wherever it sits (`ROOT / "data/x.csv"`, os.path.join, a function argument, open() for reading), unless every use of it is a write target (see `scriptReads`) (inferred).
|
|
6
6
|
// A file the script only writes (to_csv, save, open in w/a/x), or only mentions in a comment, is never an input.
|
|
7
|
+
import { hasFileInput, isReadTool } from "../credit-guard.js";
|
|
7
8
|
import { SCRIPT_RUNNERS, callWritesFile, inlineScripts, parseShell, scriptsRun, scriptsWrittenBy, stringsOf } from "./write-signal.js";
|
|
8
9
|
const WRITE_CALL = /\.\s*(?:save|saveAs|SaveAs|SaveAs2|SaveCopyAs|to_csv|to_excel|to_json|to_pickle|to_parquet|write_text|write_bytes|write|writelines|savefig)\s*\(/g;
|
|
9
10
|
const OPEN_CALL = /(?<!\w)open\s*\(/g;
|
|
@@ -267,7 +268,6 @@ export function scriptReadsUnknown(script) {
|
|
|
267
268
|
}
|
|
268
269
|
return false;
|
|
269
270
|
}
|
|
270
|
-
const WRITE_TOOLS = new Set(["Write", "Edit", "MultiEdit", "NotebookEdit"]);
|
|
271
271
|
/** Whether the command runs a script file by a plain path: not `-m <module>`, and not a path held in a variable or glob (`$SK/scripts/x.py`), which the task did not write. */
|
|
272
272
|
function runsFileByPath(words) {
|
|
273
273
|
const at = words.findIndex((w) => SCRIPT_RUNNERS.test(w.slice(w.lastIndexOf("/") + 1)));
|
|
@@ -669,11 +669,11 @@ export function callReads(call, lookup) {
|
|
|
669
669
|
if (text.includes(seg.slice(k).join("/")))
|
|
670
670
|
return true; return false; };
|
|
671
671
|
const onlyChecked = (f) => checks.some((t) => holds(t, f)) && !holds(rest, f);
|
|
672
|
-
if (call.tool
|
|
672
|
+
if (isReadTool(call.tool, call.host) && typeof call.readPath === "string")
|
|
673
673
|
add(call.readPath, "read");
|
|
674
|
-
if (!call.tool || !
|
|
674
|
+
if (!call.tool || !hasFileInput(call.tool, call.host)) {
|
|
675
675
|
for (const f of call.named ?? [])
|
|
676
|
-
if (call.tool
|
|
676
|
+
if (!isReadTool(call.tool, call.host) && !callWritesFile(call, f, lookup) && !onlyChecked(f))
|
|
677
677
|
add(f, "call");
|
|
678
678
|
}
|
|
679
679
|
for (const s of scripts)
|
|
@@ -1,13 +1,12 @@
|
|
|
1
1
|
// the one adapter from a renderer's View (src/timeline/render/types.ts) to a cocheck.changes/1 level-3 document, at the serve boundary
|
|
2
2
|
// (src/review-server.ts handleView). The page reads only the document. Envelope allow-list, exactly: engine, file, kind, pending, banner, caveats
|
|
3
3
|
// (plus the document's own spec, changes, pictures, and `source`, which carries the level). Product-only data (grids, text panes and the page's
|
|
4
|
-
// own drawing hints) lives only in `private`: pictures[].private and changes[].private. The words of a change come from the model's own
|
|
5
|
-
//
|
|
4
|
+
// own drawing hints) lives only in `private`: pictures[].private and changes[].private. The words of a change come from the model's own row notes
|
|
5
|
+
// (the path that already scrubs paths and secrets) or the View change's label, never from other View text. No `system`, no document `level`, no `box.side` (a box's side is its picture's).
|
|
6
6
|
import { checkChanges, CHANGES_SPEC } from "../../connector/changes-v1.js";
|
|
7
7
|
const SHA = /^[0-9a-f]{64}$/;
|
|
8
8
|
const CELL = /^[A-Za-z]{1,3}[1-9]\d*(?::[A-Za-z]{1,3}[1-9]\d*)?$/;
|
|
9
9
|
const NOT_MARKED = "not marked";
|
|
10
|
-
const flat = (t) => (t == null ? "" : t.replace(/\s+/g, " ").trim());
|
|
11
10
|
const num = (v) => typeof v === "number" && Number.isFinite(v);
|
|
12
11
|
const rectOk = (r) => typeof r === "object" && r !== null && num(r.x) && num(r.y) && num(r.w) && num(r.h);
|
|
13
12
|
const paneKey = (id) => id.replace(/^(before|after):/, "");
|
|
@@ -62,17 +61,7 @@ function anchorOf(byId, idOf, side, rawPane, a) {
|
|
|
62
61
|
return a.line !== undefined && Number.isInteger(a.line) && a.line >= 1 ? { picture: pic.id, lines: { from: a.line, to: a.line } } : null;
|
|
63
62
|
return null;
|
|
64
63
|
}
|
|
65
|
-
const textOf = (s) => (typeof s === "string" && s.trim() !== "" ? { text: s.trim() } : {});
|
|
66
|
-
/** A card as one change's words: the full old and new words (the page clips them for the notes column), the verb as the mark, the card's own sentence as `text`. */
|
|
67
|
-
function cardWords(c) {
|
|
68
|
-
const before = flat(c.full ? c.full.before : c.before), after = flat(c.full ? c.full.after : c.after);
|
|
69
|
-
const mark = c.verb === "Added" ? "added" : c.verb === "Removed" ? "removed" : before !== "" && after !== "" ? "changed" : before === "" && after === "" ? "described" : "changed";
|
|
70
|
-
return { label: c.place !== "" ? c.place : (c.loc ?? "Change"), mark, ...(before !== "" ? { before } : {}), ...(after !== "" ? { after } : {}), ...textOf(c.text) };
|
|
71
|
-
}
|
|
72
64
|
function wordsFor(vc, w) {
|
|
73
|
-
const card = vc.loc === null ? undefined : w.cards.find((c) => c.loc === vc.loc);
|
|
74
|
-
if (card !== undefined)
|
|
75
|
-
return cardWords(card);
|
|
76
65
|
const note = vc.loc === null ? undefined : w.rowNotes.find((r) => r.location === vc.loc);
|
|
77
66
|
if (note !== undefined)
|
|
78
67
|
return { label: vc.loc, mark: "changed", text: String(note.text) };
|
package/dist/timeline/runs.js
CHANGED
|
@@ -22,7 +22,7 @@
|
|
|
22
22
|
// - That holds across turns: the Stop that cut the ledger's turn does not decide it. An agent's calls that run on after a
|
|
23
23
|
// Stop stay with the request that launched it, not the prompt that wakes the session later.
|
|
24
24
|
import { pathKit } from "../paths.js";
|
|
25
|
-
import { reportsFailureEvents, toolNameOf } from "../hosts/index.js";
|
|
25
|
+
import { reportsFailureEvents, resumesHelper, toolEntryOf, toolNameOf } from "../hosts/index.js";
|
|
26
26
|
import { isLauncherTool } from "../credit-guard.js";
|
|
27
27
|
const EVIDENCE_KINDS = new Set(["result", "interrupted", "turn", "blocked", "denied"]);
|
|
28
28
|
/** An op's `ended` ({at, kind}) when well formed, else null. */
|
|
@@ -199,11 +199,11 @@ function markWakes(events, calls) {
|
|
|
199
199
|
const launcher = calls.get(w.toolUseId);
|
|
200
200
|
if (launcher === undefined || launcher.session !== e.session)
|
|
201
201
|
continue;
|
|
202
|
-
if (!(isLauncherTool(launcher.tool) || launcher.tool
|
|
202
|
+
if (!(isLauncherTool(launcher.tool, launcher.host) || resumesHelper(launcher.host, launcher.tool) || named.has(launcher.session + "\u0000" + launcher.id)))
|
|
203
203
|
continue;
|
|
204
204
|
const agent = [...calls.values()].filter((c) => c.session === e.session && c.agentId !== null && c.launchId === w.toolUseId).sort((x, y) => (x.startSeq ?? Infinity) - (y.startSeq ?? Infinity) || x.firstOp - y.firstOp)[0];
|
|
205
205
|
// A SendMessage that resumed an agent is no Agent call, so no agent carries its id; the notification's task id names the agent.
|
|
206
|
-
const woken = agent?.agentId ?? (launcher.tool
|
|
206
|
+
const woken = agent?.agentId ?? (resumesHelper(launcher.host, launcher.tool) ? w.taskId : null);
|
|
207
207
|
wakes.push({ session: e.session, seq: t.seq, at: t.at, toolUseId: w.toolUseId, launcher, agentId: woken, status: w.status, summary: w.summary });
|
|
208
208
|
}
|
|
209
209
|
if (wakes.length === 0)
|
|
@@ -217,8 +217,8 @@ function markWakes(events, calls) {
|
|
|
217
217
|
}
|
|
218
218
|
// ---- the record, normalised ----
|
|
219
219
|
/** An Agent or Task call whose input says `run_in_background: true` is a background launch (the meta-file case is read in agent-transcripts.ts). */
|
|
220
|
-
function launchedInBackground(tool, input) {
|
|
221
|
-
return (tool === "
|
|
220
|
+
function launchedInBackground(tool, input, host) {
|
|
221
|
+
return toolEntryOf(host, typeof tool === "string" ? tool : null)?.backgroundFlag === true && typeof input === "object" && input !== null && input.run_in_background === true;
|
|
222
222
|
}
|
|
223
223
|
export function readRecord(ops, fileStates) {
|
|
224
224
|
const calls = new Map();
|
|
@@ -298,7 +298,7 @@ export function readRecord(ops, fileStates) {
|
|
|
298
298
|
closedBy: typeof a.closedBy === "string" ? a.closedBy : null,
|
|
299
299
|
error: a.closedBy === "failure" && typeof a.error === "string" && a.error !== "" ? a.error : null,
|
|
300
300
|
errorInferred: a.closedBy === "failure" && a.errorInferred === true,
|
|
301
|
-
background: a.background === true || launchedInBackground(tool, a.tool_input ?? a.toolInput ?? a.input),
|
|
301
|
+
background: a.background === true || launchedInBackground(tool, a.tool_input ?? a.toolInput ?? a.input, typeof a.host === "string" ? a.host : null),
|
|
302
302
|
...(a.waits === true ? { waits: true } : {}),
|
|
303
303
|
ended: endedOf(a.ended),
|
|
304
304
|
narration: a.narration ?? null,
|
|
@@ -395,14 +395,14 @@ export function readRecord(ops, fileStates) {
|
|
|
395
395
|
resumes.set(key, [...list, send]);
|
|
396
396
|
};
|
|
397
397
|
for (const c of calls.values()) {
|
|
398
|
-
const to = c.tool
|
|
398
|
+
const to = resumesHelper(c.host, c.tool) && typeof c.input === "object" && c.input !== null ? plainId(c.input.to) : null;
|
|
399
399
|
noteResume(c.session, to, c);
|
|
400
400
|
}
|
|
401
401
|
for (const e of events)
|
|
402
402
|
if (e.type === "prompt")
|
|
403
403
|
for (const t of e.texts)
|
|
404
404
|
for (const w of t.wakes ?? [])
|
|
405
|
-
if (w.launcher.tool
|
|
405
|
+
if (resumesHelper(w.launcher.host, w.launcher.tool))
|
|
406
406
|
noteResume(w.session, w.agentId, w.launcher);
|
|
407
407
|
const startedBefore = (a, b) => {
|
|
408
408
|
if (a.startSeq != null && b.startSeq != null)
|
|
@@ -514,7 +514,7 @@ function requestsBySeq(session, segs) {
|
|
|
514
514
|
while (j + 1 < wakes.length && wakes[j + 1].seq < at)
|
|
515
515
|
j++;
|
|
516
516
|
// Not for a helper's launch or work: those belong to the latest real request before the launching call, never to a notification.
|
|
517
|
-
const helperWork = c.agentId !== null || isLauncherTool(c.tool) || c.tool
|
|
517
|
+
const helperWork = c.agentId !== null || isLauncherTool(c.tool, c.host) || resumesHelper(c.host, c.tool);
|
|
518
518
|
if (!helperWork && j >= 0 && wakes[j].seq > (i < 0 ? Number.NEGATIVE_INFINITY : texts[i].t.seq))
|
|
519
519
|
r = ownerOf.get(wakes[j].launcher) ?? r;
|
|
520
520
|
r.calls.push(c);
|
|
@@ -3,7 +3,7 @@ import { isUriSubject } from "../connector/subject.js";
|
|
|
3
3
|
import { subjectDisplay } from "./subject-display.js";
|
|
4
4
|
import { fromDiffObject } from "../connector/diff-object.js";
|
|
5
5
|
import { createHash } from "node:crypto";
|
|
6
|
-
import { findProfile, profileOf } from "../hosts/index.js";
|
|
6
|
+
import { findProfile, profileOf, resumesHelper } from "../hosts/index.js";
|
|
7
7
|
import { isSecretPath, secretLine } from "../secret-files.js";
|
|
8
8
|
import { EMPTY_SHA, secretShasOf } from "../secret-hashes.js";
|
|
9
9
|
import path from "node:path";
|
|
@@ -524,7 +524,7 @@ export function buildTimeline(input) {
|
|
|
524
524
|
return { mcp: { server: cut > 0 ? rest.slice(0, cut) : "", tool: cut > 0 ? rest.slice(cut + 2) : rest, inputFields: [...(c.toolLabel?.inputFields ?? [])] } };
|
|
525
525
|
};
|
|
526
526
|
const purposeOf = (c) => {
|
|
527
|
-
if (!isLauncherTool(c.tool))
|
|
527
|
+
if (!isLauncherTool(c.tool, c.host))
|
|
528
528
|
return {};
|
|
529
529
|
const label = c.toolLabel?.fields.description ?? c.toolLabel?.fields.task_name;
|
|
530
530
|
const raw = typeof label === "string" && label.trim() !== "" ? label : (c.description ?? "");
|
|
@@ -533,7 +533,7 @@ export function buildTimeline(input) {
|
|
|
533
533
|
};
|
|
534
534
|
/** A launch's brief from the call's label (the hook scrubbed and cut it); nothing for another tool or a label without one. */
|
|
535
535
|
const briefOf = (c) => {
|
|
536
|
-
if (!isLauncherTool(c.tool) || c.toolLabel === null)
|
|
536
|
+
if (!isLauncherTool(c.tool, c.host) || c.toolLabel === null)
|
|
537
537
|
return {};
|
|
538
538
|
const raw = c.toolLabel.fields.prompt ?? c.toolLabel.fields.message;
|
|
539
539
|
// Encrypted by the host: said so, never shown (records written before the hook dropped it still hold the token in `fields`).
|
|
@@ -551,7 +551,7 @@ export function buildTimeline(input) {
|
|
|
551
551
|
};
|
|
552
552
|
/** A SendMessage's recipient, summary and text from the call's label (the hook scrubbed and cut them); nothing for another tool or a label without them. */
|
|
553
553
|
const sendOf = (c) => {
|
|
554
|
-
if (c.tool
|
|
554
|
+
if (!resumesHelper(c.host, c.tool) || c.toolLabel === null)
|
|
555
555
|
return {};
|
|
556
556
|
const f = c.toolLabel.fields;
|
|
557
557
|
const one = (k) => (typeof f[k] === "string" && f[k].trim() !== "" ? lineFrom(f[k], { source: "agent-claim", label: SEND_NOTE }).text : null);
|
|
@@ -1093,15 +1093,15 @@ export function creditStates(input, R, folder) {
|
|
|
1093
1093
|
return false;
|
|
1094
1094
|
}
|
|
1095
1095
|
const ev = input.evidence?.get(id) ?? c.ended ?? undefined;
|
|
1096
|
-
if (!evidenceMayCredit(ev, { background: runsOnAfterResult(c), closedByOwnEvent: c.closedBy === "post" || c.closedBy === "failure" }, Number.isFinite(at) ? at : null, firstSweepAtOrAfter))
|
|
1096
|
+
if (!evidenceMayCredit(ev, { background: runsOnAfterResult(c, c.host), closedByOwnEvent: c.closedBy === "post" || c.closedBy === "failure" }, Number.isFinite(at) ? at : null, firstSweepAtOrAfter))
|
|
1097
1097
|
return false;
|
|
1098
1098
|
// A launch is no suspect at a sweep where a call of its own helper is open
|
|
1099
|
-
if (c.waits === true || isWaitTool(c.tool))
|
|
1099
|
+
if (c.waits === true || isWaitTool(c.tool, c.host))
|
|
1100
1100
|
return false;
|
|
1101
|
-
if (!launcherMayCredit(c.tool, s.open.some((o) => R.calls.get(o)?.launchId === id)))
|
|
1101
|
+
if (!launcherMayCredit(c.tool, s.open.some((o) => R.calls.get(o)?.launchId === id), c.host))
|
|
1102
1102
|
return false;
|
|
1103
1103
|
const fp = c.toolLabel?.fields.file_path;
|
|
1104
|
-
return toolMayCredit(c.tool, typeof fp === "string" ? fp : null, s.file_path, (a, b) => samePath(a, b, false, gKit), gKit);
|
|
1104
|
+
return toolMayCredit(c.tool, typeof fp === "string" ? fp : null, s.file_path, (a, b) => samePath(a, b, false, gKit), gKit, c.host);
|
|
1105
1105
|
};
|
|
1106
1106
|
const cannot = input.cannotWrite;
|
|
1107
1107
|
const guarded = (s, ids) => creditWithEvidence(ids, s.file_path, input.writeEvidence, guard(s), s.step_id ?? null, cannot === undefined ? undefined : (id) => cannot(id, s.file_path, s.step_id ?? null));
|
|
@@ -9,13 +9,14 @@
|
|
|
9
9
|
// A name inside a read-only command (ls, cat, grep, head, tail, open, file, stat, wc) or in read-only script text is no signal. A path
|
|
10
10
|
// with a variable in the path text, a glob or `..` in it, or a full path in another folder, is no match. When unsure the answer is "no signal", and the
|
|
11
11
|
// caller leaves the change joint: a joint change claims no maker, a wrong narrowing claims the wrong one.
|
|
12
|
-
const WRITE_TOOLS = new Set(["Write", "Edit", "MultiEdit", "NotebookEdit"]);
|
|
13
12
|
export const SCRIPT_RUNNERS = /^(python[\d.]*|node|nodejs|ruby|perl|php|deno|bun|Rscript|osascript|tsx|ts-node(?:-esm|-script)?)$/;
|
|
14
13
|
const SCRIPT_FLAGS = new Set(["-c", "-e", "-E", "-p", "--eval", "--command"]);
|
|
15
14
|
const DEST_ALL = new Set(["touch"]);
|
|
16
15
|
const DEST_LAST = new Set(["cp", "mv", "install", "ln", "rsync", "scp"]);
|
|
17
16
|
/** Whether a path as written in a command or script (`token`) is the file `file` (an absolute path). */
|
|
18
17
|
import { pathKit } from "../paths.js";
|
|
18
|
+
import { toolEntryOf } from "../hosts/index.js";
|
|
19
|
+
import { hasFileInput } from "../credit-guard.js";
|
|
19
20
|
function pathIs(token, file) {
|
|
20
21
|
const kit = pathKit();
|
|
21
22
|
// A backslash is a separator where the platform says so (Windows), and a quoting character elsewhere.
|
|
@@ -329,7 +330,7 @@ const baseOf = (p) => p.slice(p.lastIndexOf("/") + 1);
|
|
|
329
330
|
export function scriptsWrittenBy(call) {
|
|
330
331
|
const out = [];
|
|
331
332
|
const input = call.input;
|
|
332
|
-
if (call.tool === "
|
|
333
|
+
if (toolEntryOf(call.host, call.tool)?.class === "write" && hasFileInput(call.tool, call.host) && input && typeof input === "object" && typeof input.file_path === "string" && typeof input.content === "string")
|
|
333
334
|
out.push({ name: baseOf(input.file_path), text: input.content });
|
|
334
335
|
const commands = [call.command ?? "", ...(input && typeof input === "object" && typeof input.command === "string" ? [input.command] : [])];
|
|
335
336
|
for (const text of commands) {
|
|
@@ -361,7 +362,7 @@ export function scriptsRun(words) {
|
|
|
361
362
|
/** Whether the call's own command or input text shows it writing `file`. `lookup` gives the text of script files the call may run. */
|
|
362
363
|
export function callWritesFile(call, file, lookup) {
|
|
363
364
|
const input = call.input;
|
|
364
|
-
if (call.tool &&
|
|
365
|
+
if (call.tool && hasFileInput(call.tool, call.host) && input && typeof input === "object") {
|
|
365
366
|
for (const k of ["file_path", "notebook_path", "path"])
|
|
366
367
|
if (typeof input[k] === "string" && pathIs(input[k], file))
|
|
367
368
|
return true;
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "cocheck",
|
|
3
|
-
"version": "0.4.
|
|
3
|
+
"version": "0.4.1",
|
|
4
4
|
"license": "MIT",
|
|
5
5
|
"description": "CoCheck shows what AI agents changed in your files, step by step, with a before-and-after review page that says which agent action made each change.",
|
|
6
6
|
"keywords": [
|
|
@@ -57,7 +57,7 @@
|
|
|
57
57
|
"prepack": "npm run build"
|
|
58
58
|
},
|
|
59
59
|
"dependencies": {
|
|
60
|
-
"fast-xml-parser": "^5.
|
|
60
|
+
"fast-xml-parser": "^5.12.0",
|
|
61
61
|
"fflate": "^0.8.3"
|
|
62
62
|
},
|
|
63
63
|
"devDependencies": {
|
|
@@ -0,0 +1,81 @@
|
|
|
1
|
+
# Add a harness to CoCheck
|
|
2
|
+
|
|
3
|
+
CoCheck shows what an AI agent changed in a folder. It learns that from the hooks a harness runs around each of its tool calls. To add a harness you describe those hooks in a manifest and prove the description with a recorded session. This page takes you from nothing to a harness that passes CoCheck's own test (step 4), tries it on your machine (step 5) and ships it (step 6). You need `cocheck` on your PATH and Node 22.18 or newer. The full reference is `spec/v1.md`.
|
|
4
|
+
|
|
5
|
+
## 1. Start from the scaffold
|
|
6
|
+
|
|
7
|
+
```text
|
|
8
|
+
cocheck harness create myagent
|
|
9
|
+
```
|
|
10
|
+
|
|
11
|
+
This writes a folder `myagent/` with a manifest to fill in, a README and an empty `recordings/` folder. The key (`myagent`) is lowercase letters, digits and dashes, at most 32, and it cannot be a key that already exists.
|
|
12
|
+
|
|
13
|
+
## 2. Record a real session
|
|
14
|
+
|
|
15
|
+
Run your harness once with CoCheck's recorder in place of the hooks, on a small task that writes one file, for example "read data.csv, then write totals.csv with the column sums".
|
|
16
|
+
|
|
17
|
+
```text
|
|
18
|
+
cocheck harness record myagent --manifest myagent/cocheck-harness.json --out myagent/recordings -- <the command that starts your harness>
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
The recorder saves `payloads.jsonl` in `--out`: one line for every hook call, exactly as the harness sent it (`{"event", "argv", "stdin", "ts"}`). Real payloads matter: a manifest written from the harness's documentation alone is wrong about something, and the recording is how you find out what.
|
|
22
|
+
|
|
23
|
+
The command runs in a scratch folder, with the hook config written there only (your own config is never touched), and is stopped after 600 seconds unless you pass `--timeout <seconds>`. The recorder scrubs emails and your home path, and fails rather than write a token-shaped string. It refuses an `--out` that already holds a `payloads.jsonl`.
|
|
24
|
+
|
|
25
|
+
If the harness needs to be run by hand, leave off `-- <command>`: the recorder prints a scratch folder and the environment to export, you run the harness there, then finish with `cocheck harness record --stop <scratch folder>`.
|
|
26
|
+
|
|
27
|
+
If you cannot use the recorder, save each hook call's stdin yourself, one per line, in the same shape.
|
|
28
|
+
|
|
29
|
+
## 3. Write the manifest
|
|
30
|
+
|
|
31
|
+
Open `myagent/cocheck-harness.json`. Fill in, from your recording:
|
|
32
|
+
|
|
33
|
+
- `events`: each event name the harness sends, mapped to a kind (`prompt`, `tool-pre`, `tool-post`, `tool-failure`, `stop`, `subagent-stop`). And `eventFrom`: where the name is (a payload path, or the hook command's `--event` argument).
|
|
34
|
+
- `fields`: the payload path of the session id, the folder, the tool name, the tool input, the result, the prompt and the closing message. Start with `session`, `toolName` and `toolInput`: that is level 1, and already credits changes to calls.
|
|
35
|
+
- `toolClasses`: every tool the harness reports, and what it does (`write`, `edit`, `shell`, `read`, ...). A tool not listed is unknown: its input is not read, and it is credited only by what changed while its call was open. List a tool that never writes as `never-credited`.
|
|
36
|
+
- `callId`: where the call id is, or `{ "synthesized": true }` when there is none.
|
|
37
|
+
- `failure`: how a failed call looks, to reach level 2.
|
|
38
|
+
- `install`: where the harness's hook config is (`{project}` for a config inside the workspace) and what an entry looks like (`entryTemplates` when some events take another shape).
|
|
39
|
+
|
|
40
|
+
Each step up in level is one more field: see "Levels" and "What degrades without each field" in `spec/v1.md`. A mistake is named by field when CoCheck loads the manifest, for example `myagent/cocheck-harness.json: events.beforeTool: unknown kind "tool-pree"`.
|
|
41
|
+
|
|
42
|
+
## 4. Write cases and test them
|
|
43
|
+
|
|
44
|
+
A case is a recorded run plus what CoCheck must make of it (`spec/v1.md`, "Cases"). Copy `examples/toyagent/cases/write-notes.json` and replace its recording with lines from your `payloads.jsonl`:
|
|
45
|
+
|
|
46
|
+
- Put each line in `recording` with its `argv` (the `--event` words, if the harness does not name its event in the payload).
|
|
47
|
+
- Write into `files.before` the files that existed before the session, and into `files.writes` each file a call left behind, after which recorded call (`afterEvent`, counting from 1). A recording holds no file contents, so a stand-in is enough.
|
|
48
|
+
- Fill `expect.normalized` with what each event should be (at least `kind` and `session`), and `expect.credit` with each written file, the call it should be credited to, and the basis (`inferred`).
|
|
49
|
+
|
|
50
|
+
Then run:
|
|
51
|
+
|
|
52
|
+
```text
|
|
53
|
+
cocheck harness test --cases myagent/cases --manifest myagent
|
|
54
|
+
```
|
|
55
|
+
|
|
56
|
+
CoCheck replays each case through the real hook path in a temporary home (your own data and configs are never touched) and prints one line per case. A pass prints `pass <case>` and the run ends `N cases passed`, exit 0. A failure prints `FAIL <case>: <the first difference>` and exits 1. What you will probably see:
|
|
57
|
+
|
|
58
|
+
| Printed | Meaning and fix |
|
|
59
|
+
| --- | --- |
|
|
60
|
+
| `the manifest did not load: ...` | The manifest has a mistake; the field is named. |
|
|
61
|
+
| `call 3 (afterTool): normalized event kind: expected "tool-post", got "tool-pre"` | `events` maps that name to another kind. |
|
|
62
|
+
| `file notes.txt: expected credit to call:..., but no state was recorded for it` | The file was not seen to change: check `files.writes` and `afterEvent`, and that the writing tool is in `toolClasses`. |
|
|
63
|
+
| `file notes.txt: expected credit to call:a, got call:b` | The credit went to another call: check the call ids (`callId`), or that overlapping calls carry different ids. |
|
|
64
|
+
| `expected credit basis host-reported-edit, got inferred` | A harness in your own folder is held to `inferred` (`spec/v1.md`, "Outsider policy"). |
|
|
65
|
+
| `holds no credited file` | A case must credit a file, or say in its `description` why it has none (use the words "no credit"). |
|
|
66
|
+
|
|
67
|
+
## 5. Try it from your home folder
|
|
68
|
+
|
|
69
|
+
Copy the folder to `<CoCheck data home>/harnesses/myagent/` (the folder holding `cocheck-harness.json`), then install:
|
|
70
|
+
|
|
71
|
+
```text
|
|
72
|
+
cocheck install --for myagent
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
For a harness in your own folder, install prints the config path and the hook entries to add instead of editing your harness's config: add them yourself. Run your harness on a real task, then `cocheck open`. `cocheck status` shows the last event seen from each harness, and names any manifest that was skipped and why.
|
|
76
|
+
|
|
77
|
+
If your harness also runs another harness's hooks (Cursor runs the Claude hooks in `~/.claude/settings.json`), that other harness's CoCheck hook would record your harness's work under its name. Set `capabilities.readsForeignHooks` to `true` and give a `foreignHookFingerprint` that only your harness's payloads match (a key only it sends, such as Cursor's `cursor_version`). The other hook then skips those payloads, but only while your harness's own CoCheck hook is installed for that folder, so nothing goes unrecorded before you add the entries. `cocheck status` prints a line such as `Claude hook skips payloads from Cursor` while it is in force.
|
|
78
|
+
|
|
79
|
+
## 6. Ship it
|
|
80
|
+
|
|
81
|
+
A harness that works and has its cases can be added to CoCheck by a pull request: the manifest under `harnesses/myagent/` at the repository root (a data folder: no code changes), the cases under `public-kit/harness/cases/myagent/`, and the recording they came from. Once built in, a harness may earn the stronger credit bases an outsider cannot, and its cases run with every release.
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
# CoCheck harness kit
|
|
2
|
+
|
|
3
|
+
This folder is everything you need to add an AI agent harness (a coding agent, a desktop agent, a command-line assistant) to CoCheck. You do not write code: you write one JSON manifest, record a real session, and write cases that say what CoCheck must make of it.
|
|
4
|
+
|
|
5
|
+
- `GUIDE.md` - start here: one page from nothing to a harness that passes `cocheck harness test`, then trying it from your home folder and shipping it by pull request.
|
|
6
|
+
- `spec/v1.md` - the specification (v1): the event CoCheck reads, the levels a harness can join at (L0 to L3), every manifest field, what CoCheck loses when a field is missing, the rules for a harness in your own folder, and the case format.
|
|
7
|
+
- `cases/` - the conformance cases of the harnesses that ship with CoCheck (`claude`, `codex`, `copilot`), built from real recordings and trimmed to what each case needs. `cocheck harness test --cases public-kit/harness/cases/claude` runs one set. Read a case to see what a finished one looks like.
|
|
8
|
+
- `harnesses/` - manifests for harnesses that are not built in, each with its real recording and its cases: `antigravity` (Antigravity CLI), `cursor` (Cursor). Run one with `cocheck harness test --cases public-kit/harness/harnesses/antigravity/cases --manifest public-kit/harness/harnesses/antigravity`.
|
|
9
|
+
- `examples/toyagent/` - a made-up harness whose payloads look like none of the built-in ones: its manifest, its recording and its case. A worked example of an outsider harness; run it with `cocheck harness test --cases public-kit/harness/examples/toyagent/cases --manifest public-kit/harness/examples/toyagent`.
|
|
10
|
+
- `scaffold/` - the starter that `cocheck harness create` copies: a manifest to fill in, a README and an empty `recordings/` folder.
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
{
|
|
2
|
+
"schema": "cocheck-harness-case/1",
|
|
3
|
+
"harness": "claude",
|
|
4
|
+
"description": "Claude Code 2.1.302: a prompt, a shell call that reads data.csv, a shell call whose awk writes totals.csv, then the stop. The whole recording; the session transcript is left out, so no call words are read.",
|
|
5
|
+
"level": "L2",
|
|
6
|
+
"recording": [
|
|
7
|
+
{
|
|
8
|
+
"event": "SessionStart",
|
|
9
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"hook_event_name\":\"SessionStart\",\"source\":\"startup\"}",
|
|
10
|
+
"argv": []
|
|
11
|
+
},
|
|
12
|
+
{
|
|
13
|
+
"event": "UserPromptSubmit",
|
|
14
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"hook_event_name\":\"UserPromptSubmit\",\"prompt\":\"Before each shell command, write one sentence saying what you are about to do. Step 1: run cat data.csv. Step 2: write totals.csv containing column sums (header a,b,c) using a shell command.\"}",
|
|
15
|
+
"argv": []
|
|
16
|
+
},
|
|
17
|
+
{
|
|
18
|
+
"event": "PreToolUse",
|
|
19
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PreToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"cat data.csv\",\"description\":\"Show contents of data.csv\"},\"tool_use_id\":\"toolu_01Dm7osWGJgTmLryibkJFRaM\"}",
|
|
20
|
+
"argv": []
|
|
21
|
+
},
|
|
22
|
+
{
|
|
23
|
+
"event": "PostToolUse",
|
|
24
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"cat data.csv\",\"description\":\"Show contents of data.csv\"},\"tool_response\":{\"stdout\":\"a,b,c\\n1,2,3\\n4,5,6\\n7,8,9\",\"stderr\":\"\",\"interrupted\":false,\"isImage\":false,\"noOutputExpected\":false},\"tool_use_id\":\"toolu_01Dm7osWGJgTmLryibkJFRaM\",\"duration_ms\":61}",
|
|
25
|
+
"argv": []
|
|
26
|
+
},
|
|
27
|
+
{
|
|
28
|
+
"event": "PreToolUse",
|
|
29
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PreToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"awk -F, 'NR>1{a+=$1;b+=$2;c+=$3} END{print \\\"a,b,c\\\"; print a\\\",\\\"b\\\",\\\"c}' data.csv > totals.csv && cat totals.csv\",\"description\":\"Sum columns into totals.csv and show it\"},\"tool_use_id\":\"toolu_01ETFzqqLkCPkAoHJsF7gBuA\"}",
|
|
30
|
+
"argv": []
|
|
31
|
+
},
|
|
32
|
+
{
|
|
33
|
+
"event": "PostToolUse",
|
|
34
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"PostToolUse\",\"tool_name\":\"Bash\",\"tool_input\":{\"command\":\"awk -F, 'NR>1{a+=$1;b+=$2;c+=$3} END{print \\\"a,b,c\\\"; print a\\\",\\\"b\\\",\\\"c}' data.csv > totals.csv && cat totals.csv\",\"description\":\"Sum columns into totals.csv and show it\"},\"tool_response\":{\"stdout\":\"a,b,c\\n12,15,18\",\"stderr\":\"\",\"interrupted\":false,\"isImage\":false,\"noOutputExpected\":false},\"tool_use_id\":\"toolu_01ETFzqqLkCPkAoHJsF7gBuA\",\"duration_ms\":11}",
|
|
35
|
+
"argv": []
|
|
36
|
+
},
|
|
37
|
+
{
|
|
38
|
+
"event": "Stop",
|
|
39
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"permission_mode\":\"bypassPermissions\",\"effort\":{\"level\":\"medium\"},\"hook_event_name\":\"Stop\",\"stop_hook_active\":false,\"last_assistant_message\":\"I read `data.csv`, which has a header `a,b,c` and three data rows, then wrote `totals.csv` with the sum of each column:\\n\\n```\\na,b,c\\n12,15,18\\n```\",\"background_tasks\":[],\"session_crons\":[]}",
|
|
40
|
+
"argv": []
|
|
41
|
+
},
|
|
42
|
+
{
|
|
43
|
+
"event": "SessionEnd",
|
|
44
|
+
"stdin": "{\"session_id\":\"b0f5204b-5694-452a-b345-e478329f2f19\",\"cwd\":\"<folder>\",\"prompt_id\":\"d1bf2aeb-3945-471b-81c7-b6d7123b4d9e\",\"hook_event_name\":\"SessionEnd\",\"reason\":\"other\"}",
|
|
45
|
+
"argv": []
|
|
46
|
+
}
|
|
47
|
+
],
|
|
48
|
+
"files": {
|
|
49
|
+
"before": {
|
|
50
|
+
"data.csv": "a,b,c\n1,2,3\n4,5,6\n"
|
|
51
|
+
},
|
|
52
|
+
"writes": [
|
|
53
|
+
{
|
|
54
|
+
"afterEvent": 5,
|
|
55
|
+
"path": "totals.csv",
|
|
56
|
+
"content": "a,b,c\n5,7,9\n"
|
|
57
|
+
}
|
|
58
|
+
]
|
|
59
|
+
},
|
|
60
|
+
"expect": {
|
|
61
|
+
"normalized": [
|
|
62
|
+
{
|
|
63
|
+
"kind": "other",
|
|
64
|
+
"eventName": "SessionStart",
|
|
65
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19"
|
|
66
|
+
},
|
|
67
|
+
{
|
|
68
|
+
"kind": "prompt",
|
|
69
|
+
"eventName": "UserPromptSubmit",
|
|
70
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
71
|
+
"prompt": "Before each shell command, write one sentence saying what you are about to do. Step 1: run cat data.csv. Step 2: write totals.csv containing column sums (header a,b,c) using a shell command."
|
|
72
|
+
},
|
|
73
|
+
{
|
|
74
|
+
"kind": "tool-pre",
|
|
75
|
+
"eventName": "PreToolUse",
|
|
76
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
77
|
+
"toolName": "Bash",
|
|
78
|
+
"callId": "toolu_01Dm7osWGJgTmLryibkJFRaM"
|
|
79
|
+
},
|
|
80
|
+
{
|
|
81
|
+
"kind": "tool-post",
|
|
82
|
+
"eventName": "PostToolUse",
|
|
83
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
84
|
+
"toolName": "Bash",
|
|
85
|
+
"callId": "toolu_01Dm7osWGJgTmLryibkJFRaM"
|
|
86
|
+
},
|
|
87
|
+
{
|
|
88
|
+
"kind": "tool-pre",
|
|
89
|
+
"eventName": "PreToolUse",
|
|
90
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
91
|
+
"toolName": "Bash",
|
|
92
|
+
"callId": "toolu_01ETFzqqLkCPkAoHJsF7gBuA"
|
|
93
|
+
},
|
|
94
|
+
{
|
|
95
|
+
"kind": "tool-post",
|
|
96
|
+
"eventName": "PostToolUse",
|
|
97
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
98
|
+
"toolName": "Bash",
|
|
99
|
+
"callId": "toolu_01ETFzqqLkCPkAoHJsF7gBuA"
|
|
100
|
+
},
|
|
101
|
+
{
|
|
102
|
+
"kind": "stop",
|
|
103
|
+
"eventName": "Stop",
|
|
104
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19",
|
|
105
|
+
"lastAssistantMessage": "I read `data.csv`, which has a header `a,b,c` and three data rows, then wrote `totals.csv` with the sum of each column:\n\n```\na,b,c\n12,15,18\n```"
|
|
106
|
+
},
|
|
107
|
+
{
|
|
108
|
+
"kind": "other",
|
|
109
|
+
"eventName": "SessionEnd",
|
|
110
|
+
"session": "b0f5204b-5694-452a-b345-e478329f2f19"
|
|
111
|
+
}
|
|
112
|
+
],
|
|
113
|
+
"credit": {
|
|
114
|
+
"totals.csv": {
|
|
115
|
+
"creditedTo": "call:toolu_01ETFzqqLkCPkAoHJsF7gBuA",
|
|
116
|
+
"basis": "inferred"
|
|
117
|
+
}
|
|
118
|
+
}
|
|
119
|
+
}
|
|
120
|
+
}
|