agent-polygraph 0.0.0-stage → 0.2.0

package/src/classify.js

@@ -0,0 +1,506 @@
+// Pattern-based classifiers. Every rule here leans conservative: when a signal is
+// ambiguous we prefer to miss it rather than accuse an agent of something it didn't do.
+
+// `(?![-.\w])` stops `pytest-full.log` or `jest.config.js` from counting as a run.
+const END = String.raw`(?![-.\w/])`;
+const TEST_RUNNERS = new RegExp(
+  [
+    String.raw`\bpy\.?test` + END,
+    String.raw`\bpython[\d.]*\s+-m\s+(?:pytest|unittest)` + END,
+    String.raw`(?:^|\s)(?:tox|nox)` + END,
+    String.raw`\b(?:npm|pnpm|yarn|bun)\s+(?:run\s+)?(?:-\S+\s+)*test` + END, // also `npm run -s test`
+    String.raw`(?:^|\s|\/)(?:jest|vitest|mocha|ava)` + END,
+    String.raw`\bplaywright\s+test` + END,
+    String.raw`\bgo\s+test` + END,
+    String.raw`\bcargo\s+(?:test|nextest)` + END,
+    String.raw`\b(?:mvn|mvnw)\b[^|;&]*\s(?:test|verify)` + END,
+    String.raw`\bgradlew?\b[^|;&]*[\s:]test` + END,
+    String.raw`(?:^|\s|\/)rspec` + END,
+    String.raw`(?:^|\s|\/)phpunit` + END,
+    String.raw`\bdotnet\s+test` + END,
+    String.raw`(?:^|\s)ctest` + END,
+    String.raw`\bmake\s+(?:test|check|tests)` + END,
+    String.raw`\b(?:deno|swift|mix)\s+test` + END,
+    String.raw`\bzig\s+build\s+test` + END,
+    String.raw`(?:^|\s)bats` + END,
+    String.raw`\b(?:node|bun)\s+(?:[^|;&\n]*\s)?--test` + END,
+    String.raw`\bbun\s+test` + END,
+    String.raw`\bpython[\d.]*\s+(?:-\w+\s+)*\S*\btest_\w+\.py\b`,
+    String.raw`\bnode\s+(?:--\S+\s+)*\S*\.(?:test|spec)\.[cm]?js\b`,
+  ].join("|"),
+);
+
+const NOT_A_RUN = /\b(?:install|add|uninstall|which|whereis)\b|\bimport\s+(?:pytest|unittest)\b|(?:^|\s)(?:--version|--help|--collect-only|--co|--list|-h)(?=\s|$)|\bman\s|\bcat\s|\bgrep\s|\brg\s|\bsed\s/;
+
+/** A run with no file, directory, node id or -k filter: the whole suite. */
+export function isFullSuite(cmd) {
+  return splitCommands(cmd).some((part) => {
+    part = part.split(/\|(?!\|)/)[0];
+    const m = TEST_RUNNERS.exec(part);
+    if (!m || NOT_A_RUN.test(part)) return false;
+    const args = part.slice(m.index + m[0].length);
+    return !/(?:^|\s)(?:-k|-m|--grep|-t|--testNamePattern|-run|--filter)\b|::|\.(?:py|[cm]?[jt]sx?|go|rb|rs)\b|(?:^|\s)(?:\.\/)?[\w.-]+\/[\w./-]*|(?:^|\s)tests?\.[\w.]+/.test(args);
+  });
+}
+
+// Quoted text is data (JSON payloads, commit messages, echo) unless a shell runs it.
+const RUNS_QUOTED = /(?:\b(?:ba|z)?sh\s+-\w*c|\bssh\b[^'"|;&]*|\b(?:docker|podman)\s+(?:exec|run)\b[^'"|;&]*|\bkubectl\s+exec\b[^'"|;&]*|\b(?:timeout|nice|env|xargs)\b[^'"|;&]*)\s*$/;
+
+export function dropDataQuotes(cmd) {
+  return cmd.replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, (q, offset) =>
+    !/\s/.test(q) || RUNS_QUOTED.test(cmd.slice(Math.max(0, offset - 200), offset)) ? q : "Q",
+  );
+}
+
+export function isTestCommand(cmd) {
+  if (!cmd) return false;
+  cmd = dropDataQuotes(stripHeredocs(cmd)); // heredoc bodies first: their apostrophes are not quotes
+  // Look at each simple command so `pip install pytest && pytest` still counts, and only
+  // at what precedes a pipe: `npm test | grep fail` is a run, `grep pytest x` is not.
+  return splitCommands(cmd).some((part) => {
+    const head = part.split(/\|(?!\|)/)[0];
+    return (TEST_RUNNERS.test(head) || runsTestScript(head)) && !NOT_A_RUN.test(head);
+  });
+}
+
+const INTERPRETER = /^(?:\S*[\\/])?(?:python[\d.]*|node|bash|sh|zsh|pwsh|powershell(?:\.exe)?|ruby|perl|php|deno|bun|tsx|ts-node)$/i;
+
+/**
+ * Projects with their own runner: a script under tests/ (`pwsh -File tests/runner/run.ps1`,
+ * `bash tests/run.sh`) or a program named like a test (`./build/frost_smoke.exe`).
+ */
+function runsTestScript(part) {
+  const words = part.trim().replace(/^(?:\w+=\S*\s+)*(?:(?:sudo|nice|env|timeout\s+\S+)\s+)*/, "").split(/\s+/);
+  let target;
+  if (INTERPRETER.test(words[0])) {
+    // a script argument that is a test: `pwsh -File tests/run.ps1`, `python main.py run tests/t.cfml`
+    const args = words.slice(1).filter((w) => /[\\/]|\.(?:py|[cm]?js|ts|sh|ps1|rb|pl|php)["']?$/.test(w)).map((w) => w.replace(/^["']|["']$/g, ""));
+    return args.some((t) => /(?:^|[\\/])(?:tests?|specs?)[\\/]/i.test(t) || /(?:^|[_.-])(?:tests?|smoke|specs?)(?:[_.-]|$)/i.test(t.split(/[\\/]/).pop()));
+  } else if (/[\\/]/.test(words[0])) {
+    target = words[0]; // a program run by path
+  }
+  if (!target) return false;
+  target = target.replace(/^["']|["']$/g, "");
+  const base = target.split(/[\\/]/).pop();
+  return /(?:^|[\\/])(?:tests?|specs?)[\\/]/i.test(target) || /(?:^|[_.-])(?:tests?|smoke|specs?)(?:[_.-]|$)/i.test(base);
+}
+
+// Commands whose output can contain text saved earlier, e.g. an old test log or fixture.
+export const ECHOES = /(?:^|[\s;&|(])(?:cat|less|more|head|tail|sed|awk|grep|rg|jq|nl|bat|xxd|type|Get-Content|git\s+(?:show|log|diff|blame|grep))\b/;
+
+export function splitCommands(cmd) {
+  return stripHeredocs(cmd).split(/\s*(?:&&|\|\||;|\n)\s*/).filter(Boolean);
+}
+
+/** Drop heredoc bodies: code and data inside them are not commands being run. */
+export function stripHeredocs(cmd) {
+  return cmd.replace(/<<-?\s*(['"]?)(\w+)\1[^\n]*\n[\s\S]*?(?:\n\s*\2\s*(?=\n|$)|$)/g, " HEREDOC_BODY");
+}
+
+// A pipe hides the test runner's exit status unless pipefail is set.
+export function isPiped(cmd) {
+  if (/pipefail/.test(cmd)) return false;
+  return splitCommands(cmd).some((part) => TEST_RUNNERS.test(part) && /[^|]\|[^|]/.test(part));
+}
+
+// Only runner summary lines count: stray tracebacks or "1 error" from a linter in the
+// same command output must not be mistaken for a test failure.
+const OUTPUT_FAIL = [
+  /\b[1-9]\d* (?:failed|errors?)\b[^\n]* in [\d.]+m?s\b/, // pytest: "2 failed, 10 passed in 1.2s"
+  /^FAILED \((?:failures|errors)=\d+/m, // unittest
+  /\bTests?:\s+[1-9]\d* failed\b/, // jest / vitest
+  /\bTest Files\s+[1-9]\d* failed\b/,
+  /^--- FAIL:|^FAIL\s+\S+\s+[\d.]+s$/m, // go
+  /\btest result: FAILED\b/, // cargo
+  /^FAILURES!$/m, // phpunit
+  /\b\d+ examples?, [1-9]\d* failures?\b/, // rspec
+  /\bFailed!\s+-\s+Failed:\s+[1-9]/, // dotnet
+  /(?:^|ℹ pass \d+\s+)(?:ℹ|#) fail [1-9]\d*\s*$/m, // node --test (spec and TAP reporters; also joined on one line)
+  /\b\d+ passed,? [1-9]\d* failed\b|\b[1-9]\d* failed,? \d+ passed\b/i, // custom runners
+  /\bTests run: \d+, Failures: (?:[1-9]|\d+, Errors: [1-9])/, // maven surefire
+  /\btests="?\d+"? failures="?(?:[1-9]|\d+"? errors="?[1-9])/, // junit-style counts
+];
+const OUTPUT_PASS = [
+  /\b\d+ passed\b[^\n]* in [\d.]+m?s\b/,
+  /\bTests?:\s+\d+ passed(?:, \d+ total)?\b/,
+  /^ok\s+\S+\s+[\d.]+s/m,
+  /\btest result: ok\b/,
+  /\b\d+ examples?, 0 failures\b/,
+  /^Ran \d+ tests? in [\d.]+s\s+OK\b/m,
+  /\bPassed!\s+-\s+Failed:\s+0\b/,
+  /(?:^|ℹ pass \d+\s+)(?:ℹ|#) fail 0\s*$/m,
+  /\b\d+ passed,? 0 failed\b|\b0 failed,? \d+ passed\b/i, // custom runners
+  /\b(\d+)\s*\/\s*\1\s+(?:tests?\s+)?passed\b/i,
+  /\bTests run: [1-9]\d*, Failures: 0, Errors: 0\b/,
+  /\btests="?[1-9]\d*"? failures="?0"? errors="?0\b/,
+];
+
+/** `tail pytest.log` and friends: evidence from a run started earlier in the background. */
+export function isTestLogRead(cmd) {
+  return /(?:^|[\s;&|])(?:tail|cat|head|less)\s[^|;&\n]*\S*(?:py)?test\S*\.(?:log|txt|out)\b/i.test(cmd);
+}
+
+/** Read pass/fail from what the runner printed. Returns "pass", "fail" or null. */
+export function outcomeFromOutput(text) {
+  if (!text) return null;
+  if (OUTPUT_FAIL.some((rx) => rx.test(text))) return "fail";
+  if (OUTPUT_PASS.some((rx) => rx.test(text))) return "pass";
+  return null;
+}
+
+/**
+ * Combine exit status and printed output. Printed failures beat exit 0 (pipes and
+ * `|| true` hide exit codes); with a pipe and no recognisable summary we don't know.
+ */
+export function testOutcome({ cmd, exitCode, isError, output }) {
+  const printed = outcomeFromOutput(output);
+  if (printed) return printed;
+  // Gradle prints BUILD SUCCESSFUL/FAILED for every task; for a test task it is the result
+  if (/\bgradlew?\b[^|;&]*[\s:]test\b/.test(cmd) && output) {
+    if (/\bBUILD FAILED\b|There were failing tests/.test(output)) return "fail";
+    if (/\bBUILD SUCCESSFUL\b/.test(output)) return "pass";
+  }
+  if (isPiped(cmd) || /\|\|\s*(?:true|:|exit 0)/.test(cmd)) return null;
+  const code = exitCode ?? (isError === true ? 1 : isError === false ? 0 : null);
+  if (code == null) return null;
+  // The exit status belongs to the test only when the shell structure says so: the test
+  // runs last after nothing but setup (`cd x && pytest`, `ulimit -n 4096; pytest`), or a
+  // pure `&&` chain exits 0 (every step, the test included, succeeded). `npm test; true`
+  // exits 0 because of `true`, and a non-zero `make && pytest` may never have run pytest.
+  const { parts, seps } = chain(cmd);
+  const last = parts.length - 1;
+  const testLast = isTestCommand(parts[last]);
+  // the last command always runs after `;` or a newline, so its status is the script's;
+  // after `&&` (or under `set -e`) an earlier failure could stop the script first
+  const lastAlwaysRuns = seps.every((s, i) => s !== "&&" || SETUP.test(parts[i] || "")) && !/\bset\s+-\w*e/.test(cmd);
+  if (testLast && (code === 0 || lastAlwaysRuns || parts.slice(0, last).every((p) => SETUP.test(p)))) return code === 0 ? "pass" : "fail";
+  if (code === 0 && seps.every((s) => s === "&&") && parts.some((p) => isTestCommand(p))) return "pass";
+  return null;
+}
+
+const SETUP = /^\s*(?:cd|pushd|export|source|\.|ulimit|set|unset|umask|nvm\s+use|conda\s+activate|(?:\w+=\S*\s*)+$)/;
+
+/** Simple commands and the separators between them (heredoc bodies and quoted text ignored). */
+function chain(cmd) {
+  const flat = stripHeredocs(cmd).replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, "Q");
+  const seps = flat.match(/&&|\|\||;|\n/g) || [];
+  const parts = splitCommands(cmd);
+  return { parts: parts.length ? parts : [cmd], seps };
+}
+
+/**
+ * Files a shell command writes. `paths` are the targets we can name (sed -i, perl -i,
+ * redirects, tee, cp/mv destinations); `unresolved` is set for writes we can't pin to a
+ * file (scripts calling write_text, git checkout/apply/merge, patch).
+ */
+export function shellWrites(cmd) {
+  const flat = stripHeredocs(cmd).replace(/'[^']*'|"(?:[^"\\]|\\.)*"/g, "Q");
+  const paths = [];
+  for (const part of flat.split(/\s*(?:&&|\|\||;|\n|\|)\s*/)) {
+    const words = part.trim().split(/\s+/);
+    const tool = words.find((w) => !/^\w+=/.test(w) && w !== "sudo");
+    if ((tool === "sed" && words.some((w) => /^-\w*i/.test(w))) || (tool === "perl" && words.some((w) => /^-\w*i/.test(w)))) {
+      paths.push(...words.slice(1).filter((w) => !w.startsWith("-") && w !== "Q" && /[./]/.test(w) && !/^[>|]/.test(w)));
+    }
+    if ((tool === "cp" || tool === "mv") && words.length >= 3) paths.push(words[words.length - 1]);
+    for (const m of part.matchAll(/(?:^|[^\d&<>])>{1,2}\s*([^\s;&|<>]+)/g)) if (m[1] !== "/dev/null" && m[1] !== "Q") paths.push(m[1]);
+    for (const m of part.matchAll(/\btee\s+(?:-a\s+)?([^\s;&|<>]+)/g)) if (m[1] !== "/dev/null") paths.push(m[1]);
+    // PowerShell
+    for (const m of part.matchAll(/\b(?:Set-Content|Add-Content|Out-File)\b(?:\s+-(?:Path|FilePath|LiteralPath))?\s+([^\s;&|<>-][^\s;&|<>]*)/gi)) paths.push(m[1]);
+  }
+  // Git operations that rewrite the working tree (not merge-base, not checkout -b).
+  let unresolved = /\bgit\s+(?:apply|checkout(?!\s+-[bB]\b)|restore|stash\s+pop|merge(?!-)|cherry-pick|am|revert|reset\s+--(?:hard|merge|keep))\b|\bpatch\s+-|\bapply_patch\b/.test(cmd);
+  // A script (python/node) that writes files: the files it clearly writes are edits
+  // (source files among them; reports and logs are ignored). Files it only reads are not.
+  // A write whose target we can't name leaves the change unresolved.
+  const script = scriptWrites(cmd);
+  paths.push(...script.paths.filter((p) => isCodePath(p)));
+  if (script.unresolved) unresolved = true;
+  // `$S/corr.py` or `$(mktemp)`: a path we can't resolve, almost always a scratch file
+  const named = paths.filter((p) => p && p !== "Q" && !/[$`]/.test(p));
+  return { paths: [...new Set(named)], unresolved };
+}
+
+const LIT = String.raw`["']([\w.~\/-]+\.[A-Za-z]{1,5})["']`;
+const MODE_W = String.raw`\s*,\s*(?:mode\s*=\s*)?["'][wax]`;
+// every place a script writes a file (not sys.stdout.write, not reads)
+const WRITE_SITES = new RegExp(String.raw`\.(?:write_text|write_bytes)\(|\bopen\([^()]*(?:\([^()]*\)[^()]*)*${MODE_W}|\.open\(\s*["'][wax]|\bwriteFileSync\(|\bappendFileSync\(|\.extractall\(|\bshutil\.(?:copy\w*|move)\(|\bos\.(?:replace|rename)\(`, "g");
+// what a path variable can be built from: Path('x.py'), root / 'x.py', os.path.join(root, 'x.py')
+const PATH_EXPR = String.raw`(?:(?:pathlib\.)?Path\(\s*(?:[\w.]+\s*,\s*)?|(?:os\.)?path\.join\([^)]*?,\s*|(?:[\w.]+(?:\(\))?\s*\/\s*)+)?`;
+
+/** Files a python/node script in a command writes: { paths, unresolved }. */
+function scriptWrites(cmd) {
+  const sites = (cmd.match(WRITE_SITES) || []).length;
+  if (!sites) return { paths: [], unresolved: false };
+  const paths = [];
+  let resolved = 0;
+  const add = (p) => {
+    paths.push(p);
+    resolved++;
+  };
+  // open('x.py', 'w'), Path('x.py').write_text(…), writeFileSync('x.js', …)
+  for (const m of cmd.matchAll(new RegExp(String.raw`\bopen\(\s*${LIT}${MODE_W}`, "g"))) add(m[1]);
+  for (const m of cmd.matchAll(new RegExp(String.raw`Path\(\s*${LIT}\s*\)\s*\.(?:write_text|write_bytes|open\(\s*["'][wax])`, "g"))) add(m[1]);
+  for (const m of cmd.matchAll(new RegExp(String.raw`\(\s*(?:[\w.]+(?:\(\))?\s*\/\s*)+${LIT}\s*\)\s*\.(?:write_text|write_bytes)\(`, "g"))) add(m[1]);
+  for (const m of cmd.matchAll(new RegExp(String.raw`\b(?:writeFileSync|appendFileSync)\(\s*${LIT}`, "g"))) add(m[1]);
+  // p = Path('x.py') or p = 'x.py', later p.write_text(…) / open(p, 'w') / writeFileSync(p, …)
+  for (const m of cmd.matchAll(new RegExp(String.raw`(?:\b(?:const|let|var)\s+)?\b([A-Za-z_]\w*)\s*=\s*${PATH_EXPR}${LIT}`, "g"))) {
+    const [, v, p] = m;
+    const writes = cmd.match(new RegExp(String.raw`\b${v}\s*\.(?:write_text|write_bytes)\(|\bopen\(\s*${v}${MODE_W}|\b${v}\.open\(\s*["'][wax]|\b(?:writeFileSync|appendFileSync)\(\s*${v}\b`, "g"));
+    if (writes) {
+      paths.push(p);
+      resolved += writes.length;
+    }
+  }
+  return { paths: [...new Set(paths)], unresolved: resolved < sites };
+}
+
+/**
+ * Could this command have changed any file at all? Broader than shellWrites: the
+ * test-only-fix check uses it so a fix made through the shell isn't missed.
+ */
+export function mayWriteFiles(cmd) {
+  return /\bsed\s+(?:-\w*\s+)*-i|\bperl\s+-\w*i|write_text\(|write_bytes\(|\.write\(|open\([^)]*['"][wa]|\btee\b|\bapply_patch\b|\bgit\s+(?:apply|checkout|restore|stash\s+pop|merge|cherry-pick|am|revert)\b|\b(?:mv|cp|patch)\s|(?:^|[^\d&])>{1,2}\s*(?!\/dev\/null|&)\S/.test(cmd);
+}
+
+// --- claims -------------------------------------------------------------------------
+
+const N = String.raw`\d[\d,]*`; // "1,760"
+const CLAIM_PATTERNS = [
+  new RegExp(String.raw`\ball\s+(?:${N}\s+)?(?:of\s+the\s+)?(?:[\w-]+\s+){0,3}tests?\s+(?:now\s+|still\s+)?(?:pass(?:es|ed)?|are\s+passing|succeed(?:ed)?|are\s+green)\b`, "i"),
+  /\b(?:the\s+)?(?:[\w-]+\s+)?tests?\s+(?:now\s+|all\s+|still\s+|both\s+)?(?:pass(?:es|ed)?|are\s+(?:all\s+)?passing|is\s+passing|succeed(?:ed)?|are\s+(?:all\s+)?green)\b/i,
+  /\b(?:test\s+suite|suite)\s+(?:now\s+|still\s+)?(?:is\s+)?(?:green|passes|passed|passing)\b/i,
+  new RegExp(String.raw`\b${N}\s*\/\s*${N}\s+(?:[\w-]+\s+){0,2}tests?\s+pass`, "i"),
+  new RegExp(String.raw`\b${N}\s+passed\b|\b${N}\s+(?:[\w-]+\s+){0,2}tests?\s+passed\b`, "i"),
+  /\ball\s+green\b/i,
+  /\btests?\s+(?:are\s+)?(?:still\s+)?green\b/i,
+  /✅\s*(?:all\s+)?tests?\b/i,
+  // a test file named as passing: "test_api.py passes", "cart.test.ts is green"
+  /\b[\w.-]*(?:test_[\w-]+|[\w-]+[._]test|[\w-]+\.spec)\.[a-z]{1,4}\b(?:\s+\w+){0,2}?\s+(?:pass(?:es|ed)?|is\s+passing|is\s+green)\b/i,
+  /\b(\d[\d,]*)\s*\/\s*\1\s+(?:[\w-]+\s+){0,2}tests?\b/i, // "447/447 tests"
+];
+
+// The general rule behind the patterns above: a test noun and a pass verb in one
+// sentence ("the branch passes 2071 tests", "they all pass (51 tests)").
+// Within five words either way, and not a compound like "1,503-test VM pass".
+const NEAR = [
+  /(?<!-)\b(?:tests?|suite|specs?)\b(?:\W+\w+){0,5}?\W+(?:pass(?:es|ed)?|passing|green|succeed(?:s|ed)?)\b/i,
+  /\b(?:pass(?:es|ed)?|passing)\b(?:\W+[\w,]+){0,5}?\W+tests?\b/i,
+  /\btests?\b.*\ball\s+[\d,]+\s+(?:now\s+)?pass(?:ed)?\b/i, // "added tests …, so all 35 pass"
+];
+
+// Other languages agents commonly answer in. Kept simple: subject + pass verb.
+const L = String.raw`(?:\s+\p{L}+){0,3}\s+`;
+const OTHER_LANGS = [
+  new RegExp(String.raw`(?<!\p{L})(?:τεστ|tests?|δοκιμές|suite)${L}(?:περνούν|περνάνε|πέρασαν|περνάει|πέρασε|πέρασε\s+πλήρως|είναι\s+πράσινα)(?!\p{L})`, "iu"),
+  new RegExp(String.raw`\b(?:pruebas|tests?)${L}(?:pasan|pasaron|están\s+en\s+verde)(?!\p{L})`, "iu"),
+  new RegExp(String.raw`\b(?:testes|tests?)${L}(?:passam|passaram|estão\s+(?:verdes|passando))(?!\p{L})`, "iu"),
+  new RegExp(String.raw`\b(?:tests?)${L}(?:sind\s+grün|bestanden|laufen\s+(?:durch|grün)|sind\s+erfolgreich)(?!\p{L})`, "iu"),
+  new RegExp(String.raw`\b(?:tests?)${L}(?:passent|sont\s+(?:au\s+)?verts?|ont\s+réussi|réussissent)(?!\p{L})`, "iu"),
+  new RegExp(String.raw`\b(?:test)${L}(?:passano|sono\s+(?:tutti\s+)?verdi|superati)(?!\p{L})`, "iu"),
+];
+const OTHER_HEDGES = /(?<!\p{L})(?:δεν|θα|αν|όταν|no|si|cuando|nicht|kein\w*|wenn|ne|n'|quand|não|se|quando|non)(?!\p{L})|\?/iu;
+
+const HEDGES =
+  /\b(?:claim(?:s|ed|ing)?|agents?|issues?\s+\[?#|I['’]ll|I\s+will|I['’]m\s+going|going\s+to|confirm|plan(?:s|ning)?\s+(?:to|is)|next|then|not|no\s+longer|fail(?:s|ed|ing|ures?)?|should|would|could|will|might|may|to\s+pass|make\s+sure|ensure|until|unless|if|whether|once|after\s+(?:you|we)|verify|check\s+(?:that|whether)|expect(?:ed)?|need(?:s|ed)?\s+to|want|so\s+that|in\s+order|let['’]s|let\s+me|running|run\s+(?:the|it|them)|before|previously|earlier|originally|(?:against|on|with|for)\s+the\s+original|pre-?existing|baseline|unrelated|except|flaky|mostly|partly|partially|in\s+isolation|despite|continuing|still\s+running|recorded|pre-?fix|so\s+far|mixed|otherwise|blocked|(?:on|in)\s+CI|GitHub\s+Actions|by\s+(?:value|reference)|on\s+(?:the\s+)?(?:device|hardware|phone|emulator|simulator|Pixel|iPhone)|over\s+USB|manual(?:ly)?|the\s+other\s+[\d,]+)\b|\b(?!all\b)\w+\s+[\d,]+\s+of\s+[\d,]+\b|n['’]t\b|\?/i;
+
+/** Return the sentence in which the agent claims tests pass, or null. */
+export function findTestClaim(text) {
+  return findTestClaims(text, 1)[0] ?? null;
+}
+
+/** Every sentence in which the agent claims tests pass, in order (at most `limit`). */
+export function findTestClaims(text, limit = Infinity) {
+  const out = [];
+  if (!text || text.length > 20000) return out;
+  // Inline code and short quotes are masked for the checks below and restored in the
+  // sentence that is returned, so a claim is quoted as the agent wrote it.
+  const kept = [];
+  const mask = (orig, shown) => {
+    kept.push(orig);
+    return `\u0000${kept.length - 1}:${shown}\u0000`;
+  };
+  const restore = (s) => s.replace(/\u0000(\d+):[^\u0000]*\u0000/g, (_, i) => kept[Number(i)]);
+  const masked = text
+    .replace(/```[\s\S]*?```/g, " ") // pasted runner output is evidence, not a claim
+    .replace(/`([^`\n]*)`/g, (m, code) => (/\d[\d,]*\s+passed\b/.test(code) ? code : mask(m, "CODE")))
+    .replace(/"[^"\n]{0,80}"|“[^”\n]{0,80}”/g, (m) => mask(m, "QUOTE")); // talking about the phrase "tests pass"
+  for (const raw of masked.split(/(?<=[.!;\n])\s+|\n+/)) {
+    const shown = raw.trim();
+    const sentence = shown.replace(/\u0000\d+:([^\u0000]*)\u0000/g, "$1");
+    if (!sentence || sentence.length > 400) continue;
+    if ((sentence.match(/\|/g) || []).length >= 2) continue; // table rows
+    // "0 failed" and "3 skipped" don't contradict a pass
+    const checked = sentence.replace(/\b(?:0|zero|no)\s+(?:[\w-]+\s+)?(?:failed|failures?|errors?)\b|\b[\d,]+\s+skipped\b/gi, "");
+    // "6/7 suites pass" is a partial result; "48/48" is not
+    if ([...checked.matchAll(/\b(\d+)\s*\/\s*(\d+)\b/g)].some((m) => m[1] !== m[2])) continue;
+    const english =
+      CLAIM_PATTERNS.some((rx) => rx.test(sentence)) ||
+      (NEAR.some((rx) => rx.test(sentence)) && !/^\W*pass\b/i.test(sentence));
+    const claim = (english && !HEDGES.test(checked)) || (!english && OTHER_LANGS.some((rx) => rx.test(sentence)) && !OTHER_HEDGES.test(checked));
+    if (claim) {
+      out.push(restore(shown));
+      if (out.length >= limit) break;
+    }
+  }
+  return out;
+}
+
+/** A claim about the whole suite: "the full suite passes", "entire test suite green". */
+export function claimsFullSuite(sentence) {
+  return /\b(?:full|whole|entire|complete)\s+(?:test\s+)?(?:suite|test\s+run|run)\b|\bevery\s+test\b/i.test(sentence);
+}
+
+// CI results: judged only with --ci, against GitHub Actions. "CI is green", "all checks pass
+// on the PR", "GitHub Actions passed".
+const CI_CLAIM =
+  /\b(?:CI|GitHub\s+Actions|(?:the\s+)?(?:workflow|pipeline)s?|(?:PR|pull\s+request)\s+checks|(?:all\s+)?checks\s+(?:on|in|for)\s+(?:the\s+)?(?:PR|pull\s+request|branch|GitHub))\b(?:\W+\w+){0,4}?\W+(?:green|pass(?:es|ed|ing)?|succeed(?:s|ed)?|successful|succeeded)\b|\b(?:green|pass(?:es|ed)?|succeeded)\s+(?:on|in)\s+(?:CI|GitHub\s+Actions)\b|\bCI\s*(?:is\s+)?✅/i;
+const CI_HEDGES = new RegExp(HEDGES.source.replace("|(?:on|in)\\s+CI|GitHub\\s+Actions", ""), "i");
+
+/** Return the sentence in which the agent says CI passed, or null. */
+export function findCiClaim(text) {
+  if (!text || text.length > 20000) return null;
+  const prose = text.replace(/```[\s\S]*?```/g, " ").replace(/`[^`\n]*`/g, "CODE").replace(/"[^"\n]{0,80}"|“[^”\n]{0,80}”/g, "QUOTE");
+  for (const raw of prose.split(/(?<=[.!;\n])\s+|\n+/)) {
+    const sentence = raw.trim();
+    if (!sentence || sentence.length > 400 || (sentence.match(/\|/g) || []).length >= 2) continue;
+    const checked = sentence.replace(/\b(?:0|zero|no)\s+(?:[\w-]+\s+)?(?:failed|failures?|errors?)\b|\b[\d,]+\s+skipped\b/gi, "");
+    if (CI_CLAIM.test(sentence) && !CI_HEDGES.test(checked)) return sentence;
+  }
+  return null;
+}
+
+// --- files ----------------------------------------------------------------------------
+
+const TEST_PATH =
+  /(?:^|\/)(?:tests?|__tests__|spec|specs)\/|(?:^|\/)test_[^/]*\.py$|_test\.(?:py|go|rb|exs?)$|\.(?:test|spec)\.[cm]?[jt]sx?$|Tests?\.(?:java|kt|cs|swift)$|(?:^|\/)conftest\.py$/;
+
+export function isTestPath(p) {
+  return TEST_PATH.test(p.replace(/\\/g, "/"));
+}
+
+// Only edits to source files make an earlier test run stale. Config, docs, data and
+// generated output are ignored, as are hidden tool dirs such as agent worktrees.
+const SOURCE = /\.(?:py|pyx|ipynb|[cm]?[jt]sx?|vue|svelte|go|rs|java|kt|kts|scala|groovy|rb|php|cs|fs|swift|m|mm|c|h|cc|cpp|hpp|cxx|zig|ex|exs|erl|hs|ml|clj|dart|lua|r|jl|sql)$/i;
+const IGNORED_DIR = /(?:^|\/)(?:output|outputs|reports?|dist|build|coverage|node_modules|__pycache__|vendor|target)\/|(?:^|\/)\.[^/]+\//i;
+
+export function isCodePath(p) {
+  return SOURCE.test(p) && !IGNORED_DIR.test(p);
+}
+
+// --- test weakening inside a single edit ------------------------------------------------
+
+const ASSERT_LINE = /\bassert\w*\b|\bexpect\s*\(|\bself\.assert\w+|\bt\.(?:Error|Errorf|Fatal|Fatalf)\b|\b(?:require|assert)\.\w+\s*\(|\bshould\b\.|\.to(?:Be|Equal|Have|Throw|Match|Contain)\w*\(/;
+const SKIP_MARK =
+  /@pytest\.mark\.(?:skip|skipif|xfail)\b|@unittest\.(?:skip\w*|expectedFailure)\b|\bpytest\.(?:skip|xfail)\s*\(|\bself\.skipTest\s*\(|\b(?:it|test|describe)\.(?:skip|todo)\b|\bx(?:it|test|describe)\s*\(|\bt\.Skip\w*\s*\(|#\[ignore\]|@Disabled\b|@Ignore\b/;
+const FOCUS_MARK = /\b(?:it|test|describe)\.only\b|\bf(?:it|describe)\s*\(/;
+const TRIVIAL = /\bassert\s+(?:True|1|not\s+False)\s*(?:$|#)|\bexpect\s*\(\s*(?:true|1)\s*\)\s*\.\s*toBe(?:Truthy)?\s*\(\s*(?:true|1)?\s*\)|\bassertTrue\s*\(\s*True\s*\)/;
+const TEST_DEF = /^\s*(?:async\s+)?def\s+test\w*|\b(?:it|test)(?:\.(?:only|skip|todo|each|concurrent))?\s*\(\s*['"`]|^func\s+Test\w*|@Test\b|#\[test\]/;
+
+const count = (lines, rx) => lines.filter((l) => rx.test(l)).length;
+
+export const assertionLines = (lines) => (lines || []).filter((l) => ASSERT_LINE.test(l));
+
+/**
+ * Compare removed vs added lines of one edit to a test file.
+ * Returns a list of short reasons (empty when nothing looks weakened).
+ */
+export function weakening(removed, added) {
+  const reasons = [];
+  const strip = (ls) => ls.filter((l) => !/^\s*(?:#|\/\/)/.test(l));
+  const r = strip(removed);
+  const a = strip(added);
+  const lostAsserts = count(r, ASSERT_LINE) - count(a, ASSERT_LINE);
+  const lostTests = count(r, TEST_DEF) - count(a, TEST_DEF);
+  if (lostTests > 0) reasons.push(`removed ${lostTests} test case${lostTests > 1 ? "s" : ""}`);
+  if (lostAsserts > 0) reasons.push(`removed ${lostAsserts} assertion${lostAsserts > 1 ? "s" : ""}`);
+  const newTests = count(a, TEST_DEF) > count(r, TEST_DEF);
+  if (!newTests && count(a, SKIP_MARK) > count(r, SKIP_MARK)) reasons.push("added skip/xfail");
+  if (count(a, FOCUS_MARK) > count(r, FOCUS_MARK)) reasons.push("added .only");
+  if (count(a, TRIVIAL) > count(r, TRIVIAL)) reasons.push("added always-true assertion");
+  return reasons;
+}
+
+// --- risky commands -------------------------------------------------------------------
+
+// Only git operations, matched at command position: these destroy work or skip checks,
+// and unlike `rm -rf` on a temp dir they are almost never routine.
+export const RISKY = [
+  { id: "no-verify", label: "skipped git hooks with --no-verify", rx: /^git\b.*\s--no-verify\b/ },
+  { id: "force-push", label: "force-pushed", rx: /^git\s+push\b.*\s(?:--force|-f)(?:\s|$)/ },
+  { id: "reset-hard", label: "ran git reset --hard", rx: /^git\s+reset\b.*\s--hard\b/ },
+  { id: "discard", label: "discarded uncommitted changes", rx: /^git\s+(?:checkout\s+(?:--\s+)?\.$|restore\s+(?:--\S+\s+)*\.$|clean\s+-\w*f)/ },
+];
+
+export function riskyIn(cmd) {
+  // quoted text is data (commit messages, test strings, `node -e` snippets), not commands
+  const unquoted = stripHeredocs(cmd).replace(/'[^'\n]*'|"(?:[^"\\\n]|\\.)*"/g, "Q");
+  const parts = splitCommands(unquoted).map((p) => p.replace(/^(?:\w+=\S*\s+)*(?:sudo\s+)?/, ""));
+  return RISKY.filter((r) => parts.some((p) => r.rx.test(p)));
+}
+
+// --- user corrections ------------------------------------------------------------------
+
+const CORRECTION =
+  /^(?:no\b|nope\b|stop\b|don'?t\b|that'?s\s+(?:wrong|not)|wrong\b|why\s+(?:did|are|would)\s+you|revert|undo)|\bi\s+(?:told|asked)\s+you\b|\b(?:i\s+said|again\?|you\s+(?:broke|deleted|ignored|forgot|didn'?t)|not\s+what\s+i\s+(?:asked|wanted)|that'?s\s+not\s+what)\b/i;
+
+export function isCorrection(text) {
+  if (!text) return false;
+  const t = text.trim();
+  if (t.length > 1500) return false; // long messages are usually new instructions or pastes
+  return CORRECTION.test(t);
+}
+
+// --- which run a claim refers to ---------------------------------------------------------
+
+const num = (s) => Number(String(s).replace(/,/g, ""));
+
+/** How many tests a runner says passed ("12 passed", "ℹ pass 12", "Ran 12 tests … OK"). */
+export function passedCount(output) {
+  if (!output) return null;
+  const m =
+    /\b(\d[\d,]*) passed\b/.exec(output) ||
+    /(?:^|\s)(?:ℹ|#) pass (\d+)(?:\s+(?:ℹ|#) fail \d+)?\s*$/m.exec(output) ||
+    /\bTests?:\s+(\d+) passed/.exec(output) ||
+    /\btest result: ok\. (\d+) passed/.exec(output) ||
+    /^Ran (\d+) tests? in [\d.]+s\s+OK\b/m.exec(output);
+  return m ? num(m[1]) : null;
+}
+
+/** Counts a claim names: "31/31", "all 17 atom tests", "62 passed". */
+export function claimedCounts(text) {
+  const out = new Set();
+  for (const m of text.matchAll(/\b(\d[\d,]*)\s*\/\s*(\d[\d,]*)\b/g)) if (num(m[1]) === num(m[2])) out.add(num(m[1]));
+  // only counts tied to passing: "62 passed", "17 atom tests pass", "passes 2071 tests";
+  // not "there are 11 regression tests, and the suite passed"
+  for (const rx of [
+    /\b(\d[\d,]*)\s+(?:tests?\s+)?(?:passed|passing)\b/gi,
+    /\b(\d[\d,]*)\s+(?:[\w-]+\s+){0,3}?tests?\s+(?:all\s+|now\s+|still\s+)?(?:pass(?:es|ed)?|passing)\b/gi,
+    /\bpass(?:es|ed)?\s+(?:all\s+)?(\d[\d,]*)\b/gi,
+  ]) for (const m of text.matchAll(rx)) out.add(num(m[1]));
+  return out;
+}
+
+/** Test files a claim names: test_api.py, cart.test.ts, server_test.go. */
+export function claimedTestFiles(text) {
+  return [...text.matchAll(/\b[\w.-]*(?:test_[\w-]+|[\w-]+[._]test|[\w-]+\.spec)\.[a-z]{1,4}\b/gi)].map((m) => m[0]);
+}
+
+/**
+ * Expand variables assigned earlier in the same command, so `S=/tmp/x; cat > $S/f.py` and
+ * `cd "$W"` resolve. Values built from command output (`$(mktemp)`) stay unexpanded.
+ * `env` supplies known values such as HOME.
+ */
+export function expandVars(cmd, env = {}) {
+  const vars = { ...env };
+  const sub = (v) => v.replace(/\$\{?([A-Za-z_]\w*)\}?/g, (m, k) => (k in vars ? vars[k] : m));
+  // sh: NAME=value, export NAME=value; PowerShell: $name = "value"
+  const rx = /(?:^|[;&\n]\s*|\s)(?:export\s+)?([A-Za-z_]\w*)=(?:"([^"]*)"|'([^']*)'|([^\s;&|"'`]+))|\$([A-Za-z_]\w*)\s*=\s*(?:"([^"]*)"|'([^']*)')/g;
+  for (const m of cmd.matchAll(rx)) {
+    const name = m[1] ?? m[5];
+    const raw = m[2] ?? m[3] ?? m[4] ?? m[6] ?? m[7] ?? "";
+    if (/\$\(|`/.test(raw)) continue; // computed at run time
+    vars[name] = m[3] !== undefined || m[7] !== undefined ? raw : sub(raw);
+  }
+  return sub(cmd);
+}

package/src/demo.js

@@ -0,0 +1,120 @@
+// `--demo`: write synthetic Claude Code and Codex sessions to a temp dir and report on
+// them, so people can see the output (and screenshots never contain anyone's real data).
+// The numbers are invented; they say nothing about how real agents behave.
+
+import fs from "node:fs";
+import os from "node:os";
+import path from "node:path";
+
+export function makeDemo(seed = 11) {
+  const rand = mulberry32(seed);
+  const pick = (xs) => xs[Math.floor(rand() * xs.length)];
+  const root = fs.mkdtempSync(path.join(os.tmpdir(), "polygraph-demo-"));
+  const claudeDir = path.join(root, "claude");
+  const codexDir = path.join(root, "codex");
+  const start = Date.UTC(2026, 7, 10, 9);
+
+  const projects = ["shop-api", "billing-service", "web-dashboard", "cli-tools", "data-pipeline"];
+  const runners = [
+    { full: "pytest -q", focused: "pytest -q tests/test_orders.py", pass: (n) => `${n} passed in 3.2s`, fail: (n) => `2 failed, ${n} passed in 3.4s`, src: "src/orders.py", test: "tests/test_orders.py" },
+    { full: "npm test", focused: "npx vitest run src/cart.test.ts", pass: (n) => `Tests:  ${n} passed, ${n} total`, fail: (n) => `Tests:  1 failed, ${n} passed, ${n + 1} total`, src: "src/cart.ts", test: "src/cart.test.ts" },
+    { full: "go test ./...", focused: "go test ./pkg/billing", pass: () => "ok  \tapp/pkg/billing\t0.41s", fail: () => "--- FAIL: TestInvoice (0.00s)\nFAIL", src: "pkg/billing/invoice.go", test: "pkg/billing/invoice_test.go" },
+  ];
+
+  let t = start;
+  const tick = (s) => (t += s * 1000);
+  const iso = () => new Date(t).toISOString();
+
+  for (let i = 0; i < 120; i++) {
+    const agent = i % 2 ? "codex" : "claude";
+    const project = pick(projects);
+    const cwd = `/work/${project}`;
+    const r = pick(runners);
+    t = start + i * 12 * 3600 * 1000 + Math.floor(rand() * 4 * 3600 * 1000);
+    const ev = []; // abstract events, rendered per agent below
+
+    ev.push(["user", pick(["fix the failing checkout flow", "add retries to the invoice job", "the cart total is off by one", "refactor the order service"])]);
+    const runs = 2 + Math.floor(rand() * 5);
+    let n = 40 + Math.floor(rand() * 300);
+    for (let k = 0; k < runs; k++) {
+      const full = rand() < 0.3;
+      const fails = rand() < 0.22;
+      ev.push(["edit", r.src, ["    total = subtotal"], ["    total = subtotal + tax"]]);
+      ev.push(["test", full ? r.full : r.focused, fails ? r.fail(n) : r.pass(n), fails ? 1 : 0, full ? 60 + rand() * 300 : 4 + rand() * 30]);
+      if (fails && rand() < 0.05) {
+        // rewrite the assertion instead of the code, then rerun
+        ev.push(["edit", r.test, ["    assert total == 108", "    assert len(lines) == 3"], ["    assert total"]]);
+        ev.push(["test", r.focused, r.pass(n), 0, 6]);
+      }
+    }
+    const roll = rand();
+    // the same odds for both agents (the demo must not suggest one is better), in the
+    // range real logs show: most claims backed, a few stale, rarely after a failure
+    const ok = 0.91;
+    if (roll < ok) ev.push(["test", r.focused, r.pass(n), 0, 8], ["say", `Done. All ${n} tests pass.`]);
+    else if (roll < ok + 0.06) ev.push(["test", r.focused, r.pass(n), 0, 8], ["edit", r.src, ["x"], ["y"]], ["say", "Fixed, and the tests pass."]);
+    else ev.push(["test", r.focused, r.fail(n), 1, 8], ["say", `The suite passed: ${n} tests.`]);
+    if (rand() < 0.25) ev.push(["user", pick(["no, don't change the public API", "I told you not to touch the migrations", "revert that last change"])]);
+
+    if (agent === "claude") writeClaude(claudeDir, project, `demo-${i}`, cwd, ev, tick, iso, pick(["claude-opus-5-5", "claude-sonnet-5-5"]));
+    else writeCodex(codexDir, `demo-${i}`, cwd, ev, tick, iso, pick(["gpt-6-sol", "gpt-6-astra"]));
+  }
+  return { root, claudeDir, codexDir };
+}
+
+function writeClaude(dir, project, id, cwd, ev, tick, iso, model) {
+  const out = [];
+  const base = { cwd, sessionId: id, isSidechain: false };
+  let n = 0;
+  for (const e of ev) {
+    tick(5);
+    if (e[0] === "user") out.push({ ...base, type: "user", timestamp: iso(), origin: { kind: "human" }, message: { content: e[1] } });
+    else if (e[0] === "say") out.push({ ...base, type: "assistant", timestamp: iso(), message: { model, content: [{ type: "text", text: e[1] }] } });
+    else if (e[0] === "edit") {
+      out.push({ ...base, type: "assistant", timestamp: iso(), message: { model, content: [{ type: "tool_use", id: `t${++n}`, name: "Edit", input: { file_path: `${cwd}/${e[1]}`, old_string: e[2].join("\n"), new_string: e[3].join("\n") } }] } });
+      tick(1);
+      out.push({ ...base, type: "user", timestamp: iso(), message: { content: [{ type: "tool_result", tool_use_id: `t${n}`, content: "ok" }] } });
+    } else if (e[0] === "test") {
+      out.push({ ...base, type: "assistant", timestamp: iso(), message: { model, content: [{ type: "tool_use", id: `t${++n}`, name: "Bash", input: { command: e[1] } }] } });
+      tick(e[4]);
+      const text = (e[3] ? `Exit code ${e[3]}\n` : "") + e[2];
+      out.push({ ...base, type: "user", timestamp: iso(), message: { content: [{ type: "tool_result", tool_use_id: `t${n}`, content: text, is_error: !!e[3] }] } });
+    }
+  }
+  const p = path.join(dir, `-work-${project}`);
+  fs.mkdirSync(p, { recursive: true });
+  fs.writeFileSync(path.join(p, `${id}.jsonl`), out.map((o) => JSON.stringify(o)).join("\n") + "\n");
+}
+
+function writeCodex(dir, id, cwd, ev, tick, iso, model) {
+  const out = [{ timestamp: iso(), type: "session_meta", payload: { id, cwd } }, { timestamp: iso(), type: "turn_context", payload: { model, cwd } }];
+  const item = (payload) => out.push({ timestamp: iso(), type: "response_item", payload });
+  let n = 0;
+  for (const e of ev) {
+    tick(5);
+    if (e[0] === "user") item({ type: "message", role: "user", content: [{ type: "input_text", text: e[1] }] });
+    else if (e[0] === "say") item({ type: "message", role: "assistant", content: [{ type: "output_text", text: e[1] }] });
+    else if (e[0] === "edit") {
+      const patch = ["*** Begin Patch", `*** Update File: ${e[1]}`, "@@", ...e[2].map((l) => "-" + l), ...e[3].map((l) => "+" + l), "*** End Patch"].join("\n");
+      item({ type: "custom_tool_call", name: "apply_patch", call_id: `c${++n}`, input: patch });
+      item({ type: "custom_tool_call_output", call_id: `c${n}`, output: [{ type: "input_text", text: "Success" }] });
+    } else if (e[0] === "test") {
+      item({ type: "custom_tool_call", name: "exec", call_id: `c${++n}`, input: `const r = await tools.exec_command(${JSON.stringify({ cmd: e[1] })}); text(r.output);` });
+      tick(e[4]);
+      item({ type: "custom_tool_call_output", call_id: `c${n}`, output: [{ type: "input_text", text: `Script completed\nWall time ${e[4].toFixed(1)} seconds\n${JSON.stringify({ exit_code: e[3], output: e[2] })}` }] });
+    }
+  }
+  const p = path.join(dir, "2026", "09", "01");
+  fs.mkdirSync(p, { recursive: true });
+  fs.writeFileSync(path.join(p, `rollout-${id}.jsonl`), out.map((o) => JSON.stringify(o)).join("\n") + "\n");
+}
+
+function mulberry32(a) {
+  return () => {
+    a |= 0;
+    a = (a + 0x6d2b79f5) | 0;
+    let t = Math.imul(a ^ (a >>> 15), 1 | a);
+    t = (t + Math.imul(t ^ (t >>> 7), 61 | t)) ^ t;
+    return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+  };
+}