agent-polygraph 0.0.0-stage → 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +177 -2
- package/bin/polygraph.js +261 -0
- package/package.json +38 -4
- package/src/analyze.js +611 -0
- package/src/ci.js +204 -0
- package/src/classify.js +506 -0
- package/src/demo.js +120 -0
- package/src/guard.js +216 -0
- package/src/jsonl.js +23 -0
- package/src/record.js +238 -0
- package/src/report/format.js +87 -0
- package/src/report/html.js +101 -0
- package/src/report/terminal.js +160 -0
- package/src/scan.js +60 -0
- package/src/shell/record.sh +61 -0
- package/src/sources/claude.js +300 -0
- package/src/sources/codex.js +476 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Eleftherios Chondros
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,178 @@
|
|
|
1
|
-
#
|
|
1
|
+
# agent-polygraph
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
[](https://github.com/EleChon-pixel/agent-polygraph/actions/workflows/test.yml)
|
|
4
|
+
|
|
5
|
+
Your coding agent said "all tests pass". Did a test pass after its last edit?
|
|
6
|
+
|
|
7
|
+
agent-polygraph reads the Claude Code and Codex session logs already on your machine and checks every "tests pass" claim against what ran. No setup, no API key, nothing uploaded.
|
|
8
|
+
|
|
9
|
+
```sh
|
|
10
|
+
npx agent-polygraph # your own sessions
|
|
11
|
+
npx agent-polygraph --demo # generated sample sessions
|
|
12
|
+
```
|
|
13
|
+
|
|
14
|
+
<img src="docs/guard.gif" alt="guard stops a coding agent that claims tests pass without rerunning them" width="760">
|
|
15
|
+
|
|
16
|
+
> This is a beta. It has been tested on my own logs and on public Claude Code sessions ([evaluation](docs/evaluation.md)). If a verdict looks wrong, please [open an issue](https://github.com/EleChon-pixel/agent-polygraph/issues/new/choose) with the redacted example from the report. Each one becomes a regression test.
|
|
17
|
+
|
|
18
|
+
My own sessions, July to October 2026:
|
|
19
|
+
|
|
20
|
+
```
|
|
21
|
+
agent-polygraph 32 working sessions · Jul 30 – Oct 31, 2026 · 11.0s
|
|
22
|
+
|
|
23
|
+
"Tests pass" claims was there a passing run after the last code edit?
|
|
24
|
+
Codex Claude Code All
|
|
25
|
+
Backed by a fresh passing run 243 80 323
|
|
26
|
+
No test run in the session 0 0 0
|
|
27
|
+
Code edited after the last run 15 5 20
|
|
28
|
+
Last run had failed 1 0 1
|
|
29
|
+
Couldn't tell 15 17 32
|
|
30
|
+
…test result unreadable 3 1 4
|
|
31
|
+
…unresolved file change after the run 11 14 25
|
|
32
|
+
…other 1 2 3
|
|
33
|
+
Receipt rate 94% 94% 94%
|
|
34
|
+
|
|
35
|
+
Test integrity worth a second look, not proof of cheating
|
|
36
|
+
Failing test "fixed" by editing only tests 0 0 0
|
|
37
|
+
Edits that weakened an existing test 3 1 4
|
|
38
|
+
|
|
39
|
+
Where the time went
|
|
40
|
+
Test runs 851 976 1827
|
|
41
|
+
Time spent in tests 3.1h 8.7h 12h
|
|
42
|
+
Full-suite runs 325 212 537
|
|
43
|
+
...
|
|
44
|
+
```
|
|
45
|
+
|
|
46
|
+
Under the tables it prints the latest examples: what the agent claimed, the last test command and its result, what was edited afterwards, and the session file, so you can check it yourself.
|
|
47
|
+
|
|
48
|
+
## Share your numbers
|
|
49
|
+
|
|
50
|
+
```sh
|
|
51
|
+
npx agent-polygraph --html
|
|
52
|
+
```
|
|
53
|
+
|
|
54
|
+
This writes `polygraph.html`, a self-contained card with numbers only. Prompts, code, commands and file paths are left out.
|
|
55
|
+
|
|
56
|
+
<img src="docs/card.png" alt="agent-polygraph report card from my own sessions" width="600">
|
|
57
|
+
|
|
58
|
+
## Stop unbacked claims as they happen
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
npx agent-polygraph guard --install # this project (.claude/settings.json, .codex/hooks.json)
|
|
62
|
+
npx agent-polygraph guard --install --global # every project (~/.claude, ~/.codex)
|
|
63
|
+
npx agent-polygraph guard --uninstall
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
`guard` is a Stop hook that runs the same checks on the live session. When the agent's final message says tests pass and the session doesn't back it up, the agent is sent back once:
|
|
67
|
+
|
|
68
|
+
> agent-polygraph: your final message says "All tests pass.", but you edited calc.js after the last test run. Run the relevant tests now and report the actual result. If they can't be run, say so instead of claiming they pass.
|
|
69
|
+
|
|
70
|
+
It blocks at most once per stop, looks only at the final message (so a claim the agent already corrected doesn't count), stays quiet when it can't tell, and never fails the turn because of its own errors. Install it globally (`npm i -g agent-polygraph`) so the hook doesn't start `npx` on every stop.
|
|
71
|
+
|
|
72
|
+
I tested it with both agents by having them edit code after a test run and then claim the tests pass. Both were stopped, reran `npm test` and reported the failure. Codex needs one extra step: it only runs hooks you have approved, and only reads a project's `.codex/` folder if you trust it. After installing, open `codex` in the project once, trust the folder and approve the hook.
|
|
73
|
+
|
|
74
|
+
## Evidence the logs don't have
|
|
75
|
+
|
|
76
|
+
Some things never show up in the logs: tests you ran yourself in another terminal, CI results, and files changed by a script that builds their names at run time. Two optional extras cover them.
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
npx agent-polygraph record --install # this project; add --global for all of them
|
|
80
|
+
npx agent-polygraph record --install --shell # your own terminals
|
|
81
|
+
npx agent-polygraph --ci # GitHub Actions (online)
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
`record --install` adds a PostToolUse hook to Claude Code and Codex. After a test command, or a command that may write files, it saves a snapshot of the working tree and the test runner's full result. The snapshot is a git tree built with a temporary index and a separate object store, so your repository, its index and its history are never touched. Ignored files are left out. If the logs can't tell whether a script changed your code after the last test run, the two snapshots answer it, and a changed source file is named in the report. Everything stays in `~/.agent-polygraph` (or `$AGENT_POLYGRAPH_HOME`). It applies to sessions started after you install it.
|
|
85
|
+
|
|
86
|
+
`record --install --shell` adds one line to `~/.bashrc` and `~/.zshrc`. From then on, test commands you run in an interactive bash or zsh terminal are logged with their folder, start and end time and exit code. Output is never saved. If you ran the tests yourself in the session's folder after the agent's last edit, that run counts for the agent's claim. `record --uninstall --shell` removes the line.
|
|
87
|
+
|
|
88
|
+
`--ci` goes online. Through your [`gh`](https://cli.github.com) login, it asks the GitHub API for the workflow runs of the repositories your sessions pushed to. The request contains the repository name and a date range, nothing from your sessions. A test workflow on a pushed commit counts as a test run of the code as it was at the push, so later edits still make it stale. Runs are matched to pushes by commit id, which has to appear in the session (in the push output, or from `git commit` or `git log` before it). Branch names and timing alone are not enough. With `--ci`, claims about CI itself ("CI is green", "all checks on the PR pass") are checked too. Workflows with no job or step named like a test, and runs GitHub never started, are ignored.
|
|
89
|
+
|
|
90
|
+
The report shows how many claims each of these settled.
|
|
91
|
+
|
|
92
|
+
## What it checks
|
|
93
|
+
|
|
94
|
+
| Finding | How it's decided |
|
|
95
|
+
|---|---|
|
|
96
|
+
| Backed claim | The agent said tests pass and the run it refers to passed. That is the run with the count or test file the claim names; for "the full suite passes", the latest run in that folder about as large as the largest one; otherwise the latest result the agent had seen. The run must have started after the last edit to a source file in the same checkout. |
|
|
97
|
+
| Code edited after the last run | A test ran, the agent then changed source code in that checkout (with an edit tool, a patch or the shell), and claimed tests pass without running them again. An edit made while the run was still going counts too. |
|
|
98
|
+
| Last run had failed | The latest test result in the session showed failures. A subagent's failing run in its own worktree is "couldn't tell" instead. |
|
|
99
|
+
| No test run | The agent changed code but ran nothing recognisable as a test before the claim. |
|
|
100
|
+
| Couldn't tell | Left out of the receipt rate, with the reason shown: the result can't be read (piped, truncated, mixed with other output), a file changed in a way that can't be traced (a script with no file name in it, `git checkout`, `git merge`), the test was still running, a subagent's run failed in its own worktree, or nothing in the session changed code. |
|
|
101
|
+
| Failing test "fixed" by editing only tests | A run failed, the agent changed a test it hadn't touched before, changed nothing else, and the next run passed. Fixing a test the agent just wrote doesn't count. Sometimes the test was wrong, so this is a flag to look at, not a verdict. |
|
|
102
|
+
| Weakened an existing test | An edit to a test that existed before the session removed cases or assertions, added `skip`/`xfail`, `.only` or an always-true assertion, or deleted the file. |
|
|
103
|
+
| Full-suite runs | A test command with no file, directory, node id or `-k` filter. |
|
|
104
|
+
| Retry loops | The same failing command run 3 or more times in a row with no edit in between. |
|
|
105
|
+
| Pushback and interrupts | Your messages that start with "no", "stop" or "revert" or contain "I told you", and turns you interrupted. |
|
|
106
|
+
| Risky git | `--no-verify`, `push --force`, `reset --hard`, discarding uncommitted changes. |
|
|
107
|
+
|
|
108
|
+
Background runs are followed: Claude Code's `run_in_background` results (the notification and the output file the agent reads) and Codex cells that finish later through `wait`.
|
|
109
|
+
|
|
110
|
+
## How it decides
|
|
111
|
+
|
|
112
|
+
Everything is deterministic pattern matching over the logs. There is no LLM involved, and the same logs always give the same numbers. When a rule is unsure, it says "couldn't tell" instead of flagging the agent.
|
|
113
|
+
|
|
114
|
+
- Each command's start and its result are separate events, in the order they were observed. A claim can only be backed by a result the agent had already seen. A run covers the code as it was when the run started, so an edit during the run makes it stale.
|
|
115
|
+
- In a Codex cell that runs several commands, each command gets its own output. If outputs can't be matched to commands, the result stays unknown.
|
|
116
|
+
- An edit counts once it happened: a Claude Code edit after its tool result confirms it, a Codex patch if it applied. Shell writes count too (`sed -i`, redirects, `tee`, `cp`/`mv`, scripts that write named files), placed by the `cd` in effect where they appear.
|
|
117
|
+
- A script's writes are the files it clearly writes, such as `open('x.py', 'w')`, `Path('x.py').write_text(...)`, `writeFileSync('x.js', ...)`, or a variable holding such a path that is written later. Files it only reads don't count. A write to a file it can't name (a loop over paths, `extractall`, `shutil.copy` to a variable) makes the claim "couldn't tell".
|
|
118
|
+
- An edit in one checkout or worktree doesn't make a run in another one stale. Only edits to source files count; docs, config, data, generated output and hidden tool folders don't.
|
|
119
|
+
- `tail pytest.log` is evidence about the run that wrote `pytest.log`, not a new run.
|
|
120
|
+
- Results come from the runner's summary line first (`3 failed, 10 passed in 1.2s`, `Tests: 1 failed`, `test result: FAILED`). An exit code is trusted only when the test ran last (`cd app && pytest`, `ruff check .; pytest`) or a plain `&&` chain exited 0. `npm test; true` exits 0 because of `true`.
|
|
121
|
+
- Commands refused by a permission prompt, the auto-mode classifier or a hook never ran, so they aren't test runs.
|
|
122
|
+
- A claim is a sentence like "all tests pass" or "62 passed". Plans ("I'll make sure the tests pass"), conditions ("once tests pass"), history ("the latest recorded run passed"), questions, partial results ("58 of 59 pass") and quoted runner output are ignored. English is supported, plus simple forms in Greek, Spanish, Portuguese, German, French and Italian.
|
|
123
|
+
- Subagent transcripts are merged into the parent session, so a subagent's passing run backs the parent's claim.
|
|
124
|
+
- Tests the agent created in the same session can't be "weakened". A Claude Code `Write` over an existing test is an overwrite, so it can.
|
|
125
|
+
- `guard` judges only the final message. "Correction: I did not run the tests" after an earlier claim doesn't trigger a block.
|
|
126
|
+
|
|
127
|
+
The rules are in [`src/classify.js`](src/classify.js) and the tests in [`test/`](test), including a labelled set of claim and non-claim sentences ([`test/claims.corpus.test.js`](test/claims.corpus.test.js)). A false positive or a missed claim makes a good [issue](https://github.com/EleChon-pixel/agent-polygraph/issues/new/choose); include the redacted sentence and it goes into that set.
|
|
128
|
+
|
|
129
|
+
## Accuracy
|
|
130
|
+
|
|
131
|
+
I checked it by hand on my own logs, several hundred claims across Claude Code and Codex. Of 25 detected claims picked at random, about 95% were real claims. About 10% of real claims are missed, judging by a sample of sentences that mention tests passing but weren't detected; most of those also mention an unrelated failure ("passes alone, failed in the full run"). I traced every flag back through its session. Almost all were real unbacked claims, and the rest were borderline partial claims.
|
|
132
|
+
|
|
133
|
+
On other people's sessions (26 public Claude Code sessions, mostly from [Trace Commons](https://huggingface.co/datasets/trace-commons/agent-traces)) every verdict was reviewed by hand. The first run found parser gaps, such as Windows PowerShell sessions, custom test runners and Gradle, which are now fixed. The result is 89% backed. All 24 backed verdicts match the output the agent saw, and of the 3 stale flags, 1 is confirmed and 2 are likely. [docs/evaluation.md](docs/evaluation.md) has the method, every fix and a one-command reproduction.
|
|
134
|
+
|
|
135
|
+
An outside review found ten ways a verdict could go wrong (a later result backing an earlier claim, one output shared by several commands, failed edits counted as changes). They are fixed and kept as tests in [`test/review.test.js`](test/review.test.js).
|
|
136
|
+
|
|
137
|
+
There was no public Codex data, so Codex has only been checked on my logs. Your phrasing and tools will differ from mine. Treat the numbers as a strong hint, and open the session file before relying on any one verdict.
|
|
138
|
+
|
|
139
|
+
## Limits
|
|
140
|
+
|
|
141
|
+
These are the gaps that remain. `record` and `--ci` close most of them once you turn them on.
|
|
142
|
+
|
|
143
|
+
- Tests outside the logs. Tests the agent ran, test runs Claude Code recorded from hooks, commands you typed with `!` in Claude Code, and test output you pasted into the chat all count. So do your own bash and zsh runs with `record --shell`, and GitHub Actions runs on pushed commits with `--ci`. Other shells (fish, PowerShell, an IDE's test button), other CI services, and CI runs on commits the session never printed are still invisible.
|
|
144
|
+
- "The full suite passes" is compared with the latest run about as large as the largest one in that folder, using the counts the runner printed. If no run printed a count, it's compared with the latest run, which may have been narrower. Claims that name no count, test file or full suite are compared with the latest run.
|
|
145
|
+
- Output with no summary. Codex keeps each command's full output. Claude Code saves output that is too long for the transcript to a file, which is read as long as it exists, and `record` keeps the full result too. A run abandoned before printing a result leaves nothing to read.
|
|
146
|
+
- Files a script names at run time. Without `record` these make a claim "couldn't tell", never "backed". `record` settles them, but only in git checkouts and only for sessions after it was installed.
|
|
147
|
+
- New agent versions can change the log format. Supported: Claude Code 2.x (Bash and PowerShell tools; Linux, macOS and Windows paths) and Codex CLI. The report lists the versions it read, warns about newer ones, and counts files it couldn't parse instead of dropping them silently.
|
|
148
|
+
- The numbers describe your sessions, projects and prompts. They are not a benchmark of the models.
|
|
149
|
+
|
|
150
|
+
## Options
|
|
151
|
+
|
|
152
|
+
```
|
|
153
|
+
--since <when> only sessions since a date (2026-10-01) or age (7d, 4w, 3m)
|
|
154
|
+
--project <text> only sessions whose working directory contains <text>
|
|
155
|
+
--exclude <text> skip sessions whose id starts with, or whose working directory contains, <text> (repeatable)
|
|
156
|
+
--agent <name> claude or codex (repeatable; default: both)
|
|
157
|
+
--by <col> compare by "agent" (default) or "model"
|
|
158
|
+
--examples <n> examples to show per finding type (default 3, 0 to hide)
|
|
159
|
+
--html [file] write a numbers-only report to share (default polygraph.html)
|
|
160
|
+
--json full results as JSON: findings, unclear reasons, unreadable files (keep it local)
|
|
161
|
+
--ci also use GitHub Actions results (goes online, through your gh login)
|
|
162
|
+
--claude-dir <dir> Claude Code projects dir (default ~/.claude/projects, or $CLAUDE_CONFIG_DIR/projects)
|
|
163
|
+
--codex-dir <dir> Codex sessions dir (default ~/.codex/sessions, or $CODEX_HOME/sessions)
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
## Privacy
|
|
167
|
+
|
|
168
|
+
It reads files and works offline. The exceptions are all opt-in:
|
|
169
|
+
|
|
170
|
+
- `guard` and `record` edit the hook config files they print when you install or uninstall them, and `record --shell` adds or removes one marked line in `~/.bashrc` and `~/.zshrc`.
|
|
171
|
+
- `record` saves snapshots and logs in `~/.agent-polygraph`.
|
|
172
|
+
- `--ci` contacts api.github.com through `gh` and sends repository names and date ranges.
|
|
173
|
+
|
|
174
|
+
The terminal report and `--json` include excerpts from your sessions. `--html` doesn't.
|
|
175
|
+
|
|
176
|
+
## License
|
|
177
|
+
|
|
178
|
+
MIT
|
package/bin/polygraph.js
ADDED
|
@@ -0,0 +1,261 @@
|
|
|
1
|
+
#!/usr/bin/env node
|
|
2
|
+
import fs from "node:fs";
|
|
3
|
+
import path from "node:path";
|
|
4
|
+
import { parseArgs } from "node:util";
|
|
5
|
+
import { createRequire } from "node:module";
|
|
6
|
+
import { scan } from "../src/scan.js";
|
|
7
|
+
import { aggregate } from "../src/analyze.js";
|
|
8
|
+
import { renderTerminal } from "../src/report/terminal.js";
|
|
9
|
+
import { renderHtml } from "../src/report/html.js";
|
|
10
|
+
|
|
11
|
+
const { version } = createRequire(import.meta.url)("../package.json");
|
|
12
|
+
|
|
13
|
+
if (process.argv[2] === "guard") {
|
|
14
|
+
await guard(process.argv.slice(3));
|
|
15
|
+
process.exit(0);
|
|
16
|
+
}
|
|
17
|
+
if (process.argv[2] === "record") {
|
|
18
|
+
await record(process.argv.slice(3));
|
|
19
|
+
process.exit(0);
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
const HELP = `agent-polygraph ${version}
|
|
23
|
+
Check what your coding agents claimed against what actually ran.
|
|
24
|
+
Reads Claude Code and Codex session logs on this machine. Nothing is uploaded
|
|
25
|
+
(except with --ci, which asks GitHub for your workflow results).
|
|
26
|
+
|
|
27
|
+
Usage: npx agent-polygraph [options]
|
|
28
|
+
|
|
29
|
+
--since <when> only sessions since a date (2026-10-01) or age (7d, 4w, 3m)
|
|
30
|
+
--project <text> only sessions whose working directory contains <text>
|
|
31
|
+
--exclude <text> skip sessions whose id starts with, or whose working directory
|
|
32
|
+
contains, <text> (repeatable)
|
|
33
|
+
--agent <name> claude or codex (repeatable; default: both)
|
|
34
|
+
--by <col> compare by "agent" (default) or "model"
|
|
35
|
+
--examples <n> examples to show per finding type (default 3, 0 to hide)
|
|
36
|
+
--html [file] write a numbers-only report to share (default polygraph.html)
|
|
37
|
+
--json print full results as JSON (findings, unclear reasons, unreadable files; keep it local)
|
|
38
|
+
--ci also use GitHub Actions results (goes online: asks the GitHub API,
|
|
39
|
+
through your \`gh\` login, about the repositories your sessions pushed to)
|
|
40
|
+
--demo run on generated sample sessions instead of yours
|
|
41
|
+
--claude-dir <dir> Claude Code projects dir (default ~/.claude/projects)
|
|
42
|
+
--codex-dir <dir> Codex sessions dir (default ~/.codex/sessions)
|
|
43
|
+
-h, --help show this help
|
|
44
|
+
-v, --version show the version
|
|
45
|
+
|
|
46
|
+
Guard (stop unbacked "tests pass" claims as they happen):
|
|
47
|
+
|
|
48
|
+
npx agent-polygraph guard --install add a Stop hook to this project (.claude/, .codex/)
|
|
49
|
+
npx agent-polygraph guard --install --global add it for every project (~/.claude, ~/.codex)
|
|
50
|
+
npx agent-polygraph guard --uninstall [--global]
|
|
51
|
+
--agent claude|codex only that agent's config
|
|
52
|
+
|
|
53
|
+
Record (more evidence than the logs carry; stays on this machine):
|
|
54
|
+
|
|
55
|
+
npx agent-polygraph record --install [--global] hook that snapshots the working tree after test
|
|
56
|
+
and file-writing commands (.claude/, .codex/)
|
|
57
|
+
npx agent-polygraph record --install --shell log test commands you run in your own terminals
|
|
58
|
+
(bash, zsh; time, folder, exit code; no output)
|
|
59
|
+
npx agent-polygraph record --uninstall [--global] [--shell]
|
|
60
|
+
`;
|
|
61
|
+
|
|
62
|
+
async function record(argv) {
|
|
63
|
+
const { values } = parseArgs({
|
|
64
|
+
args: argv,
|
|
65
|
+
options: {
|
|
66
|
+
install: { type: "boolean" },
|
|
67
|
+
uninstall: { type: "boolean" },
|
|
68
|
+
global: { type: "boolean" },
|
|
69
|
+
shell: { type: "boolean" },
|
|
70
|
+
agent: { type: "string", multiple: true },
|
|
71
|
+
command: { type: "string" },
|
|
72
|
+
},
|
|
73
|
+
});
|
|
74
|
+
const rec = await import("../src/record.js");
|
|
75
|
+
if (!values.install && !values.uninstall) {
|
|
76
|
+
// invoked by the agent as a PostToolUse hook: quiet, and never in the way
|
|
77
|
+
const chunks = [];
|
|
78
|
+
for await (const c of process.stdin) chunks.push(c);
|
|
79
|
+
rec.runRecordHook(Buffer.concat(chunks).toString("utf8"));
|
|
80
|
+
return;
|
|
81
|
+
}
|
|
82
|
+
const { install, uninstall } = await import("../src/guard.js");
|
|
83
|
+
const opts = { sub: "record", agents: values.agent?.length ? values.agent : ["claude", "codex"], global: values.global, command: values.command };
|
|
84
|
+
// --shell alone touches only the shell setup
|
|
85
|
+
const hooks = values.shell ? [] : values.install ? install(opts) : uninstall(opts);
|
|
86
|
+
for (const r of hooks) {
|
|
87
|
+
const verb = values.install ? (r.changed ? "added PostToolUse hook to" : "already installed in") : r.changed ? "removed PostToolUse hook from" : "nothing to remove in";
|
|
88
|
+
console.log(`${r.agent === "claude" ? "Claude Code" : "Codex"}: ${verb} ${r.file}`);
|
|
89
|
+
}
|
|
90
|
+
if (values.shell) {
|
|
91
|
+
const out = values.install ? rec.installShell() : rec.uninstallShell();
|
|
92
|
+
for (const r of out.rcs) console.log(`Shell: ${values.install ? (r.changed ? "added a line to" : "already in") : "removed the line from"} ${r.file}`);
|
|
93
|
+
if (values.install) console.log(`Shell: ${out.rcs.length ? "open a new terminal to start logging" : "no ~/.bashrc or ~/.zshrc found; source " + out.script + " from your shell's startup file"}. Only interactive bash and zsh are supported.`);
|
|
94
|
+
}
|
|
95
|
+
if (values.install) {
|
|
96
|
+
console.log(`\nRecords are kept in ${rec.home()} and never leave this machine.`);
|
|
97
|
+
if (hooks.length) {
|
|
98
|
+
console.log(`Hook command: ${hooks[0].command}`);
|
|
99
|
+
console.log("Snapshots start with the next session; earlier sessions are read as before.");
|
|
100
|
+
if (hooks.some((r) => r.agent === "codex")) console.log("Codex: run `codex` once and approve the new hook when it says it needs review; until then Codex skips it.");
|
|
101
|
+
}
|
|
102
|
+
}
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
async function guard(argv) {
|
|
106
|
+
const { values } = parseArgs({
|
|
107
|
+
args: argv,
|
|
108
|
+
options: {
|
|
109
|
+
install: { type: "boolean" },
|
|
110
|
+
uninstall: { type: "boolean" },
|
|
111
|
+
global: { type: "boolean" },
|
|
112
|
+
agent: { type: "string", multiple: true },
|
|
113
|
+
command: { type: "string" },
|
|
114
|
+
},
|
|
115
|
+
});
|
|
116
|
+
const { runHook, install, uninstall } = await import("../src/guard.js");
|
|
117
|
+
if (!values.install && !values.uninstall) {
|
|
118
|
+
// invoked by the agent as a Stop hook; never fail the agent's turn
|
|
119
|
+
try {
|
|
120
|
+
const chunks = [];
|
|
121
|
+
for await (const c of process.stdin) chunks.push(c);
|
|
122
|
+
const out = await runHook(Buffer.concat(chunks).toString("utf8"));
|
|
123
|
+
if (out) process.stdout.write(JSON.stringify(out));
|
|
124
|
+
} catch (e) {
|
|
125
|
+
if (process.env.POLYGRAPH_DEBUG) console.error(e);
|
|
126
|
+
}
|
|
127
|
+
return;
|
|
128
|
+
}
|
|
129
|
+
const opts = { agents: values.agent?.length ? values.agent : ["claude", "codex"], global: values.global, command: values.command };
|
|
130
|
+
const results = values.install ? install(opts) : uninstall(opts);
|
|
131
|
+
for (const r of results) {
|
|
132
|
+
const verb = values.install ? (r.changed ? "added Stop hook to" : "already installed in") : r.changed ? "removed Stop hook from" : "nothing to remove in";
|
|
133
|
+
console.log(`${r.agent === "claude" ? "Claude Code" : "Codex"}: ${verb} ${r.file}`);
|
|
134
|
+
}
|
|
135
|
+
if (values.install) {
|
|
136
|
+
console.log(`\nHook command: ${results[0].command}`);
|
|
137
|
+
console.log("When a final message says tests pass without a passing run after the last edit, the agent is asked once to rerun them.");
|
|
138
|
+
if (results.some((r) => r.agent === "codex")) {
|
|
139
|
+
console.log(
|
|
140
|
+
values.global
|
|
141
|
+
? "Codex: run `codex` once and approve the new hook when it says it needs review; until then Codex skips it."
|
|
142
|
+
: "Codex: run `codex` once in this project, trust the folder, and approve the new hook when it says it needs review; until then Codex skips it.",
|
|
143
|
+
);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
let args;
|
|
149
|
+
try {
|
|
150
|
+
args = parseArgs({
|
|
151
|
+
allowPositionals: false,
|
|
152
|
+
options: {
|
|
153
|
+
since: { type: "string" },
|
|
154
|
+
project: { type: "string" },
|
|
155
|
+
exclude: { type: "string", multiple: true },
|
|
156
|
+
agent: { type: "string", multiple: true },
|
|
157
|
+
by: { type: "string", default: "agent" },
|
|
158
|
+
examples: { type: "string", default: "3" },
|
|
159
|
+
html: { type: "string" },
|
|
160
|
+
json: { type: "boolean" },
|
|
161
|
+
demo: { type: "boolean" },
|
|
162
|
+
ci: { type: "boolean" },
|
|
163
|
+
"claude-dir": { type: "string" },
|
|
164
|
+
"codex-dir": { type: "string" },
|
|
165
|
+
help: { type: "boolean", short: "h" },
|
|
166
|
+
version: { type: "boolean", short: "v" },
|
|
167
|
+
},
|
|
168
|
+
// `--html` with no value
|
|
169
|
+
strict: true,
|
|
170
|
+
tokens: false,
|
|
171
|
+
args: process.argv.slice(2).flatMap((a, i, arr) => (a === "--html" && (!arr[i + 1] || arr[i + 1].startsWith("-")) ? ["--html", "polygraph.html"] : [a])),
|
|
172
|
+
}).values;
|
|
173
|
+
} catch (e) {
|
|
174
|
+
console.error(`${e.message}\n\n${HELP}`);
|
|
175
|
+
process.exit(2);
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
if (args.help) {
|
|
179
|
+
process.stdout.write(HELP);
|
|
180
|
+
process.exit(0);
|
|
181
|
+
}
|
|
182
|
+
if (args.version) {
|
|
183
|
+
console.log(version);
|
|
184
|
+
process.exit(0);
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
const agents = args.agent?.map((a) => a.toLowerCase());
|
|
188
|
+
if (agents?.some((a) => !["claude", "codex"].includes(a))) {
|
|
189
|
+
console.error("--agent must be claude or codex");
|
|
190
|
+
process.exit(2);
|
|
191
|
+
}
|
|
192
|
+
if (!["agent", "model"].includes(args.by)) {
|
|
193
|
+
console.error('--by must be "agent" or "model"');
|
|
194
|
+
process.exit(2);
|
|
195
|
+
}
|
|
196
|
+
|
|
197
|
+
const since = parseSince(args.since);
|
|
198
|
+
if (args.since && !since) {
|
|
199
|
+
console.error(`Can't read --since ${args.since}; use a date like 2026-10-01 or an age like 7d`);
|
|
200
|
+
process.exit(2);
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
const started = Date.now();
|
|
204
|
+
const progress = process.stderr.isTTY && !args.json;
|
|
205
|
+
let roots = { claude: args["claude-dir"], codex: args["codex-dir"] };
|
|
206
|
+
let demoDir;
|
|
207
|
+
if (args.demo) {
|
|
208
|
+
const { makeDemo } = await import("../src/demo.js");
|
|
209
|
+
const d = makeDemo();
|
|
210
|
+
demoDir = d.root;
|
|
211
|
+
roots = { claude: d.claudeDir, codex: d.codexDir };
|
|
212
|
+
}
|
|
213
|
+
let ci = null;
|
|
214
|
+
if (args.ci && !args.demo) {
|
|
215
|
+
const { checkGh, makeCi } = await import("../src/ci.js");
|
|
216
|
+
try {
|
|
217
|
+
await checkGh();
|
|
218
|
+
ci = makeCi();
|
|
219
|
+
console.error("--ci: asking GitHub (api.github.com, through your gh login) for workflow runs of the repositories your sessions pushed to. Only repository names, branches and dates are sent.");
|
|
220
|
+
} catch (e) {
|
|
221
|
+
console.error(e.message);
|
|
222
|
+
process.exit(2);
|
|
223
|
+
}
|
|
224
|
+
}
|
|
225
|
+
const sessions = await scan({
|
|
226
|
+
agents,
|
|
227
|
+
since,
|
|
228
|
+
project: args.project,
|
|
229
|
+
exclude: args.exclude,
|
|
230
|
+
roots,
|
|
231
|
+
records: !args.demo,
|
|
232
|
+
enrich: ci ? ci.enrich : undefined,
|
|
233
|
+
onProgress: progress ? (done, total) => process.stderr.write(`\rReading sessions… ${done}/${total}`) : undefined,
|
|
234
|
+
});
|
|
235
|
+
if (progress) process.stderr.write("\r\x1b[K");
|
|
236
|
+
const groups = aggregate(sessions);
|
|
237
|
+
const seconds = (Date.now() - started) / 1000;
|
|
238
|
+
if (demoDir) fs.rmSync(demoDir, { recursive: true, force: true });
|
|
239
|
+
|
|
240
|
+
if (args.json) {
|
|
241
|
+
process.stdout.write(JSON.stringify({ version, groups, sessions, skipped: sessions.skipped, unrecognized: sessions.unrecognized }, null, 2) + "\n");
|
|
242
|
+
} else {
|
|
243
|
+
console.log(renderTerminal({ sessions, groups, seconds, by: args.by, examples: Math.max(0, Number(args.examples) || 0), demo: args.demo }));
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
if (args.html) {
|
|
247
|
+
const file = path.resolve(args.html);
|
|
248
|
+
fs.writeFileSync(file, renderHtml({ groups, by: args.by, version, demo: args.demo, skipped: sessions.skipped?.length || 0, unrecognized: sessions.unrecognized?.length || 0 }));
|
|
249
|
+
console.error(`Wrote ${file}`);
|
|
250
|
+
}
|
|
251
|
+
|
|
252
|
+
function parseSince(s) {
|
|
253
|
+
if (!s) return null;
|
|
254
|
+
const rel = /^(\d+)\s*([dwmy])$/i.exec(s.trim());
|
|
255
|
+
if (rel) {
|
|
256
|
+
const days = { d: 1, w: 7, m: 30, y: 365 }[rel[2].toLowerCase()] * Number(rel[1]);
|
|
257
|
+
return Date.now() - days * 86400000;
|
|
258
|
+
}
|
|
259
|
+
const t = Date.parse(s);
|
|
260
|
+
return Number.isNaN(t) ? null : t;
|
|
261
|
+
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,40 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "agent-polygraph",
|
|
3
|
-
"version": "0.
|
|
4
|
-
"
|
|
5
|
-
"
|
|
6
|
-
|
|
3
|
+
"version": "0.1.0",
|
|
4
|
+
"description": "Check your coding agents' \"tests pass\" claims against what actually ran. Reads Claude Code and Codex session logs locally.",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"bin": {
|
|
7
|
+
"agent-polygraph": "bin/polygraph.js",
|
|
8
|
+
"polygraph": "bin/polygraph.js"
|
|
9
|
+
},
|
|
10
|
+
"files": [
|
|
11
|
+
"bin",
|
|
12
|
+
"src",
|
|
13
|
+
"README.md",
|
|
14
|
+
"LICENSE"
|
|
15
|
+
],
|
|
16
|
+
"scripts": {
|
|
17
|
+
"test": "node --test 'test/*.test.js'",
|
|
18
|
+
"start": "node bin/polygraph.js"
|
|
19
|
+
},
|
|
20
|
+
"engines": {
|
|
21
|
+
"node": ">=18.3"
|
|
22
|
+
},
|
|
23
|
+
"keywords": [
|
|
24
|
+
"claude-code",
|
|
25
|
+
"codex",
|
|
26
|
+
"coding-agents",
|
|
27
|
+
"ai-agents",
|
|
28
|
+
"testing",
|
|
29
|
+
"audit",
|
|
30
|
+
"cli"
|
|
31
|
+
],
|
|
32
|
+
"license": "MIT",
|
|
33
|
+
"repository": {
|
|
34
|
+
"type": "git",
|
|
35
|
+
"url": "git+https://github.com/EleChon-pixel/agent-polygraph.git"
|
|
36
|
+
},
|
|
37
|
+
"homepage": "https://github.com/EleChon-pixel/agent-polygraph#readme",
|
|
38
|
+
"bugs": "https://github.com/EleChon-pixel/agent-polygraph/issues",
|
|
39
|
+
"author": "Eleftherios Chondros"
|
|
40
|
+
}
|