heeded 0.0.0-stage → 0.1.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +147 -2
- package/dist/check.js +495 -0
- package/dist/claude-code.js +250 -0
- package/dist/cli.js +258 -0
- package/dist/judge.js +359 -0
- package/dist/model.js +112 -0
- package/dist/redact.js +21 -0
- package/dist/report.js +186 -0
- package/dist/rules.js +402 -0
- package/dist/shell.js +182 -0
- package/package.json +23 -4
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Francesco Musso
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
CHANGED
|
@@ -1,3 +1,148 @@
|
|
|
1
|
-
#
|
|
1
|
+
# heeded
|
|
2
2
|
|
|
3
|
-
|
|
3
|
+
did your coding agent follow its instructions?
|
|
4
|
+
|
|
5
|
+
you write rules for your agent in `CLAUDE.md` and hope they stick. heeded reads the transcripts claude code keeps on
|
|
6
|
+
disk and tells you which rules were kept, which were broken, when and where.
|
|
7
|
+
|
|
8
|
+
```text
|
|
9
|
+
$ npx heeded
|
|
10
|
+
|
|
11
|
+
~/repos/app · 76 sessions, 412 subagents · oct 11 – nov 10
|
|
12
|
+
3 files · 41 rules · 12 checked · 29 need a judge
|
|
13
|
+
|
|
14
|
+
CLAUDE.md
|
|
15
|
+
✗ :12 never git add -A kept 95% of 214 broken 9× last 2 days ago
|
|
16
|
+
nov 8 19df6c20 git add -A && git commit -m "fix: …"
|
|
17
|
+
nov 2 8c0e8fab git add -A
|
|
18
|
+
✓ :14 never force-push kept 100% of 61
|
|
19
|
+
○ :21 use pnpm, not npm never applied ~40 tokens every session
|
|
20
|
+
· :30 failing test first kept 80% of 20 judged broken 4× last 3 days ago
|
|
21
|
+
~/.claude/CLAUDE.md
|
|
22
|
+
✗ :3 answer in italian kept 96% of 3,812 broken 152× last today
|
|
23
|
+
after compaction kept 69% of 410
|
|
24
|
+
○ :17 ask before deploys never applied in 8 judged ~30 tokens every session
|
|
25
|
+
|
|
26
|
+
kept by context depth 0–100k 100–200k 200–400k 400k+
|
|
27
|
+
98% 97% 91% 84%
|
|
28
|
+
|
|
29
|
+
6 rules no call or reply shows: heeded rules
|
|
30
|
+
up to date: new sessions and changes to instruction files are read on the next run
|
|
31
|
+
```
|
|
32
|
+
|
|
33
|
+
<sub>a sketch: the numbers are invented.</sub>
|
|
34
|
+
|
|
35
|
+
| mark | the rule |
|
|
36
|
+
| --- | --- |
|
|
37
|
+
| `✗` | was broken: every case confirmed by the judge, listed with its date, session and call |
|
|
38
|
+
| `✓` | was kept every time it applied |
|
|
39
|
+
| `?` | has candidates only: a pattern flagged them, the judge hasn't looked yet |
|
|
40
|
+
| `○` | never applied in these sessions: it sits in every session's context all the same |
|
|
41
|
+
| `·` | needs a judge, no pattern can decide it: a sample of its cases was judged |
|
|
42
|
+
|
|
43
|
+
- `kept 95% of 214`: the rule applied 214 times and was kept in 95% of them.
|
|
44
|
+
- `after compaction`: the same rule, counted only after claude code compacted the context.
|
|
45
|
+
- each rule is under the file it is written in: the folder's files first, then the others (`~/.claude/CLAUDE.md` is
|
|
46
|
+
loaded in every project, so a rule there can be never applied here and needed elsewhere).
|
|
47
|
+
- `:12`: the line the rule starts on in that file, as the file is now. no number: its words aren't in the file any
|
|
48
|
+
more.
|
|
49
|
+
- `kept by context depth`: every rule together, by how full the context was at the time.
|
|
50
|
+
- `never applied in 8 judged`: none of the 8 cases the judge read was about the rule.
|
|
51
|
+
- `rules no call or reply shows`: rules about how to work ("plan before coding"), which no single call or reply can
|
|
52
|
+
show; `heeded rules` lists them.
|
|
53
|
+
|
|
54
|
+
## use
|
|
55
|
+
|
|
56
|
+
needs node 22 or later and claude code, signed in: heeded calls your own `claude` to read rules and to judge.
|
|
57
|
+
|
|
58
|
+
1. **run it in a project** you've worked on with claude code:
|
|
59
|
+
|
|
60
|
+
```sh
|
|
61
|
+
cd ~/code/my-app
|
|
62
|
+
npx heeded
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
2. **say yes, or not yet.** reading the rules and judging cost tokens from your claude account, so heeded shows the
|
|
66
|
+
estimate and asks before spending anything:
|
|
67
|
+
|
|
68
|
+
```text
|
|
69
|
+
11 rule sections to read, then cases to judge · claude sonnet and haiku · up to ~175k tokens
|
|
70
|
+
go ahead? [y/N]
|
|
71
|
+
```
|
|
72
|
+
|
|
73
|
+
`y` reads the rules, judges a first round of cases and prints the report. enter spends nothing: the report shows
|
|
74
|
+
what is already cached, on a first run the sessions found and the sections still to read.
|
|
75
|
+
|
|
76
|
+
3. **act on the report.** each rule is under its file, with its line (`:12`). a rule broken often may need
|
|
77
|
+
rewriting, or saying why it matters. a rule never applied costs its tokens in every session: move it to the project
|
|
78
|
+
it's for, or remove it. `heeded rules` shows how each rule is checked, and the ones no call or reply can show.
|
|
79
|
+
|
|
80
|
+
4. **run it again** whenever you like: after more sessions, or after editing an instruction file. every run reads the
|
|
81
|
+
transcripts again, for free, and asks only when something new costs tokens. what was read or judged is cached and
|
|
82
|
+
never paid for twice; with nothing new, the report ends `up to date`.
|
|
83
|
+
|
|
84
|
+
```sh
|
|
85
|
+
npx heeded # the report, for the claude code sessions run in this folder
|
|
86
|
+
npx heeded rules # the rules found, and how each one is checked
|
|
87
|
+
npx heeded --all # judge every case waiting, not one run's 150k tokens
|
|
88
|
+
npx heeded --json # everything, as json
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
to type `heeded` instead of `npx heeded`, install it: `npm install -g heeded`.
|
|
92
|
+
|
|
93
|
+
## how it works
|
|
94
|
+
|
|
95
|
+
1. **find the sessions.** the transcripts of the sessions run in this folder and its worktrees, subagents included,
|
|
96
|
+
under `~/.claude/projects/`.
|
|
97
|
+
2. **read the rules.** claude code writes into each transcript the instruction files it loaded, word for word. heeded
|
|
98
|
+
splits them into sections and asks `claude` (sonnet) for each section's rules, each with a check:
|
|
99
|
+
- a pattern over a tool call, like `git add -A` in a bash command;
|
|
100
|
+
- the language replies must be in;
|
|
101
|
+
- otherwise, the calls or replies the rule is about, left to the judge.
|
|
102
|
+
|
|
103
|
+
a check that fails its own examples is set aside. a section is read once, then cached.
|
|
104
|
+
3. **check every call.** one pass over the transcripts: each call and reply against the rules in effect at that point.
|
|
105
|
+
a shell command is read the way a shell reads it, so `echo "rm -rf"` isn't running `rm`.
|
|
106
|
+
4. **judge.** a pattern finds candidates, not violations. each candidate, and a sample of the cases no pattern can
|
|
107
|
+
decide, goes to `claude` (haiku) with an excerpt of its session: your prompt, what the agent did since, the call
|
|
108
|
+
itself. it answers broken, kept, or doesn't apply. only your words grant a permission; an agent's never do. each
|
|
109
|
+
verdict is cached. a rule the judge decides is tried on 8 of its cases first: if none is about it, it stops there;
|
|
110
|
+
if some are, it goes on to 20 cases that applied, or 40 judged. up to 3 calls run at once.
|
|
111
|
+
5. **report.** each rule under its file, at its line in the file as it is now: kept, broken with the evidence, never
|
|
112
|
+
applied; by context depth and after compaction.
|
|
113
|
+
|
|
114
|
+
steps 2 and 4 cost tokens: heeded asks once, before spending anything. a run judges within 150k tokens and nothing is
|
|
115
|
+
paid for twice, so coverage grows run after run. `heeded --all` goes to the end instead: its question shows what waits
|
|
116
|
+
now and the most it can reach, then it judges round after round until nothing waits. off a terminal (ci, a script) it
|
|
117
|
+
never spends.
|
|
118
|
+
|
|
119
|
+
`heeded rules` shows what each rule became:
|
|
120
|
+
|
|
121
|
+
```text
|
|
122
|
+
$ npx heeded rules
|
|
123
|
+
|
|
124
|
+
~/repos/app · 1 file · 5 rules
|
|
125
|
+
|
|
126
|
+
CLAUDE.md
|
|
127
|
+
✓ :8 never git add -A Bash command when (^|\s)git\s+add(\s|$) broken \s(-A|--all)\b
|
|
128
|
+
✓ :12 answer in italian language it
|
|
129
|
+
· :15 commit only when asked judged on Bash command when (^|\s)git\s+commit(\s|$)
|
|
130
|
+
· :19 say what you changed judged on replies
|
|
131
|
+
! :23 never kill the dev server failed its examples: keeps "pkill -f vite-test"
|
|
132
|
+
```
|
|
133
|
+
|
|
134
|
+
`✓` a pattern decides it: `when` picks the calls it applies to, `broken` the ones that break it. `·` the judge
|
|
135
|
+
decides, on the calls or replies shown. `!` its check was set aside, and why.
|
|
136
|
+
|
|
137
|
+
## privacy
|
|
138
|
+
|
|
139
|
+
local and read-only: heeded never changes your transcripts, instruction files or settings, and writes only to its own
|
|
140
|
+
cache (`~/.cache/heeded`, or under `$XDG_CACHE_HOME`). two things leave your machine, through your own `claude`: the
|
|
141
|
+
text of instruction sections not read before, and the excerpts the judge reads. both have secrets redacted first, and
|
|
142
|
+
what is written to files like `.env` is never sent. no telemetry.
|
|
143
|
+
|
|
144
|
+
`CLAUDE_CONFIG_DIR` points heeded at another claude code folder (default `~/.claude`).
|
|
145
|
+
|
|
146
|
+
## license
|
|
147
|
+
|
|
148
|
+
[mit](LICENSE)
|
package/dist/check.js
ADDED
|
@@ -0,0 +1,495 @@
|
|
|
1
|
+
import { excerpts } from './judge.js';
|
|
2
|
+
import { redact } from './redact.js';
|
|
3
|
+
import { commands } from './shell.js';
|
|
4
|
+
/** The most common words of each language a reply can be detected in. */
|
|
5
|
+
const STOPWORDS = {
|
|
6
|
+
en: `the be to of and a in that have i it for not on with he as you do at this but his by from they we say her she or an
|
|
7
|
+
will my one all would there their what so up out if about who get which go me when make can like no just him know
|
|
8
|
+
take into your some could them see other than then now only its also how our well way even because any these us is
|
|
9
|
+
are was were been has had does did should here where why very`,
|
|
10
|
+
it: `il lo la i gli le un uno una di da in con su per tra fra a e o ma che non come anche se più è sono era sia del della
|
|
11
|
+
dei degli delle al alla allo ai agli alle dal dalla nel nella nei sul sulla questo questa questi quello quella ci si mi
|
|
12
|
+
ti ne cosa perché quando dove tutto molto ancora già poi solo ho hai ha abbiamo hanno essere fare può adesso quindi ora`,
|
|
13
|
+
es: `el la los las un una unos unas de del a al en con por para que y o pero no sí es son era fue ser está están estar
|
|
14
|
+
como más muy también ya este esta estos estas ese esa eso lo le les se me te nos su sus mi tu cuando donde porque hay
|
|
15
|
+
hacer puede todo nada algo bien entre sobre sin hasta desde ahora así luego aquí pues`,
|
|
16
|
+
fr: `le la les un une des de du à au aux en dans par pour sur avec sans que qui et ou mais ne pas plus est sont était
|
|
17
|
+
être avoir ai as ont ce cette ces cet il elle ils elles je tu nous vous on se me te lui leur son sa ses mon ma mes
|
|
18
|
+
comme aussi très bien tout tous faire peut fait donc alors si quand où vais`,
|
|
19
|
+
de: `der die das den dem des ein eine einen einem einer und oder aber nicht ist sind war waren sein haben hat habe ich du
|
|
20
|
+
er sie es wir ihr mit von zu zum zur auf für bei aus nach über unter vor auch noch nur schon sehr wie wenn dass weil
|
|
21
|
+
was wer wo hier dann jetzt kann muss soll mehr kein keine diese dieser dieses man sich im deshalb`,
|
|
22
|
+
pt: `o a os as um uma uns umas de do da dos das em no na nos nas por para com sem que e ou mas não sim é são era foi ser
|
|
23
|
+
estar está estão ter tem como mais muito também já este esta isso esse essa ele ela eles elas eu você nós se me te
|
|
24
|
+
lhe seu sua quando onde porque há fazer pode tudo nada bem entre sobre até então agora vou depois ainda aqui pelo pela
|
|
25
|
+
ao aos num numa`,
|
|
26
|
+
};
|
|
27
|
+
/** The languages a reply can be detected in. */
|
|
28
|
+
export const LANGUAGES = Object.keys(STOPWORDS);
|
|
29
|
+
/** Each stopword's language; a word shared by two languages (`de`, `que`, `a`) tells none apart, so it is left out. */
|
|
30
|
+
const WORDS = new Map();
|
|
31
|
+
for (const [code, words] of Object.entries(STOPWORDS)) {
|
|
32
|
+
for (const w of words.split(/\s+/))
|
|
33
|
+
WORDS.set(w, WORDS.has(w) && WORDS.get(w) !== code ? null : code);
|
|
34
|
+
}
|
|
35
|
+
/**
|
|
36
|
+
* The language a reply is written in, or null when it can't be told: under 20 words once code, quoted blocks (someone
|
|
37
|
+
* else's words), URLs and paths are removed, or no clear winner (at least 5 stopwords, twice the runner-up's).
|
|
38
|
+
*/
|
|
39
|
+
export function language(text) {
|
|
40
|
+
const prose = text.replace(/(```|~~~)[\s\S]*?(\1|$)/g, ' ').replace(/^[ \t]*>.*$/gm, ' ').replace(/`[^`\n]*`/g, ' ')
|
|
41
|
+
.replace(/\S*[/\\]\S*/g, ' ');
|
|
42
|
+
const words = prose.toLowerCase().match(/\p{L}+/gu) ?? [];
|
|
43
|
+
if (words.length < 20)
|
|
44
|
+
return null;
|
|
45
|
+
const hits = new Map();
|
|
46
|
+
for (const w of words) {
|
|
47
|
+
const code = WORDS.get(w);
|
|
48
|
+
if (code)
|
|
49
|
+
hits.set(code, (hits.get(code) ?? 0) + 1);
|
|
50
|
+
}
|
|
51
|
+
const [first, second] = [...hits].sort((a, b) => b[1] - a[1]);
|
|
52
|
+
return first && first[1] >= 5 && first[1] >= 2 * (second?.[1] ?? 0) ? first[0] : null;
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Whether a pattern repeats a group that holds a quantifier or an alternation, at any depth — `(a+)+`, `((a+))+`,
|
|
56
|
+
* `(a|a)+` — the shapes that can take exponential time on a long value. Character classes are read as one atom.
|
|
57
|
+
*/
|
|
58
|
+
function exponential(source) {
|
|
59
|
+
const open = []; // per open group: whether it holds a quantifier or an alternation
|
|
60
|
+
for (let i = 0; i < source.length; i++) {
|
|
61
|
+
const c = source[i];
|
|
62
|
+
if (c === '\\')
|
|
63
|
+
i++;
|
|
64
|
+
else if (c === '[') {
|
|
65
|
+
for (i++; i < source.length && source[i] !== ']'; i++)
|
|
66
|
+
if (source[i] === '\\')
|
|
67
|
+
i++;
|
|
68
|
+
}
|
|
69
|
+
else if (c === '(')
|
|
70
|
+
open.push(false);
|
|
71
|
+
else if (c === ')') {
|
|
72
|
+
const holds = open.pop() ?? false;
|
|
73
|
+
const repeated = /[*+{]/.test(source[i + 1] ?? '');
|
|
74
|
+
if (holds && repeated)
|
|
75
|
+
return true;
|
|
76
|
+
if (open.length && (holds || repeated))
|
|
77
|
+
open[open.length - 1] = true;
|
|
78
|
+
}
|
|
79
|
+
else if (/[*+{|]/.test(c) && open.length)
|
|
80
|
+
open[open.length - 1] = true;
|
|
81
|
+
}
|
|
82
|
+
return false;
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* The check's patterns as regular expressions, or why they can't be used. Never `g` or `y` (they keep state between
|
|
86
|
+
* calls) nor `u` (model-written patterns often escape characters it rejects); `i` only when the check asks.
|
|
87
|
+
*/
|
|
88
|
+
export function compile(check) {
|
|
89
|
+
const flags = check.ignoreCase ? 'i' : '';
|
|
90
|
+
const compiled = { field: check.field, when: {}, broken: {} };
|
|
91
|
+
for (const part of ['when', 'broken']) {
|
|
92
|
+
for (const key of ['tool', 'value', 'data']) {
|
|
93
|
+
const source = check[part]?.[key];
|
|
94
|
+
if (source === undefined)
|
|
95
|
+
continue;
|
|
96
|
+
if (exponential(source))
|
|
97
|
+
return `regex may take exponential time: ${source}`;
|
|
98
|
+
try {
|
|
99
|
+
compiled[part][key] = new RegExp(source, flags);
|
|
100
|
+
}
|
|
101
|
+
catch {
|
|
102
|
+
return `regex doesn't compile: ${source}`;
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
return compiled;
|
|
107
|
+
}
|
|
108
|
+
// The last command line read: every rule in effect looks at the same call in turn.
|
|
109
|
+
let last;
|
|
110
|
+
/** What a check's field shows: one view per simple command of a shell `command`, else the value whole; none when the
|
|
111
|
+
* check has no field or the call's field isn't a string. */
|
|
112
|
+
function views(field, input) {
|
|
113
|
+
const value = field === undefined ? undefined : input[field];
|
|
114
|
+
if (typeof value !== 'string')
|
|
115
|
+
return [];
|
|
116
|
+
if (field !== 'command')
|
|
117
|
+
return [{ does: value, carries: '', text: value }];
|
|
118
|
+
if (last?.line !== value)
|
|
119
|
+
last = { line: value, views: commands(value) };
|
|
120
|
+
return last.views;
|
|
121
|
+
}
|
|
122
|
+
/** Each view is matched on its first 4,096 characters: model-written patterns over a huge command or file stay cheap. */
|
|
123
|
+
const hits = (p, v) => (!p.value || p.value.test(v.does.slice(0, 4096)))
|
|
124
|
+
&& (!p.data || p.data.test(v.carries.slice(0, 4096)));
|
|
125
|
+
const needs = (p) => p.value !== undefined || p.data !== undefined;
|
|
126
|
+
/** Whether one of a call's views matches the patterns; a loop, not a closure: it runs for every rule on every call. */
|
|
127
|
+
function found(p, all) {
|
|
128
|
+
if (!needs(p))
|
|
129
|
+
return true;
|
|
130
|
+
for (const v of all)
|
|
131
|
+
if (hits(p, v))
|
|
132
|
+
return true;
|
|
133
|
+
return false;
|
|
134
|
+
}
|
|
135
|
+
/**
|
|
136
|
+
* Whether the rule applies to a call and, when it does, the view that broke it: the call applies when its tool
|
|
137
|
+
* matches and one view matches `when`; it is broken when, besides, that same view matches `broken`.
|
|
138
|
+
*/
|
|
139
|
+
function assess(c, tool, input) {
|
|
140
|
+
if (c.when.tool && !c.when.tool.test(tool))
|
|
141
|
+
return { verdict: 'none' };
|
|
142
|
+
const all = views(c.field, input);
|
|
143
|
+
const applying = needs(c.when) ? all.filter((v) => hits(c.when, v)) : all;
|
|
144
|
+
if (needs(c.when) && !applying.length)
|
|
145
|
+
return { verdict: 'none' };
|
|
146
|
+
if (c.broken.tool && !c.broken.tool.test(tool))
|
|
147
|
+
return { verdict: 'kept' };
|
|
148
|
+
if (!needs(c.broken))
|
|
149
|
+
return { verdict: 'broken' };
|
|
150
|
+
const by = applying.find((v) => hits(c.broken, v));
|
|
151
|
+
return by ? { verdict: 'broken', by } : { verdict: 'kept' };
|
|
152
|
+
}
|
|
153
|
+
export function verdict(c, tool, input) {
|
|
154
|
+
return assess(c, tool, input).verdict;
|
|
155
|
+
}
|
|
156
|
+
/** Whether a call is one the check is about, whatever would break it: asked on every call, so it allocates nothing. */
|
|
157
|
+
export function applies(c, tool, input) {
|
|
158
|
+
return (!c.when.tool || c.when.tool.test(tool)) && found(c.when, views(c.field, input));
|
|
159
|
+
}
|
|
160
|
+
/**
|
|
161
|
+
* What broke the rule, as written: the simple command (followed by what it carries, when the check reads that and the
|
|
162
|
+
* command as written doesn't show it: a heredoc's body), the field's value, or the tool when the check has no field.
|
|
163
|
+
*/
|
|
164
|
+
export function culprit(c, tool, input) {
|
|
165
|
+
const by = assess(c, tool, input).by;
|
|
166
|
+
if (by)
|
|
167
|
+
return (c.when.data || c.broken.data) && !by.text.includes(by.carries) ? `${by.text} ${by.carries}` : by.text;
|
|
168
|
+
const value = c.field === undefined ? undefined : input[c.field];
|
|
169
|
+
return typeof value === 'string' ? value : tool;
|
|
170
|
+
}
|
|
171
|
+
/** Context depth buckets, in tokens: where the context window fills up. */
|
|
172
|
+
const DEPTHS = [0, 100_000, 200_000, 400_000];
|
|
173
|
+
const buckets = (counts) => counts.map((c, i) => ({ from: DEPTHS[i], to: DEPTHS[i + 1] ?? null, ...c }));
|
|
174
|
+
const zero = () => ({ applied: 0, broken: 0 });
|
|
175
|
+
const add = (c, broken) => {
|
|
176
|
+
c.applied++;
|
|
177
|
+
if (broken)
|
|
178
|
+
c.broken++;
|
|
179
|
+
};
|
|
180
|
+
/** The cases a rule keeps waiting, at most: a run judges far fewer. */
|
|
181
|
+
const WAITING = 40;
|
|
182
|
+
/** A judgment rule's sample: its first 8 cases, and if none of them applies, no more: a rule that doesn't concern the
|
|
183
|
+
* folder (a global rule about a tool it never uses) stops costing. Once one applies, judged until 20 of its cases are
|
|
184
|
+
* decided, or 40 judged in all. */
|
|
185
|
+
const PROBE = 8;
|
|
186
|
+
const DECIDED = 20;
|
|
187
|
+
const JUDGED = 40;
|
|
188
|
+
const allowed = (t) => (t.rule.kind !== 'judgment' ? Infinity
|
|
189
|
+
: Math.max(0, t.all.applied ? Math.min(JUDGED - t.judged, DECIDED - t.all.applied) : PROBE - t.judged));
|
|
190
|
+
/** Whether a rule's sample may ever take another case. Unlike `allowed`, never true again once false: a sample stopped
|
|
191
|
+
* after 8 cases that didn't apply opens again when a later case applies. */
|
|
192
|
+
const open = (t) => t.rule.kind !== 'judgment' || Math.min(JUDGED - t.judged, DECIDED - t.all.applied) > 0;
|
|
193
|
+
/** The most a rule may still take: a judgment rule whose sample hasn't ended, every case up to 40 judged. */
|
|
194
|
+
const most = (t) => (t.rule.kind !== 'judgment' ? t.unjudged : allowed(t) ? Math.min(t.unjudged, JUDGED - t.judged) : 0);
|
|
195
|
+
/** A rule's cases waiting: latest first (ties by key), an excerpt once (two events can read alike), at most 40. */
|
|
196
|
+
function trim(list) {
|
|
197
|
+
list.sort((a, b) => (a.at !== b.at ? (a.at > b.at ? -1 : 1) : a.key < b.key ? -1 : a.key > b.key ? 1 : 0));
|
|
198
|
+
const keys = new Set();
|
|
199
|
+
let n = 0;
|
|
200
|
+
for (const x of list) {
|
|
201
|
+
if (n === WAITING || keys.has(x.key))
|
|
202
|
+
continue;
|
|
203
|
+
keys.add(x.key);
|
|
204
|
+
list[n++] = x;
|
|
205
|
+
}
|
|
206
|
+
list.length = n;
|
|
207
|
+
}
|
|
208
|
+
/** Into a rule's cases waiting, trimmed now and then rather than kept in order: rules × events is the hot loop. */
|
|
209
|
+
function wait(list, x) {
|
|
210
|
+
list.push(x);
|
|
211
|
+
if (list.length > 2 * WAITING)
|
|
212
|
+
trim(list);
|
|
213
|
+
}
|
|
214
|
+
/** A violation in one line, secrets redacted: what in the call broke the rule, a reply's language and first words. */
|
|
215
|
+
function evidence(e, c, detected) {
|
|
216
|
+
if (e.kind === 'reply') {
|
|
217
|
+
const words = redact(e.text).split(/[\p{Cc}\s]+/u).filter(Boolean);
|
|
218
|
+
return `${detected}: "${words.slice(0, 8).join(' ')}${words.length > 8 ? ' …' : ''}"`;
|
|
219
|
+
}
|
|
220
|
+
const line = redact(culprit(c, e.tool, e.input)).replace(/[\p{Cc}\s]+/gu, ' ').trim();
|
|
221
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
222
|
+
}
|
|
223
|
+
/**
|
|
224
|
+
* Counts, over a stream of events, how often each rule applied and how often it was broken. Each transcript starts with
|
|
225
|
+
* no file in effect; the rules in effect are those of the files its `instructions` events recorded. A call a tool
|
|
226
|
+
* rule's patterns say broke it is a candidate, and a call or reply a judgment rule is about is a case of it: either
|
|
227
|
+
* counts by the judge's verdict, and waits for one until then.
|
|
228
|
+
*/
|
|
229
|
+
export function counter(rulesOf, verdictOf = () => null) {
|
|
230
|
+
const tallies = new Map();
|
|
231
|
+
let inEffect = new Map();
|
|
232
|
+
let checked = []; // the tool and language rules in effect
|
|
233
|
+
let finders = [];
|
|
234
|
+
let replying = [];
|
|
235
|
+
// Resumed and forked sessions copy history into new files: an event counts once. Only events a rule applied to are
|
|
236
|
+
// remembered.
|
|
237
|
+
const seen = new Set();
|
|
238
|
+
let session = '';
|
|
239
|
+
let agent = null;
|
|
240
|
+
// Every version of every section seen, with the last time it was in effect during a call or reply; and those of the
|
|
241
|
+
// files in effect now.
|
|
242
|
+
const versions = new Map();
|
|
243
|
+
let current = [];
|
|
244
|
+
const files = new Set();
|
|
245
|
+
let sessions = 0;
|
|
246
|
+
let subagents = 0;
|
|
247
|
+
let active = false;
|
|
248
|
+
let from = null;
|
|
249
|
+
let to = null;
|
|
250
|
+
const excerpt = excerpts();
|
|
251
|
+
function refresh() {
|
|
252
|
+
const ids = new Set();
|
|
253
|
+
const groups = new Map();
|
|
254
|
+
const whens = new Map();
|
|
255
|
+
checked = [];
|
|
256
|
+
replying = [];
|
|
257
|
+
current = [];
|
|
258
|
+
for (const file of inEffect.values()) {
|
|
259
|
+
const found = rulesOf(file);
|
|
260
|
+
if (!found)
|
|
261
|
+
continue;
|
|
262
|
+
files.add(file.path);
|
|
263
|
+
for (const v of found.versions) {
|
|
264
|
+
const key = `${v.identity}\0${v.section.hash}`;
|
|
265
|
+
let seen = versions.get(key);
|
|
266
|
+
if (!seen)
|
|
267
|
+
versions.set(key, (seen = { ...v, at: '' }));
|
|
268
|
+
current.push(seen);
|
|
269
|
+
}
|
|
270
|
+
for (const rule of found.rules) {
|
|
271
|
+
let t = tallies.get(rule.id);
|
|
272
|
+
if (!t) {
|
|
273
|
+
// verify demoted every tool rule, and gave no cases to every judgment rule, whose check doesn't compile
|
|
274
|
+
const compiled = (rule.kind === 'tool' && !rule.demoted)
|
|
275
|
+
|| (rule.kind === 'judgment' && !rule.noCases && rule.check?.reply !== true)
|
|
276
|
+
? compile(rule.check) : null;
|
|
277
|
+
t = { rule, files: new Set(), all: zero(), after: zero(), depth: DEPTHS.map(zero), evidence: [], compiled, judged: 0,
|
|
278
|
+
unjudged: 0, waiting: [] };
|
|
279
|
+
tallies.set(rule.id, t);
|
|
280
|
+
}
|
|
281
|
+
t.files.add(file.path);
|
|
282
|
+
// A quote read two ways (two versions of a section) is one rule: it is counted as it was first read, which its
|
|
283
|
+
// tally's check was compiled from.
|
|
284
|
+
const { demoted, noCases, kind, check } = t.rule;
|
|
285
|
+
if (!ids.has(rule.id) && !demoted && !noCases) {
|
|
286
|
+
ids.add(rule.id);
|
|
287
|
+
if (kind !== 'judgment')
|
|
288
|
+
checked.push(t);
|
|
289
|
+
else if (check?.reply === true)
|
|
290
|
+
replying.push(t);
|
|
291
|
+
else {
|
|
292
|
+
const { field, when, ignoreCase } = check;
|
|
293
|
+
const key = JSON.stringify([field, when?.tool, ignoreCase === true]);
|
|
294
|
+
let group = groups.get(key);
|
|
295
|
+
if (!group)
|
|
296
|
+
groups.set(key, (group = { field, tool: t.compiled.when.tool, whens: [] }));
|
|
297
|
+
const same = JSON.stringify([key, when?.value, when?.data]);
|
|
298
|
+
const w = whens.get(same);
|
|
299
|
+
if (w)
|
|
300
|
+
w.tallies.push(t);
|
|
301
|
+
else
|
|
302
|
+
group.whens.push(whens.set(same, { when: t.compiled.when, tallies: [t] }).get(same));
|
|
303
|
+
}
|
|
304
|
+
}
|
|
305
|
+
}
|
|
306
|
+
}
|
|
307
|
+
finders = [...groups.values()];
|
|
308
|
+
}
|
|
309
|
+
/** Of each section, the version in effect latest. */
|
|
310
|
+
function latest() {
|
|
311
|
+
const last = new Map();
|
|
312
|
+
for (const v of versions.values()) {
|
|
313
|
+
const l = last.get(v.identity);
|
|
314
|
+
if (!l || v.at > l.at || (v.at === l.at && v.read))
|
|
315
|
+
last.set(v.identity, v);
|
|
316
|
+
}
|
|
317
|
+
return [...last.values()];
|
|
318
|
+
}
|
|
319
|
+
/**
|
|
320
|
+
* The sections to read: of each section, the version in effect latest when it wasn't read, unless a version of it
|
|
321
|
+
* read already was in effect as late. The others take that version's rules meanwhile.
|
|
322
|
+
*/
|
|
323
|
+
function unread() {
|
|
324
|
+
return latest().filter((v) => !v.read && (v.readAt === null || v.at > v.readAt));
|
|
325
|
+
}
|
|
326
|
+
function pending() {
|
|
327
|
+
return unread().map(({ section, identity, at }) => ({ section, identity, at }));
|
|
328
|
+
}
|
|
329
|
+
/** The versions read, with when they were last in effect: what the cache remembers moves forward with them. */
|
|
330
|
+
function readVersions() {
|
|
331
|
+
return latest().filter((v) => v.read && v.at).map(({ section, identity, at }) => ({ section, identity, at }));
|
|
332
|
+
}
|
|
333
|
+
/** One more case a rule applied to; a violation's evidence, as `text` says it. */
|
|
334
|
+
function record(t, e, broken, text, quote) {
|
|
335
|
+
add(t.all, broken);
|
|
336
|
+
if (e.compactions > 0)
|
|
337
|
+
add(t.after, broken);
|
|
338
|
+
add(t.depth[DEPTHS.findLastIndex((d) => e.depth >= d)], broken);
|
|
339
|
+
if (broken)
|
|
340
|
+
t.evidence.push({ at: e.at, session, agent, text: text(), ...(quote ? { quote } : {}) });
|
|
341
|
+
}
|
|
342
|
+
/** A case the judge decides, a tool rule's candidate or a judgment rule's case: counted by its verdict, or waiting. */
|
|
343
|
+
function decide(t, e) {
|
|
344
|
+
seen.add(e.id);
|
|
345
|
+
const x = excerpt.of(e); // its text is built only when read
|
|
346
|
+
const v = verdictOf(t.rule.id, x.key);
|
|
347
|
+
if (!v) {
|
|
348
|
+
t.unjudged++;
|
|
349
|
+
// A rule whose sample is full takes no more cases; one stopped by its first 8 still does, as it may open again.
|
|
350
|
+
if (open(t))
|
|
351
|
+
wait(t.waiting, x);
|
|
352
|
+
return;
|
|
353
|
+
}
|
|
354
|
+
t.judged++;
|
|
355
|
+
if (v.verdict === 'none')
|
|
356
|
+
return;
|
|
357
|
+
// A judgment rule's evidence is its case's own line; a tool rule's, what in the call broke it.
|
|
358
|
+
const own = () => {
|
|
359
|
+
const line = x.text.slice(x.text.lastIndexOf('\n') + 1);
|
|
360
|
+
return line.length > 200 ? `${line.slice(0, 200)}…` : line;
|
|
361
|
+
};
|
|
362
|
+
record(t, e, v.verdict === 'broken', t.rule.kind === 'judgment' ? own : () => evidence(e, t.compiled, null), v.quote);
|
|
363
|
+
}
|
|
364
|
+
function count(e) {
|
|
365
|
+
// A session or subagent counts once it did something: a transcript with no call or reply isn't one.
|
|
366
|
+
if (!active) {
|
|
367
|
+
active = true;
|
|
368
|
+
if (agent)
|
|
369
|
+
subagents++;
|
|
370
|
+
else
|
|
371
|
+
sessions++;
|
|
372
|
+
}
|
|
373
|
+
if (!from || e.at < from)
|
|
374
|
+
from = e.at;
|
|
375
|
+
if (!to || e.at > to)
|
|
376
|
+
to = e.at;
|
|
377
|
+
let detected;
|
|
378
|
+
for (const t of checked) {
|
|
379
|
+
let v = 'none';
|
|
380
|
+
if (e.kind === 'call' && t.compiled)
|
|
381
|
+
v = verdict(t.compiled, e.tool, e.input);
|
|
382
|
+
else if (e.kind === 'reply' && t.rule.kind === 'language') {
|
|
383
|
+
detected ??= language(e.text);
|
|
384
|
+
if (detected)
|
|
385
|
+
v = detected === t.rule.check?.language ? 'kept' : 'broken';
|
|
386
|
+
}
|
|
387
|
+
if (v === 'none')
|
|
388
|
+
continue;
|
|
389
|
+
seen.add(e.id);
|
|
390
|
+
// A tool rule's patterns find candidates: whether one broke the rule is the judge's to say.
|
|
391
|
+
if (v === 'broken' && t.compiled)
|
|
392
|
+
decide(t, e);
|
|
393
|
+
else
|
|
394
|
+
record(t, e, v === 'broken', () => evidence(e, null, detected ?? null));
|
|
395
|
+
}
|
|
396
|
+
if (e.kind === 'reply')
|
|
397
|
+
for (const t of replying)
|
|
398
|
+
decide(t, e);
|
|
399
|
+
// The agent's own files (its memory) are no case of the user's rules about their project.
|
|
400
|
+
else if (!e.own) {
|
|
401
|
+
for (const f of finders) {
|
|
402
|
+
if (f.tool && !f.tool.test(e.tool))
|
|
403
|
+
continue;
|
|
404
|
+
const all = views(f.field, e.input);
|
|
405
|
+
for (const w of f.whens)
|
|
406
|
+
if (found(w.when, all))
|
|
407
|
+
for (const t of w.tallies)
|
|
408
|
+
decide(t, e);
|
|
409
|
+
}
|
|
410
|
+
}
|
|
411
|
+
}
|
|
412
|
+
return {
|
|
413
|
+
on(e) {
|
|
414
|
+
if (e.kind === 'transcript') {
|
|
415
|
+
({ session, agent } = e);
|
|
416
|
+
active = false;
|
|
417
|
+
inEffect = new Map();
|
|
418
|
+
checked = [];
|
|
419
|
+
finders = [];
|
|
420
|
+
replying = [];
|
|
421
|
+
current = [];
|
|
422
|
+
}
|
|
423
|
+
else if (e.kind === 'instructions') {
|
|
424
|
+
if (e.replace)
|
|
425
|
+
inEffect = new Map();
|
|
426
|
+
for (const file of e.files)
|
|
427
|
+
inEffect.set(file.path, file);
|
|
428
|
+
refresh();
|
|
429
|
+
}
|
|
430
|
+
else if (e.kind === 'call' || e.kind === 'reply') {
|
|
431
|
+
for (const c of current)
|
|
432
|
+
if (e.at > c.at)
|
|
433
|
+
c.at = e.at;
|
|
434
|
+
if (!seen.has(e.id))
|
|
435
|
+
count(e);
|
|
436
|
+
}
|
|
437
|
+
// Copies are counted once, but their events still lead to the ones after them.
|
|
438
|
+
excerpt.push(e);
|
|
439
|
+
},
|
|
440
|
+
/** Per rule, its latest cases without a verdict: for a judgment rule, only as many as its sample still takes. */
|
|
441
|
+
waiting() {
|
|
442
|
+
const of = (kind) => [...tallies.values()].filter((t) => t.rule.kind === kind).map((t) => {
|
|
443
|
+
trim(t.waiting);
|
|
444
|
+
return t.waiting.slice(0, allowed(t)).map((x) => ({ id: `${t.rule.id}-${x.key}`, rule: t.rule, excerpt: x.text, at: x.at }));
|
|
445
|
+
}).filter((l) => l.length);
|
|
446
|
+
return { tool: of('tool'), judgment: of('judgment') };
|
|
447
|
+
},
|
|
448
|
+
pending,
|
|
449
|
+
readVersions,
|
|
450
|
+
result() {
|
|
451
|
+
const rules = [...tallies.values()].map((t) => {
|
|
452
|
+
const { rule, files, all, after, depth, evidence, judged, unjudged } = t;
|
|
453
|
+
return {
|
|
454
|
+
id: rule.id,
|
|
455
|
+
name: rule.name,
|
|
456
|
+
quote: rule.quote,
|
|
457
|
+
files: [...files],
|
|
458
|
+
heading: rule.heading,
|
|
459
|
+
lines: {},
|
|
460
|
+
status: rule.kind === 'judgment' ? 'judgment' : rule.demoted ? 'demoted'
|
|
461
|
+
: all.broken ? 'broken' : all.applied ? 'kept' : unjudged ? 'to-confirm' : 'never-applied',
|
|
462
|
+
...(rule.demoted || rule.noCases ? { reason: rule.demoted ?? rule.noCases } : {}),
|
|
463
|
+
...(rule.check ? { check: rule.check } : {}),
|
|
464
|
+
applied: all.applied,
|
|
465
|
+
broken: all.broken,
|
|
466
|
+
judged,
|
|
467
|
+
toJudge: Math.min(allowed(t), unjudged),
|
|
468
|
+
afterCompaction: after,
|
|
469
|
+
byDepth: buckets(depth),
|
|
470
|
+
tokens: Math.ceil(rule.quote.length / 4),
|
|
471
|
+
evidence: evidence.toReversed().sort((a, b) => (a.at < b.at ? 1 : a.at > b.at ? -1 : 0)),
|
|
472
|
+
};
|
|
473
|
+
});
|
|
474
|
+
// Broken rules first, the least kept leading; then kept, to confirm, never applied, judgment, demoted.
|
|
475
|
+
const rank = ['broken', 'kept', 'to-confirm', 'never-applied', 'judgment', 'demoted'];
|
|
476
|
+
const keptShare = (r) => (r.applied ? (r.applied - r.broken) / r.applied : 0);
|
|
477
|
+
rules.sort((a, b) => rank.indexOf(a.status) - rank.indexOf(b.status) || keptShare(a) - keptShare(b)
|
|
478
|
+
|| b.applied - a.applied);
|
|
479
|
+
const sum = DEPTHS.map(zero);
|
|
480
|
+
for (const t of tallies.values()) {
|
|
481
|
+
t.depth.forEach((c, i) => {
|
|
482
|
+
sum[i].applied += c.applied;
|
|
483
|
+
sum[i].broken += c.broken;
|
|
484
|
+
});
|
|
485
|
+
}
|
|
486
|
+
// Sections by their text: a worktree's copy of a file is one section to read. A text with rules from before under
|
|
487
|
+
// one of its paths is one to read again, not also one never read.
|
|
488
|
+
const reread = new Set(unread().filter((v) => v.before).map((v) => v.section.hash));
|
|
489
|
+
const never = new Set(unread().filter((v) => !reread.has(v.section.hash)).map((v) => v.section.hash));
|
|
490
|
+
return { from, to, sessions, subagents, files: [...files], notRead: never.size, toReread: reread.size,
|
|
491
|
+
toJudge: rules.reduce((n, r) => n + r.toJudge, 0), atMost: [...tallies.values()].reduce((n, t) => n + most(t), 0),
|
|
492
|
+
rules, byDepth: buckets(sum) };
|
|
493
|
+
},
|
|
494
|
+
};
|
|
495
|
+
}
|