@autor3search/javascript 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +500 -0
- package/bin/autor3search-javascript.js +4 -0
- package/package.json +50 -0
- package/src/adapters/bench/index.js +31 -0
- package/src/adapters/bench/vitest.js +128 -0
- package/src/adapters/driver-child.js +72 -0
- package/src/adapters/driver-hooks.js +18 -0
- package/src/adapters/driver.js +192 -0
- package/src/adapters/gates/index.js +64 -0
- package/src/adapters/gates/lint.js +46 -0
- package/src/adapters/gates/test.js +27 -0
- package/src/adapters/gates/typecheck.js +98 -0
- package/src/adapters/gates/util.js +45 -0
- package/src/adapters/vitest-shim.js +51 -0
- package/src/bench/parse.js +120 -0
- package/src/bench/set.js +101 -0
- package/src/bench/stats.js +443 -0
- package/src/cli/cmd-baseline.js +136 -0
- package/src/cli/cmd-doctor.js +38 -0
- package/src/cli/cmd-eval.js +231 -0
- package/src/cli/cmd-init.js +136 -0
- package/src/cli/cmd-profile.js +38 -0
- package/src/cli/cmd-report.js +96 -0
- package/src/cli/cmd-status.js +68 -0
- package/src/cli/cmd-stop.js +98 -0
- package/src/cli/cmd-version.js +31 -0
- package/src/cli/context.js +98 -0
- package/src/cli/main.js +62 -0
- package/src/config.js +209 -0
- package/src/discover.js +239 -0
- package/src/doctor.js +282 -0
- package/src/duration.js +75 -0
- package/src/freeze.js +234 -0
- package/src/gitx.js +115 -0
- package/src/measure.js +127 -0
- package/src/pipeline.js +391 -0
- package/src/profile.js +100 -0
- package/src/results.js +124 -0
- package/src/runner.js +198 -0
- package/src/scope.js +92 -0
- package/src/state/index.js +312 -0
- package/src/state/lock.js +189 -0
- package/src/state/stop.js +56 -0
- package/src/verdict.js +214 -0
- package/templates/program.md +264 -0
package/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Gal Be
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
package/README.md
ADDED
|
@@ -0,0 +1,500 @@
|
|
|
1
|
+
# autor3search-javascript
|
|
2
|
+
|
|
3
|
+
[](https://www.npmjs.com/package/@autor3search/javascript)
|
|
4
|
+
[](https://nodejs.org)
|
|
5
|
+
[](LICENSE)
|
|
6
|
+
|
|
7
|
+
An autonomous performance-optimization harness for JavaScript repositories. You
|
|
8
|
+
point a coding agent at your repository, run four commands, hand the agent
|
|
9
|
+
`program.md`, and go to sleep. The agent proposes one performance change at a
|
|
10
|
+
time; the harness gates it for correctness, measures it against a pinned
|
|
11
|
+
baseline, and returns a verdict — `KEEP` or `DISCARD` — that the agent cannot
|
|
12
|
+
argue with, weaken, or reinterpret. In the morning `report` tells you what
|
|
13
|
+
happened.
|
|
14
|
+
|
|
15
|
+
A standalone JavaScript tool: it discovers Vitest benchmarks, measures them,
|
|
16
|
+
and decides. It needs nothing but Node and the repository you point it at.
|
|
17
|
+
The discipline it enforces — frozen tests, out-of-tree state, interleaved A/B
|
|
18
|
+
measurement, an honest significance test, and a single scalar the agent
|
|
19
|
+
cannot game — comes from the autoresearch line of harnesses, and is
|
|
20
|
+
implemented here from scratch for the JavaScript toolchain. What that
|
|
21
|
+
toolchain can and cannot guarantee is stated in full in
|
|
22
|
+
[Limitations](#limitations) below.
|
|
23
|
+
|
|
24
|
+
## Install
|
|
25
|
+
|
|
26
|
+
```bash
|
|
27
|
+
npm install -g @autor3search/javascript
|
|
28
|
+
```
|
|
29
|
+
|
|
30
|
+
Or run it without installing anything, which is the better option when you
|
|
31
|
+
want a specific version pinned for a run:
|
|
32
|
+
|
|
33
|
+
```bash
|
|
34
|
+
npx @autor3search/javascript@0.2.0 doctor
|
|
35
|
+
```
|
|
36
|
+
|
|
37
|
+
Node 20 or newer. The harness measures with Vitest, so the repository you
|
|
38
|
+
point it at needs `vitest` installed — `doctor` checks for it and says where
|
|
39
|
+
it looked if it is missing.
|
|
40
|
+
|
|
41
|
+
## Start here
|
|
42
|
+
|
|
43
|
+
Open your coding agent inside the JavaScript repository you want to make
|
|
44
|
+
faster, and paste this:
|
|
45
|
+
|
|
46
|
+
```text
|
|
47
|
+
Install and run autor3search-javascript on this repository, then optimize it.
|
|
48
|
+
|
|
49
|
+
Setup:
|
|
50
|
+
1. npm install -g @autor3search/javascript
|
|
51
|
+
Or prefix every command below with `npx` instead — either way, keep using
|
|
52
|
+
the same form for the whole run, so the version that recorded the baseline
|
|
53
|
+
is the version that scores against it.
|
|
54
|
+
2. autor3search-javascript init
|
|
55
|
+
Show me the benchmarks it discovered. If it reports none, STOP and tell me:
|
|
56
|
+
this tool can only optimize what it can measure.
|
|
57
|
+
3. Review .autor3search/config.yaml and program.md, then:
|
|
58
|
+
git add -A && git commit -m "autor3search-javascript init"
|
|
59
|
+
4. autor3search-javascript doctor
|
|
60
|
+
Show me any warnings. If the machine looks unfit to measure, stop and ask
|
|
61
|
+
me before continuing.
|
|
62
|
+
5. autor3search-javascript baseline -tag <today, e.g. sep7>
|
|
63
|
+
|
|
64
|
+
Then:
|
|
65
|
+
6. Read program.md in this repository, in full. It is your instruction set for
|
|
66
|
+
the rest of this run. Follow it exactly.
|
|
67
|
+
|
|
68
|
+
Rules for the whole run:
|
|
69
|
+
- One hypothesis per commit. Commit before each experiment, then run
|
|
70
|
+
`autor3search-javascript eval --json -desc "<idea>"` and apply its verdict
|
|
71
|
+
before touching anything else: KEEP means the commit stays; anything else
|
|
72
|
+
(DISCARD, FAIL, CRASH, ABORTED) means `git reset --hard HEAD~1`.
|
|
73
|
+
- Never redirect eval's stdout into run.log. The harness already holds that
|
|
74
|
+
file open, and a second writer destroys the transcript you need when
|
|
75
|
+
something FAILs.
|
|
76
|
+
- Never edit program.md, .autor3search/config.yaml, results.tsv, any
|
|
77
|
+
*.test.* / *.spec.* / *.bench.* file, package.json, a lockfile, or a
|
|
78
|
+
Vitest/Vite config file. They are not yours.
|
|
79
|
+
- Never pass -force to any autor3search-javascript command. (I may run
|
|
80
|
+
`autor3search-javascript stop -force` myself; that one is mine, not yours.)
|
|
81
|
+
- Print one context line before each experiment, so I can see where you are:
|
|
82
|
+
[exp <n> | <branch> | vs <measure_commit> | stop: autor3search-javascript stop]
|
|
83
|
+
|
|
84
|
+
Run the loop until I stop you. I stop you by running
|
|
85
|
+
`autor3search-javascript stop` in my own terminal — you will see it as
|
|
86
|
+
"stop_requested": true in a verdict. When you do: apply that verdict, do not
|
|
87
|
+
start another experiment, run `autor3search-javascript report`, summarize what
|
|
88
|
+
you tried, and exit the loop.
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
That's the whole handoff. The agent installs the tool, discovers your
|
|
92
|
+
benchmarks, freezes a baseline, and then follows `program.md` — generated for
|
|
93
|
+
your repository by `init` — which tells it how to run the keep-or-discard loop.
|
|
94
|
+
`program.md` names the benchmarks in scope, spells out the
|
|
95
|
+
KEEP/DISCARD/FAIL/CRASH contract, lists everything the agent must never touch,
|
|
96
|
+
and ends with a bank of generic V8/JavaScript performance ideas for when the
|
|
97
|
+
agent is out of hypotheses.
|
|
98
|
+
|
|
99
|
+
What you get back: one commit per accepted change on a branch named
|
|
100
|
+
`autor3search-javascript/<tag>`, and a `results.tsv` recording every experiment
|
|
101
|
+
that was tried, including the ones that failed.
|
|
102
|
+
`autor3search-javascript report` summarizes it.
|
|
103
|
+
|
|
104
|
+
Two things worth knowing before you start it:
|
|
105
|
+
|
|
106
|
+
- **It needs benchmarks.** This optimizes what it can measure, and refuses to
|
|
107
|
+
guess: `init` looks for Vitest `bench()` calls in `*.bench.*` files and
|
|
108
|
+
refuses outright when it finds none.
|
|
109
|
+
- **Numbers are only as good as the machine.** Run `doctor` and read it. A
|
|
110
|
+
thermally throttled laptop on battery produces noise dressed as data, and a
|
|
111
|
+
JS runtime adds JIT tier-up and GC scheduling on top — see
|
|
112
|
+
[Limitations](#limitations) for how much.
|
|
113
|
+
|
|
114
|
+
Everything past this point is for the human setting the run up, or for
|
|
115
|
+
understanding what the agent in step 6 is actually bound by.
|
|
116
|
+
|
|
117
|
+
## The idea
|
|
118
|
+
|
|
119
|
+
| Owns | Belongs to |
|
|
120
|
+
|---|---|
|
|
121
|
+
| Gating (scope, config integrity, frozen files, correctness), measuring, scoring | the harness |
|
|
122
|
+
| Your application source | the agent, inside `scope` |
|
|
123
|
+
| `program.md` and `.autor3search/config.yaml` | you |
|
|
124
|
+
| Frozen test/bench copies, the baseline record, the pinned measurement worktree | the user cache, outside the repository |
|
|
125
|
+
|
|
126
|
+
`*.test.*`, `*.spec.*` and `*.bench.*` files are frozen at `baseline` time and
|
|
127
|
+
restored before every `eval`. The agent may read them, may complain about
|
|
128
|
+
them in its `-desc`, but cannot change what they check or what they measure.
|
|
129
|
+
Everything the verdict depends on lives outside the repository the agent is
|
|
130
|
+
editing. The scope gate also rejects dependency files and any Vitest/Vite
|
|
131
|
+
config outright, because those are loaded by the bench runner itself — an
|
|
132
|
+
ordinary, in-scope config file could otherwise redirect what a frozen
|
|
133
|
+
benchmark imports, or stub out the code path it measures, without the frozen
|
|
134
|
+
copy changing by one byte. That closes the config route specifically; it is
|
|
135
|
+
not a claim that nothing else the agent does inside the repo can move the
|
|
136
|
+
goalposts — see [Limitations](#limitations) for what is still open.
|
|
137
|
+
|
|
138
|
+
## Quick start
|
|
139
|
+
|
|
140
|
+
```
|
|
141
|
+
autor3search-javascript init # discovers benchmarks, writes config + program.md
|
|
142
|
+
git add -A && git commit -m "autor3search-javascript init"
|
|
143
|
+
autor3search-javascript doctor # is this machine fit to measure on?
|
|
144
|
+
autor3search-javascript baseline -tag sep8 # creates the run branch, freezes tests+benches, pins HEAD
|
|
145
|
+
```
|
|
146
|
+
|
|
147
|
+
The commit between `init` and `baseline` matters: `baseline` refuses a dirty
|
|
148
|
+
working tree, because a baseline pinned against what's on disk rather than
|
|
149
|
+
what's in git could never be reproduced — the pinned worktree it creates is a
|
|
150
|
+
git worktree checked out at a real commit, and there has to be one.
|
|
151
|
+
|
|
152
|
+
## Watching a run, and stopping it
|
|
153
|
+
|
|
154
|
+
`autor3search-javascript status` shows where things stand without touching
|
|
155
|
+
anything:
|
|
156
|
+
|
|
157
|
+
```
|
|
158
|
+
run tag demo
|
|
159
|
+
branch autor3search-javascript/demo (checked out)
|
|
160
|
+
baseline bdf3c4c (run started here)
|
|
161
|
+
measuring vs 50c69c7 (advanced past the baseline by earlier KEEPs)
|
|
162
|
+
worktree /Users/you/Library/Caches/autor3search-javascript/.../baseline-worktree
|
|
163
|
+
experiments 1 run (1 keep, 0 discard, 0 fail, 0 crash) — next is #2
|
|
164
|
+
eval idle
|
|
165
|
+
stop not requested
|
|
166
|
+
```
|
|
167
|
+
|
|
168
|
+
Three ways to end a run, in increasing order of force:
|
|
169
|
+
|
|
170
|
+
1. **`autor3search-javascript stop`** — a graceful request. The experiment
|
|
171
|
+
under way finishes, is measured and scored normally; the agent sees
|
|
172
|
+
`"stop_requested": true` on that verdict, applies it as usual (`KEEP`
|
|
173
|
+
stays, anything else resets), then exits the loop. Nothing is thrown away.
|
|
174
|
+
2. **`autor3search-javascript stop --force`** — writes the same request, then
|
|
175
|
+
sends `SIGTERM` to the running `eval`, which tears down its own child
|
|
176
|
+
process groups so no Vitest worker is left burning CPU. The agent sees
|
|
177
|
+
`"status": "ABORTED"`, exit code 2, and no `results.tsv` row — nothing was
|
|
178
|
+
measured, so nothing was recorded. `stop --force` reports what HEAD looks
|
|
179
|
+
like afterward; it does not touch the repository for you.
|
|
180
|
+
3. **Ctrl+C** — the same abort path as `stop --force`, sent directly to a
|
|
181
|
+
foreground `eval`.
|
|
182
|
+
|
|
183
|
+
`autor3search-javascript stop --clear` cancels a pending stop request so the
|
|
184
|
+
loop continues — that is your decision, never the agent's to make on its own.
|
|
185
|
+
|
|
186
|
+
## Commands
|
|
187
|
+
|
|
188
|
+
| Command | Does |
|
|
189
|
+
|---|---|
|
|
190
|
+
| `init` | Scans the repo, discovers benchmarks by parsing (not running) them, writes `.autor3search/config.yaml` and `program.md`. Refuses if it finds no benchmarks. |
|
|
191
|
+
| `doctor` | Reports whether this machine can measure reliably: Node/git versions, CPU count and load, whether Vitest resolves, power state, thermal state, disk space. Informational only — always exits 0. |
|
|
192
|
+
| `baseline` | Creates the run branch, freezes every test and bench file as a golden copy, pins a git worktree at the current commit, records the config hash. Refuses a dirty tree, an existing `results.tsv`, a reused tag, or a repo with no benchmarks. |
|
|
193
|
+
| `profile` | Runs the declared benchmarks under `--cpu-prof`/`--heap-prof` and prints the hottest self-time functions, plus where the raw profiles were written. |
|
|
194
|
+
| `eval` | Runs one experiment: scope check, config-integrity check, restores any frozen file the agent touched, runs the correctness gates, measures candidate against baseline (interleaved), scores, and returns a verdict. The only command that decides anything. |
|
|
195
|
+
| `status` | Read-only snapshot: branch, baseline, measurement pointer, worktree, experiment counts, whether `eval` is running, whether a stop is pending. |
|
|
196
|
+
| `stop` | Requests (or, with `--force`, forces) the run to end. `--clear` cancels a pending request. |
|
|
197
|
+
| `report` | Summarizes `results.tsv`: counts by status, cumulative speedup (the product of every kept score), and the largest individual wins. |
|
|
198
|
+
| `version` | Prints the installed version and commit. |
|
|
199
|
+
|
|
200
|
+
Every command accepts `-C <dir>` to run against a repository other than the
|
|
201
|
+
current directory.
|
|
202
|
+
|
|
203
|
+
## Where run state lives
|
|
204
|
+
|
|
205
|
+
Everything the score depends on — the frozen golden copies, the baseline
|
|
206
|
+
record, the pinned measurement worktree — is written to:
|
|
207
|
+
|
|
208
|
+
```
|
|
209
|
+
<user cache>/autor3search-javascript/<repo hash>/<tag>/
|
|
210
|
+
```
|
|
211
|
+
|
|
212
|
+
(`~/Library/Caches` on macOS, `$XDG_CACHE_HOME` or `~/.cache` on Linux,
|
|
213
|
+
`%LOCALAPPDATA%` on Windows). `<repo hash>` is a hash of the repository's real
|
|
214
|
+
absolute path, so two checkouts of the same project never collide and never
|
|
215
|
+
share state.
|
|
216
|
+
|
|
217
|
+
Set `AUTOR3SEARCH_JAVASCRIPT_STATE_HOME` to relocate it — a relative path is
|
|
218
|
+
refused outright, because it would resolve differently depending on which
|
|
219
|
+
directory each command happened to be run from, and `eval` run from a
|
|
220
|
+
subdirectory would then silently address different state than `stop` run from
|
|
221
|
+
the repository root.
|
|
222
|
+
|
|
223
|
+
Every directory the harness creates there is mode `0700`, and on POSIX systems
|
|
224
|
+
each level from the state home down is checked before it is used: a directory
|
|
225
|
+
owned by another user, or one that group or others can write, is refused with
|
|
226
|
+
the `chmod` that fixes it. This matters because the frozen store and its
|
|
227
|
+
manifest both live here — whoever can write to them can replace the benchmarks
|
|
228
|
+
the score is measured against, consistently enough that the hash check still
|
|
229
|
+
passes. Under the default cache location the parent already restricts access;
|
|
230
|
+
the check is what makes `AUTOR3SEARCH_JAVASCRIPT_STATE_HOME` safe to point at a
|
|
231
|
+
shared directory. Windows reports synthetic mode bits and has no owner to
|
|
232
|
+
compare against, so the check is skipped there rather than made to look like it
|
|
233
|
+
ran.
|
|
234
|
+
|
|
235
|
+
## Worked example
|
|
236
|
+
|
|
237
|
+
`test/e2e.test.js` builds this exact scenario from
|
|
238
|
+
`test/helpers/bench-repo.js`; `testdata/demo/README.md` documents it in the
|
|
239
|
+
repository. A word counter, written badly on purpose:
|
|
240
|
+
|
|
241
|
+
```js
|
|
242
|
+
// the slow version
|
|
243
|
+
export function countWords(s) {
|
|
244
|
+
const counts = {}
|
|
245
|
+
for (const field of s.split(/\s+/)) {
|
|
246
|
+
let word = ''
|
|
247
|
+
for (const ch of field) {
|
|
248
|
+
const lower = ch.toLowerCase()
|
|
249
|
+
if (/[a-z0-9]/.test(lower)) word = word + lower
|
|
250
|
+
}
|
|
251
|
+
if (word !== '') counts[word] = (counts[word] ?? 0) + 1
|
|
252
|
+
}
|
|
253
|
+
return counts
|
|
254
|
+
}
|
|
255
|
+
```
|
|
256
|
+
|
|
257
|
+
with a frozen test (`src/wordcount.test.js`) checking its behaviour and a
|
|
258
|
+
frozen benchmark (`src/wordcount.bench.js`) measuring it on a fixed input.
|
|
259
|
+
Both are restored before every `eval`, so a candidate has to keep the exact
|
|
260
|
+
same behaviour to be scored at all. The candidate:
|
|
261
|
+
|
|
262
|
+
```js
|
|
263
|
+
// the fast version
|
|
264
|
+
export function countWords(s) {
|
|
265
|
+
const counts = new Map()
|
|
266
|
+
for (const field of s.split(' ')) {
|
|
267
|
+
let word = ''
|
|
268
|
+
for (let i = 0; i < field.length; i++) {
|
|
269
|
+
const c = field.charCodeAt(i)
|
|
270
|
+
if (c >= 65 && c <= 90) word += String.fromCharCode(c + 32)
|
|
271
|
+
else if ((c >= 97 && c <= 122) || (c >= 48 && c <= 57)) word += field[i]
|
|
272
|
+
}
|
|
273
|
+
if (word !== '') counts.set(word, (counts.get(word) ?? 0) + 1)
|
|
274
|
+
}
|
|
275
|
+
return Object.fromEntries(counts)
|
|
276
|
+
}
|
|
277
|
+
```
|
|
278
|
+
|
|
279
|
+
replaces the regex-per-character scan and string concatenation with charcode
|
|
280
|
+
comparisons and a `Map`.
|
|
281
|
+
|
|
282
|
+
This is a real run, not an illustration — `init`, `doctor`, `baseline -tag
|
|
283
|
+
demo`, then the swap above, then `eval`, all against the code shown here, on
|
|
284
|
+
a MacBook Pro (Apple M5, 10 logical cores), macOS Darwin 25.6.0, Node
|
|
285
|
+
v22.23.1:
|
|
286
|
+
|
|
287
|
+
| | baseline | candidate | change | p |
|
|
288
|
+
|---|---|---|---|---|
|
|
289
|
+
| `countWords` (ns/op) | 168,105 | 77,333 | **−54.00%** | 0.00001 |
|
|
290
|
+
|
|
291
|
+
```
|
|
292
|
+
VERDICT: KEEP (improved) — score 0.4600 (-54.00%)
|
|
293
|
+
```
|
|
294
|
+
|
|
295
|
+
`score` is the geometric mean of `candidate/baseline` across the declared
|
|
296
|
+
benchmarks (here, one), so with a single benchmark it equals the ratio
|
|
297
|
+
directly: 0.46, i.e. 2.17x faster. Run it yourself — noise is real, and on a
|
|
298
|
+
busier machine or a shorter `count` this same diff can land as a `DISCARD`
|
|
299
|
+
instead; that is not a bug, it is the point.
|
|
300
|
+
|
|
301
|
+
## What the harness enforces
|
|
302
|
+
|
|
303
|
+
| Try this | The harness |
|
|
304
|
+
|---|---|
|
|
305
|
+
| Weaken a test | Restored from the frozen copy before every `eval` |
|
|
306
|
+
| Add an "easier" benchmark | Rejected: a new file matching `*.bench.*` not present at baseline is `new_test_file` |
|
|
307
|
+
| Rewrite the benchmark to measure something trivial | Restored too — bench files are frozen exactly like test files |
|
|
308
|
+
| Symlink a frozen file, or a directory on its path | Refused: `symlink_swap` |
|
|
309
|
+
| Hard-link over a frozen file | Refused: `hardlink_swap` |
|
|
310
|
+
| Edit a file outside `scope` | `scope_violation`, checked before anything is built or measured |
|
|
311
|
+
| Bank ordinary noise as a win | The Mann-Whitney test behind `significant` has to clear it first |
|
|
312
|
+
| Speed up A by wrecking B | The regression guard trips on B alone, discarding the whole change |
|
|
313
|
+
| Change a dependency (`package.json`, any lockfile) | Rejected outright, regardless of `scope` |
|
|
314
|
+
| Loosen `max_regress_pct` or `count` mid-run | `config.yaml`'s hash is pinned at `baseline`; any change fails with `config_changed` |
|
|
315
|
+
| Compare against a stale, cached baseline | Every `eval` re-measures both sides, interleaved, in the same process |
|
|
316
|
+
| Coast on an earlier win forever | The measurement baseline advances to the just-kept commit after every `KEEP`, so the next `eval` has to earn its own improvement |
|
|
317
|
+
|
|
318
|
+
## Scoring
|
|
319
|
+
|
|
320
|
+
`score` is the geometric mean, across the declared benchmarks, of
|
|
321
|
+
`candidate_time / baseline_time`:
|
|
322
|
+
|
|
323
|
+
```
|
|
324
|
+
score = exp( mean( log(candidate_i / baseline_i) ) ) for i in benchmarks
|
|
325
|
+
```
|
|
326
|
+
|
|
327
|
+
below 1 is faster. A `KEEP` requires **all** of:
|
|
328
|
+
|
|
329
|
+
1. **No regression guard trips.** Any benchmark that got significantly
|
|
330
|
+
*worse* — significant at the raw, uncorrected `alpha` (0.05) and past
|
|
331
|
+
`max_regress_pct` — discards the whole experiment, regardless of how good
|
|
332
|
+
the overall score is. This check deliberately skips the Bonferroni
|
|
333
|
+
correction used below: correcting here would only make the guard *less*
|
|
334
|
+
sensitive to harm, which is backwards for a guard. Be conservative about
|
|
335
|
+
banking a win, be liberal about catching damage — that asymmetry is
|
|
336
|
+
intentional.
|
|
337
|
+
2. **The score clears `1 - min_effect_pct/100`** (default 1%, so `score <
|
|
338
|
+
0.99`). A sub-1% shave is discarded by design as `improvement_below_min_effect`
|
|
339
|
+
even when it is real: not worth a commit in an unattended loop.
|
|
340
|
+
3. **At least one benchmark improved past a Bonferroni-corrected significance
|
|
341
|
+
bar**, `alpha / k` where `k` is the number of benchmarks compared.
|
|
342
|
+
Comparing several benchmarks against the same uncorrected `alpha` inflates
|
|
343
|
+
the chance that at least one shows a spurious "significant" improvement
|
|
344
|
+
even when nothing changed — that is what the correction is for.
|
|
345
|
+
|
|
346
|
+
## Releasing
|
|
347
|
+
|
|
348
|
+
Publishing runs from GitHub Actions with npm **trusted publishing** (OIDC).
|
|
349
|
+
There is no npm token in this repository, in its secrets, or on any
|
|
350
|
+
maintainer's machine: npm mints a short-lived credential from GitHub's own
|
|
351
|
+
identity for that one workflow run, and attaches a provenance attestation
|
|
352
|
+
linking the published tarball to the commit and run that built it.
|
|
353
|
+
|
|
354
|
+
To cut a release:
|
|
355
|
+
|
|
356
|
+
```bash
|
|
357
|
+
npm version patch # or minor / major — commits and tags
|
|
358
|
+
git push --follow-tags
|
|
359
|
+
```
|
|
360
|
+
|
|
361
|
+
The tag triggers `.github/workflows/release.yml`, which refuses to publish if
|
|
362
|
+
the tag and `package.json` disagree, and runs the full suite on Linux first.
|
|
363
|
+
The OIDC path is confirmed working: a run against an already-published version
|
|
364
|
+
authenticated and got as far as `cannot publish over the previously published
|
|
365
|
+
versions`, which is the failure you want to see — it means the credential
|
|
366
|
+
exchange succeeded and only the version was wrong.
|
|
367
|
+
|
|
368
|
+
Two things to know if you are wiring this up on a fork or a new package:
|
|
369
|
+
|
|
370
|
+
- The trusted publisher on npmjs.com names the **workflow filename**, so
|
|
371
|
+
renaming `release.yml` breaks publishing until the setting is updated.
|
|
372
|
+
- npm cannot publish a package's **first** version this way — a trusted
|
|
373
|
+
publisher can only be configured on a package that already exists
|
|
374
|
+
([npm/cli#8544](https://github.com/npm/cli/issues/8544)). That one publish
|
|
375
|
+
needs a token; every release after it is token-free.
|
|
376
|
+
|
|
377
|
+
## Platform support
|
|
378
|
+
|
|
379
|
+
| | |
|
|
380
|
+
|---|---|
|
|
381
|
+
| **Linux** | supported — CI runs the full suite on Node 20 and 22 |
|
|
382
|
+
| **macOS** | supported — CI runs the full suite on Node 20 and 22 |
|
|
383
|
+
| **Windows** | **not supported** |
|
|
384
|
+
|
|
385
|
+
On Windows 448 of 458 tests pass, so measurement itself works. What does not
|
|
386
|
+
work is stopping: Node cannot deliver SIGINT to a child process group there
|
|
387
|
+
the way it does on POSIX, so an interrupted `eval` never reaches the `ABORTED`
|
|
388
|
+
path — it exits with a null code instead of 2 and can leave its claim behind.
|
|
389
|
+
An unattended harness that cannot be reliably stopped is not something to be
|
|
390
|
+
quiet about, so `doctor` says so on Windows rather than letting you find out
|
|
391
|
+
at 3am. Two further failures are the test suite's own POSIX assumptions
|
|
392
|
+
(a `chmod`-unreadable directory, and path separators) rather than product bugs.
|
|
393
|
+
|
|
394
|
+
WSL reports as Linux and is unaffected. Node 20 or newer is required
|
|
395
|
+
everywhere.
|
|
396
|
+
|
|
397
|
+
## Limitations
|
|
398
|
+
|
|
399
|
+
Stated here rather than left for you to discover:
|
|
400
|
+
|
|
401
|
+
- **A `KEEP` is evidence, not proof.** Any significance threshold admits
|
|
402
|
+
false positives by construction — `alpha = 0.05` means a true no-op change
|
|
403
|
+
still looks "significant" one time in twenty, by design, however the
|
|
404
|
+
harness is implemented.
|
|
405
|
+
- **Laptops are noisy.** Background processes, thermal throttling, and on
|
|
406
|
+
Apple Silicon specifically, the scheduler moving work between performance
|
|
407
|
+
and efficiency cores, all show up as measurement noise, not signal.
|
|
408
|
+
`doctor` reports load, power source and thermal state and warns when they
|
|
409
|
+
look bad — it cannot fix any of them. If experiments look erratic on a
|
|
410
|
+
laptop, raise `min_effect_pct` or measure on a quiet machine instead.
|
|
411
|
+
- **No benchmarks, no value.** `init` refuses outright on a repository with
|
|
412
|
+
no `*.bench.*` file, rather than accepting a config with nothing to gate
|
|
413
|
+
on and pretending a verdict means something.
|
|
414
|
+
- **A small measurement asymmetry survives within a round.** Rounds
|
|
415
|
+
alternate which side runs first, cancelling drift *between* rounds, but
|
|
416
|
+
within a single round one side always runs first and the other second —
|
|
417
|
+
any bias monotonic across just that round (a CPU still ramping up, a
|
|
418
|
+
background job starting mid-measurement) lands on whichever side ran
|
|
419
|
+
second. Averaging over rounds does not remove this, because it is a
|
|
420
|
+
constant offset, not noise; that's why round counts are kept even.
|
|
421
|
+
- **Microbenchmarks are not your application.** A `bench()` that measures a
|
|
422
|
+
cold path or a trivial helper produces numbers that are entirely real and
|
|
423
|
+
entirely useless. Benchmark what actually dominates your workload.
|
|
424
|
+
- **`count` below 4 can never reach significance**, whatever the size of the
|
|
425
|
+
improvement — with 2 or 3 rounds per side the best achievable two-sided
|
|
426
|
+
p-value (0.333 or 0.1) is already above the default `alpha`. `config`
|
|
427
|
+
refuses a `count` under 4 rather than silently running a harness that can
|
|
428
|
+
never `KEEP`.
|
|
429
|
+
- **`bytes/op` is an approximate hint, never scored.** V8 exposes no
|
|
430
|
+
allocation counter, so it is a heap-size delta measured across a
|
|
431
|
+
forced-GC window — noisy by construction, since it can only see whatever
|
|
432
|
+
the GC left uncollected at the sampling point, not what was actually
|
|
433
|
+
allocated. It can also be entirely absent: the driver child needs
|
|
434
|
+
`--expose-gc`, and a bench file that fails to import or throws while
|
|
435
|
+
running is dropped from the hint rather than measured. Either way it is
|
|
436
|
+
silently omitted from the eval output rather than failing the run — a
|
|
437
|
+
hint that couldn't be measured must never fail a real, correctly-measured
|
|
438
|
+
experiment. It exists to point at allocation-heavy code, nothing more.
|
|
439
|
+
- **There is no `benchtime` setting.** Vitest exposes no global "run this
|
|
440
|
+
benchmark for N seconds" option, so per-benchmark duration lives in the
|
|
441
|
+
`bench()` call itself (via
|
|
442
|
+
Vitest's own `time`/`iterations` options in the bench file), not in
|
|
443
|
+
`.autor3search/config.yaml`. `count` controls how many measured rounds
|
|
444
|
+
the harness runs, not how long any one of them takes.
|
|
445
|
+
- **This harness is not a compiled binary.** It is an npm-installed
|
|
446
|
+
JavaScript program, running as the same OS user as the agent it's grading
|
|
447
|
+
— so an agent that decided to, could in principle edit this package's own
|
|
448
|
+
installed files. A harness the agent cannot reach at all would not have
|
|
449
|
+
that hole; this one has to close it by other means. Out-of-tree state (the frozen copies,
|
|
450
|
+
the baseline record, the pinned worktree, all living under the user
|
|
451
|
+
cache rather than the repository), together with rejecting dependency
|
|
452
|
+
files and known Vitest/Vite config filenames outright, protects the
|
|
453
|
+
metric from *most* of what the agent could do inside the repository being
|
|
454
|
+
optimized: editing its own tests or benchmarks, weakening the config that
|
|
455
|
+
constrains it, or redirecting the bench runner's import through a config
|
|
456
|
+
file. It does not close every such route — only the config filenames
|
|
457
|
+
Vitest 2.1.9 is known to load at the repository root are rejected, so a
|
|
458
|
+
bench runner reconfigured to load from somewhere else, a workspace-globbed
|
|
459
|
+
config in a subdirectory, or some other toolchain file this project has
|
|
460
|
+
not enumerated could still retarget what gets measured. That residual gap
|
|
461
|
+
is stated here rather than left for you to find out the hard way.
|
|
462
|
+
- **A narrow TOCTOU window exists in the freeze module.** `src/freeze.js`
|
|
463
|
+
checks that a frozen path (and every directory on the way to it) is not a
|
|
464
|
+
symlink or hard link, then reads or writes it. Between that check and the
|
|
465
|
+
write, an agent racing the harness in the same process tree could in
|
|
466
|
+
principle swap the path. This is theoretical under this project's threat
|
|
467
|
+
model — an unattended overnight agent isn't racing filesystem syscalls
|
|
468
|
+
against itself — but it is real, not closed by design, and is disclosed
|
|
469
|
+
here rather than implied away.
|
|
470
|
+
|
|
471
|
+
## Repos with no benchmarks
|
|
472
|
+
|
|
473
|
+
`autor3search-javascript init` refuses to write a config for a repository
|
|
474
|
+
with no `*.bench.*` file:
|
|
475
|
+
|
|
476
|
+
```
|
|
477
|
+
no benchmarks found in <repo>.
|
|
478
|
+
|
|
479
|
+
autor3search-javascript optimizes what it can measure, and refuses to guess. Write at least one
|
|
480
|
+
Vitest benchmark covering the code you want made faster, in a *.bench.js file:
|
|
481
|
+
|
|
482
|
+
import { bench } from 'vitest'
|
|
483
|
+
import { thing } from './thing.js'
|
|
484
|
+
|
|
485
|
+
bench('thing', () => { thing() })
|
|
486
|
+
|
|
487
|
+
Benchmark the path that actually dominates your workload — one that exercises a cold path or a
|
|
488
|
+
trivial helper produces numbers that are entirely real and entirely useless. Then run init again.
|
|
489
|
+
```
|
|
490
|
+
|
|
491
|
+
Write a `bench()` around the function that shows up at the top of a real
|
|
492
|
+
profile of your application, not around whatever is easiest to isolate. Once
|
|
493
|
+
`init` finds at least one, it discovers it by *parsing* the file (via
|
|
494
|
+
`@babel/parser`), not by running it — so this works even on a tree that does
|
|
495
|
+
not currently build, which matters because that is also when a candidate
|
|
496
|
+
most needs to fail loudly rather than be silently skipped.
|
|
497
|
+
|
|
498
|
+
## License
|
|
499
|
+
|
|
500
|
+
MIT © 2026 Gal Be
|
package/package.json
ADDED
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
{
|
|
2
|
+
"name": "@autor3search/javascript",
|
|
3
|
+
"version": "0.2.0",
|
|
4
|
+
"description": "Autonomous AI-driven performance optimization for any JavaScript repository",
|
|
5
|
+
"type": "module",
|
|
6
|
+
"license": "MIT",
|
|
7
|
+
"author": "Gal Be <galevgi@gmail.com>",
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "git+https://github.com/autor3search/javascript.git"
|
|
11
|
+
},
|
|
12
|
+
"keywords": [
|
|
13
|
+
"benchmark",
|
|
14
|
+
"performance",
|
|
15
|
+
"optimization",
|
|
16
|
+
"vitest",
|
|
17
|
+
"ai-agent",
|
|
18
|
+
"harness"
|
|
19
|
+
],
|
|
20
|
+
"bin": {
|
|
21
|
+
"autor3search-javascript": "bin/autor3search-javascript.js"
|
|
22
|
+
},
|
|
23
|
+
"engines": {
|
|
24
|
+
"node": ">=20"
|
|
25
|
+
},
|
|
26
|
+
"files": [
|
|
27
|
+
"bin",
|
|
28
|
+
"src",
|
|
29
|
+
"templates"
|
|
30
|
+
],
|
|
31
|
+
"scripts": {
|
|
32
|
+
"test": "vitest run",
|
|
33
|
+
"test:watch": "vitest"
|
|
34
|
+
},
|
|
35
|
+
"dependencies": {
|
|
36
|
+
"@babel/parser": "^7.25.0",
|
|
37
|
+
"picomatch": "^4.0.2",
|
|
38
|
+
"yaml": "^2.5.0"
|
|
39
|
+
},
|
|
40
|
+
"devDependencies": {
|
|
41
|
+
"vitest": "^2.1.0"
|
|
42
|
+
},
|
|
43
|
+
"homepage": "https://github.com/autor3search/javascript#readme",
|
|
44
|
+
"bugs": {
|
|
45
|
+
"url": "https://github.com/autor3search/javascript/issues"
|
|
46
|
+
},
|
|
47
|
+
"publishConfig": {
|
|
48
|
+
"access": "public"
|
|
49
|
+
}
|
|
50
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* The BenchRunner seam.
|
|
3
|
+
*
|
|
4
|
+
* Everything downstream of here — measure, stats, verdict, pipeline — is
|
|
5
|
+
* language-neutral and knows nothing about Vitest. Adding a second runner
|
|
6
|
+
* means adding a module here, not touching the scoring core.
|
|
7
|
+
*
|
|
8
|
+
* @typedef {object} BenchRunner
|
|
9
|
+
* @property {string} name
|
|
10
|
+
* @property {(dir: string, opts: object) => Promise<import('../../bench/set.js').BenchSet>} run
|
|
11
|
+
* Runs ONE measured round and returns one observation per benchmark.
|
|
12
|
+
*/
|
|
13
|
+
import { vitestRunner } from './vitest.js'
|
|
14
|
+
|
|
15
|
+
export { vitestRunner }
|
|
16
|
+
|
|
17
|
+
const REGISTRY = new Map([[vitestRunner.name, vitestRunner]])
|
|
18
|
+
|
|
19
|
+
/**
|
|
20
|
+
* @param {string} name
|
|
21
|
+
* @returns {BenchRunner}
|
|
22
|
+
*/
|
|
23
|
+
export function getBenchRunner(name) {
|
|
24
|
+
const runner = REGISTRY.get(name)
|
|
25
|
+
if (!runner) {
|
|
26
|
+
throw new Error(
|
|
27
|
+
`${JSON.stringify(name)} is not a registered bench adapter (have: ${[...REGISTRY.keys()].join(', ')})`,
|
|
28
|
+
)
|
|
29
|
+
}
|
|
30
|
+
return runner
|
|
31
|
+
}
|