@autor3search/javascript 0.2.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/LICENSE +21 -0
  2. package/README.md +500 -0
  3. package/bin/autor3search-javascript.js +4 -0
  4. package/package.json +50 -0
  5. package/src/adapters/bench/index.js +31 -0
  6. package/src/adapters/bench/vitest.js +128 -0
  7. package/src/adapters/driver-child.js +72 -0
  8. package/src/adapters/driver-hooks.js +18 -0
  9. package/src/adapters/driver.js +192 -0
  10. package/src/adapters/gates/index.js +64 -0
  11. package/src/adapters/gates/lint.js +46 -0
  12. package/src/adapters/gates/test.js +27 -0
  13. package/src/adapters/gates/typecheck.js +98 -0
  14. package/src/adapters/gates/util.js +45 -0
  15. package/src/adapters/vitest-shim.js +51 -0
  16. package/src/bench/parse.js +120 -0
  17. package/src/bench/set.js +101 -0
  18. package/src/bench/stats.js +443 -0
  19. package/src/cli/cmd-baseline.js +136 -0
  20. package/src/cli/cmd-doctor.js +38 -0
  21. package/src/cli/cmd-eval.js +231 -0
  22. package/src/cli/cmd-init.js +136 -0
  23. package/src/cli/cmd-profile.js +38 -0
  24. package/src/cli/cmd-report.js +96 -0
  25. package/src/cli/cmd-status.js +68 -0
  26. package/src/cli/cmd-stop.js +98 -0
  27. package/src/cli/cmd-version.js +31 -0
  28. package/src/cli/context.js +98 -0
  29. package/src/cli/main.js +62 -0
  30. package/src/config.js +209 -0
  31. package/src/discover.js +239 -0
  32. package/src/doctor.js +282 -0
  33. package/src/duration.js +75 -0
  34. package/src/freeze.js +234 -0
  35. package/src/gitx.js +115 -0
  36. package/src/measure.js +127 -0
  37. package/src/pipeline.js +391 -0
  38. package/src/profile.js +100 -0
  39. package/src/results.js +124 -0
  40. package/src/runner.js +198 -0
  41. package/src/scope.js +92 -0
  42. package/src/state/index.js +312 -0
  43. package/src/state/lock.js +189 -0
  44. package/src/state/stop.js +56 -0
  45. package/src/verdict.js +214 -0
  46. package/templates/program.md +264 -0
package/LICENSE ADDED
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Gal Be
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
package/README.md ADDED
@@ -0,0 +1,500 @@
1
+ # autor3search-javascript
2
+
3
+ [![npm](https://img.shields.io/npm/v/@autor3search/javascript)](https://www.npmjs.com/package/@autor3search/javascript)
4
+ [![node](https://img.shields.io/node/v/@autor3search/javascript)](https://nodejs.org)
5
+ [![license](https://img.shields.io/npm/l/@autor3search/javascript)](LICENSE)
6
+
7
+ An autonomous performance-optimization harness for JavaScript repositories. You
8
+ point a coding agent at your repository, run four commands, hand the agent
9
+ `program.md`, and go to sleep. The agent proposes one performance change at a
10
+ time; the harness gates it for correctness, measures it against a pinned
11
+ baseline, and returns a verdict — `KEEP` or `DISCARD` — that the agent cannot
12
+ argue with, weaken, or reinterpret. In the morning `report` tells you what
13
+ happened.
14
+
15
+ A standalone JavaScript tool: it discovers Vitest benchmarks, measures them,
16
+ and decides. It needs nothing but Node and the repository you point it at.
17
+ The discipline it enforces — frozen tests, out-of-tree state, interleaved A/B
18
+ measurement, an honest significance test, and a single scalar the agent
19
+ cannot game — comes from the autoresearch line of harnesses, and is
20
+ implemented here from scratch for the JavaScript toolchain. What that
21
+ toolchain can and cannot guarantee is stated in full in
22
+ [Limitations](#limitations) below.
23
+
24
+ ## Install
25
+
26
+ ```bash
27
+ npm install -g @autor3search/javascript
28
+ ```
29
+
30
+ Or run it without installing anything, which is the better option when you
31
+ want a specific version pinned for a run:
32
+
33
+ ```bash
34
+ npx @autor3search/javascript@0.2.0 doctor
35
+ ```
36
+
37
+ Node 20 or newer. The harness measures with Vitest, so the repository you
38
+ point it at needs `vitest` installed — `doctor` checks for it and says where
39
+ it looked if it is missing.
40
+
41
+ ## Start here
42
+
43
+ Open your coding agent inside the JavaScript repository you want to make
44
+ faster, and paste this:
45
+
46
+ ```text
47
+ Install and run autor3search-javascript on this repository, then optimize it.
48
+
49
+ Setup:
50
+ 1. npm install -g @autor3search/javascript
51
+ Or prefix every command below with `npx` instead — either way, keep using
52
+ the same form for the whole run, so the version that recorded the baseline
53
+ is the version that scores against it.
54
+ 2. autor3search-javascript init
55
+ Show me the benchmarks it discovered. If it reports none, STOP and tell me:
56
+ this tool can only optimize what it can measure.
57
+ 3. Review .autor3search/config.yaml and program.md, then:
58
+ git add -A && git commit -m "autor3search-javascript init"
59
+ 4. autor3search-javascript doctor
60
+ Show me any warnings. If the machine looks unfit to measure, stop and ask
61
+ me before continuing.
62
+ 5. autor3search-javascript baseline -tag <today, e.g. sep7>
63
+
64
+ Then:
65
+ 6. Read program.md in this repository, in full. It is your instruction set for
66
+ the rest of this run. Follow it exactly.
67
+
68
+ Rules for the whole run:
69
+ - One hypothesis per commit. Commit before each experiment, then run
70
+ `autor3search-javascript eval --json -desc "<idea>"` and apply its verdict
71
+ before touching anything else: KEEP means the commit stays; anything else
72
+ (DISCARD, FAIL, CRASH, ABORTED) means `git reset --hard HEAD~1`.
73
+ - Never redirect eval's stdout into run.log. The harness already holds that
74
+ file open, and a second writer destroys the transcript you need when
75
+ something FAILs.
76
+ - Never edit program.md, .autor3search/config.yaml, results.tsv, any
77
+ *.test.* / *.spec.* / *.bench.* file, package.json, a lockfile, or a
78
+ Vitest/Vite config file. They are not yours.
79
+ - Never pass -force to any autor3search-javascript command. (I may run
80
+ `autor3search-javascript stop -force` myself; that one is mine, not yours.)
81
+ - Print one context line before each experiment, so I can see where you are:
82
+ [exp <n> | <branch> | vs <measure_commit> | stop: autor3search-javascript stop]
83
+
84
+ Run the loop until I stop you. I stop you by running
85
+ `autor3search-javascript stop` in my own terminal — you will see it as
86
+ "stop_requested": true in a verdict. When you do: apply that verdict, do not
87
+ start another experiment, run `autor3search-javascript report`, summarize what
88
+ you tried, and exit the loop.
89
+ ```
90
+
91
+ That's the whole handoff. The agent installs the tool, discovers your
92
+ benchmarks, freezes a baseline, and then follows `program.md` — generated for
93
+ your repository by `init` — which tells it how to run the keep-or-discard loop.
94
+ `program.md` names the benchmarks in scope, spells out the
95
+ KEEP/DISCARD/FAIL/CRASH contract, lists everything the agent must never touch,
96
+ and ends with a bank of generic V8/JavaScript performance ideas for when the
97
+ agent is out of hypotheses.
98
+
99
+ What you get back: one commit per accepted change on a branch named
100
+ `autor3search-javascript/<tag>`, and a `results.tsv` recording every experiment
101
+ that was tried, including the ones that failed.
102
+ `autor3search-javascript report` summarizes it.
103
+
104
+ Two things worth knowing before you start it:
105
+
106
+ - **It needs benchmarks.** This optimizes what it can measure, and refuses to
107
+ guess: `init` looks for Vitest `bench()` calls in `*.bench.*` files and
108
+ refuses outright when it finds none.
109
+ - **Numbers are only as good as the machine.** Run `doctor` and read it. A
110
+ thermally throttled laptop on battery produces noise dressed as data, and a
111
+ JS runtime adds JIT tier-up and GC scheduling on top — see
112
+ [Limitations](#limitations) for how much.
113
+
114
+ Everything past this point is for the human setting the run up, or for
115
+ understanding what the agent in step 6 is actually bound by.
116
+
117
+ ## The idea
118
+
119
+ | Owns | Belongs to |
120
+ |---|---|
121
+ | Gating (scope, config integrity, frozen files, correctness), measuring, scoring | the harness |
122
+ | Your application source | the agent, inside `scope` |
123
+ | `program.md` and `.autor3search/config.yaml` | you |
124
+ | Frozen test/bench copies, the baseline record, the pinned measurement worktree | the user cache, outside the repository |
125
+
126
+ `*.test.*`, `*.spec.*` and `*.bench.*` files are frozen at `baseline` time and
127
+ restored before every `eval`. The agent may read them, may complain about
128
+ them in its `-desc`, but cannot change what they check or what they measure.
129
+ Everything the verdict depends on lives outside the repository the agent is
130
+ editing. The scope gate also rejects dependency files and any Vitest/Vite
131
+ config outright, because those are loaded by the bench runner itself — an
132
+ ordinary, in-scope config file could otherwise redirect what a frozen
133
+ benchmark imports, or stub out the code path it measures, without the frozen
134
+ copy changing by one byte. That closes the config route specifically; it is
135
+ not a claim that nothing else the agent does inside the repo can move the
136
+ goalposts — see [Limitations](#limitations) for what is still open.
137
+
138
+ ## Quick start
139
+
140
+ ```
141
+ autor3search-javascript init # discovers benchmarks, writes config + program.md
142
+ git add -A && git commit -m "autor3search-javascript init"
143
+ autor3search-javascript doctor # is this machine fit to measure on?
144
+ autor3search-javascript baseline -tag sep8 # creates the run branch, freezes tests+benches, pins HEAD
145
+ ```
146
+
147
+ The commit between `init` and `baseline` matters: `baseline` refuses a dirty
148
+ working tree, because a baseline pinned against what's on disk rather than
149
+ what's in git could never be reproduced — the pinned worktree it creates is a
150
+ git worktree checked out at a real commit, and there has to be one.
151
+
152
+ ## Watching a run, and stopping it
153
+
154
+ `autor3search-javascript status` shows where things stand without touching
155
+ anything:
156
+
157
+ ```
158
+ run tag demo
159
+ branch autor3search-javascript/demo (checked out)
160
+ baseline bdf3c4c (run started here)
161
+ measuring vs 50c69c7 (advanced past the baseline by earlier KEEPs)
162
+ worktree /Users/you/Library/Caches/autor3search-javascript/.../baseline-worktree
163
+ experiments 1 run (1 keep, 0 discard, 0 fail, 0 crash) — next is #2
164
+ eval idle
165
+ stop not requested
166
+ ```
167
+
168
+ Three ways to end a run, in increasing order of force:
169
+
170
+ 1. **`autor3search-javascript stop`** — a graceful request. The experiment
171
+ under way finishes, is measured and scored normally; the agent sees
172
+ `"stop_requested": true` on that verdict, applies it as usual (`KEEP`
173
+ stays, anything else resets), then exits the loop. Nothing is thrown away.
174
+ 2. **`autor3search-javascript stop --force`** — writes the same request, then
175
+ sends `SIGTERM` to the running `eval`, which tears down its own child
176
+ process groups so no Vitest worker is left burning CPU. The agent sees
177
+ `"status": "ABORTED"`, exit code 2, and no `results.tsv` row — nothing was
178
+ measured, so nothing was recorded. `stop --force` reports what HEAD looks
179
+ like afterward; it does not touch the repository for you.
180
+ 3. **Ctrl+C** — the same abort path as `stop --force`, sent directly to a
181
+ foreground `eval`.
182
+
183
+ `autor3search-javascript stop --clear` cancels a pending stop request so the
184
+ loop continues — that is your decision, never the agent's to make on its own.
185
+
186
+ ## Commands
187
+
188
+ | Command | Does |
189
+ |---|---|
190
+ | `init` | Scans the repo, discovers benchmarks by parsing (not running) them, writes `.autor3search/config.yaml` and `program.md`. Refuses if it finds no benchmarks. |
191
+ | `doctor` | Reports whether this machine can measure reliably: Node/git versions, CPU count and load, whether Vitest resolves, power state, thermal state, disk space. Informational only — always exits 0. |
192
+ | `baseline` | Creates the run branch, freezes every test and bench file as a golden copy, pins a git worktree at the current commit, records the config hash. Refuses a dirty tree, an existing `results.tsv`, a reused tag, or a repo with no benchmarks. |
193
+ | `profile` | Runs the declared benchmarks under `--cpu-prof`/`--heap-prof` and prints the hottest self-time functions, plus where the raw profiles were written. |
194
+ | `eval` | Runs one experiment: scope check, config-integrity check, restores any frozen file the agent touched, runs the correctness gates, measures candidate against baseline (interleaved), scores, and returns a verdict. The only command that decides anything. |
195
+ | `status` | Read-only snapshot: branch, baseline, measurement pointer, worktree, experiment counts, whether `eval` is running, whether a stop is pending. |
196
+ | `stop` | Requests (or, with `--force`, forces) the run to end. `--clear` cancels a pending request. |
197
+ | `report` | Summarizes `results.tsv`: counts by status, cumulative speedup (the product of every kept score), and the largest individual wins. |
198
+ | `version` | Prints the installed version and commit. |
199
+
200
+ Every command accepts `-C <dir>` to run against a repository other than the
201
+ current directory.
202
+
203
+ ## Where run state lives
204
+
205
+ Everything the score depends on — the frozen golden copies, the baseline
206
+ record, the pinned measurement worktree — is written to:
207
+
208
+ ```
209
+ <user cache>/autor3search-javascript/<repo hash>/<tag>/
210
+ ```
211
+
212
+ (`~/Library/Caches` on macOS, `$XDG_CACHE_HOME` or `~/.cache` on Linux,
213
+ `%LOCALAPPDATA%` on Windows). `<repo hash>` is a hash of the repository's real
214
+ absolute path, so two checkouts of the same project never collide and never
215
+ share state.
216
+
217
+ Set `AUTOR3SEARCH_JAVASCRIPT_STATE_HOME` to relocate it — a relative path is
218
+ refused outright, because it would resolve differently depending on which
219
+ directory each command happened to be run from, and `eval` run from a
220
+ subdirectory would then silently address different state than `stop` run from
221
+ the repository root.
222
+
223
+ Every directory the harness creates there is mode `0700`, and on POSIX systems
224
+ each level from the state home down is checked before it is used: a directory
225
+ owned by another user, or one that group or others can write, is refused with
226
+ the `chmod` that fixes it. This matters because the frozen store and its
227
+ manifest both live here — whoever can write to them can replace the benchmarks
228
+ the score is measured against, consistently enough that the hash check still
229
+ passes. Under the default cache location the parent already restricts access;
230
+ the check is what makes `AUTOR3SEARCH_JAVASCRIPT_STATE_HOME` safe to point at a
231
+ shared directory. Windows reports synthetic mode bits and has no owner to
232
+ compare against, so the check is skipped there rather than made to look like it
233
+ ran.
234
+
235
+ ## Worked example
236
+
237
+ `test/e2e.test.js` builds this exact scenario from
238
+ `test/helpers/bench-repo.js`; `testdata/demo/README.md` documents it in the
239
+ repository. A word counter, written badly on purpose:
240
+
241
+ ```js
242
+ // the slow version
243
+ export function countWords(s) {
244
+ const counts = {}
245
+ for (const field of s.split(/\s+/)) {
246
+ let word = ''
247
+ for (const ch of field) {
248
+ const lower = ch.toLowerCase()
249
+ if (/[a-z0-9]/.test(lower)) word = word + lower
250
+ }
251
+ if (word !== '') counts[word] = (counts[word] ?? 0) + 1
252
+ }
253
+ return counts
254
+ }
255
+ ```
256
+
257
+ with a frozen test (`src/wordcount.test.js`) checking its behaviour and a
258
+ frozen benchmark (`src/wordcount.bench.js`) measuring it on a fixed input.
259
+ Both are restored before every `eval`, so a candidate has to keep the exact
260
+ same behaviour to be scored at all. The candidate:
261
+
262
+ ```js
263
+ // the fast version
264
+ export function countWords(s) {
265
+ const counts = new Map()
266
+ for (const field of s.split(' ')) {
267
+ let word = ''
268
+ for (let i = 0; i < field.length; i++) {
269
+ const c = field.charCodeAt(i)
270
+ if (c >= 65 && c <= 90) word += String.fromCharCode(c + 32)
271
+ else if ((c >= 97 && c <= 122) || (c >= 48 && c <= 57)) word += field[i]
272
+ }
273
+ if (word !== '') counts.set(word, (counts.get(word) ?? 0) + 1)
274
+ }
275
+ return Object.fromEntries(counts)
276
+ }
277
+ ```
278
+
279
+ replaces the regex-per-character scan and string concatenation with charcode
280
+ comparisons and a `Map`.
281
+
282
+ This is a real run, not an illustration — `init`, `doctor`, `baseline -tag
283
+ demo`, then the swap above, then `eval`, all against the code shown here, on
284
+ a MacBook Pro (Apple M5, 10 logical cores), macOS Darwin 25.6.0, Node
285
+ v22.23.1:
286
+
287
+ | | baseline | candidate | change | p |
288
+ |---|---|---|---|---|
289
+ | `countWords` (ns/op) | 168,105 | 77,333 | **−54.00%** | 0.00001 |
290
+
291
+ ```
292
+ VERDICT: KEEP (improved) — score 0.4600 (-54.00%)
293
+ ```
294
+
295
+ `score` is the geometric mean of `candidate/baseline` across the declared
296
+ benchmarks (here, one), so with a single benchmark it equals the ratio
297
+ directly: 0.46, i.e. 2.17x faster. Run it yourself — noise is real, and on a
298
+ busier machine or a shorter `count` this same diff can land as a `DISCARD`
299
+ instead; that is not a bug, it is the point.
300
+
301
+ ## What the harness enforces
302
+
303
+ | Try this | The harness |
304
+ |---|---|
305
+ | Weaken a test | Restored from the frozen copy before every `eval` |
306
+ | Add an "easier" benchmark | Rejected: a new file matching `*.bench.*` not present at baseline is `new_test_file` |
307
+ | Rewrite the benchmark to measure something trivial | Restored too — bench files are frozen exactly like test files |
308
+ | Symlink a frozen file, or a directory on its path | Refused: `symlink_swap` |
309
+ | Hard-link over a frozen file | Refused: `hardlink_swap` |
310
+ | Edit a file outside `scope` | `scope_violation`, checked before anything is built or measured |
311
+ | Bank ordinary noise as a win | The Mann-Whitney test behind `significant` has to clear it first |
312
+ | Speed up A by wrecking B | The regression guard trips on B alone, discarding the whole change |
313
+ | Change a dependency (`package.json`, any lockfile) | Rejected outright, regardless of `scope` |
314
+ | Loosen `max_regress_pct` or `count` mid-run | `config.yaml`'s hash is pinned at `baseline`; any change fails with `config_changed` |
315
+ | Compare against a stale, cached baseline | Every `eval` re-measures both sides, interleaved, in the same process |
316
+ | Coast on an earlier win forever | The measurement baseline advances to the just-kept commit after every `KEEP`, so the next `eval` has to earn its own improvement |
317
+
318
+ ## Scoring
319
+
320
+ `score` is the geometric mean, across the declared benchmarks, of
321
+ `candidate_time / baseline_time`:
322
+
323
+ ```
324
+ score = exp( mean( log(candidate_i / baseline_i) ) ) for i in benchmarks
325
+ ```
326
+
327
+ below 1 is faster. A `KEEP` requires **all** of:
328
+
329
+ 1. **No regression guard trips.** Any benchmark that got significantly
330
+ *worse* — significant at the raw, uncorrected `alpha` (0.05) and past
331
+ `max_regress_pct` — discards the whole experiment, regardless of how good
332
+ the overall score is. This check deliberately skips the Bonferroni
333
+ correction used below: correcting here would only make the guard *less*
334
+ sensitive to harm, which is backwards for a guard. Be conservative about
335
+ banking a win, be liberal about catching damage — that asymmetry is
336
+ intentional.
337
+ 2. **The score clears `1 - min_effect_pct/100`** (default 1%, so `score <
338
+ 0.99`). A sub-1% shave is discarded by design as `improvement_below_min_effect`
339
+ even when it is real: not worth a commit in an unattended loop.
340
+ 3. **At least one benchmark improved past a Bonferroni-corrected significance
341
+ bar**, `alpha / k` where `k` is the number of benchmarks compared.
342
+ Comparing several benchmarks against the same uncorrected `alpha` inflates
343
+ the chance that at least one shows a spurious "significant" improvement
344
+ even when nothing changed — that is what the correction is for.
345
+
346
+ ## Releasing
347
+
348
+ Publishing runs from GitHub Actions with npm **trusted publishing** (OIDC).
349
+ There is no npm token in this repository, in its secrets, or on any
350
+ maintainer's machine: npm mints a short-lived credential from GitHub's own
351
+ identity for that one workflow run, and attaches a provenance attestation
352
+ linking the published tarball to the commit and run that built it.
353
+
354
+ To cut a release:
355
+
356
+ ```bash
357
+ npm version patch # or minor / major — commits and tags
358
+ git push --follow-tags
359
+ ```
360
+
361
+ The tag triggers `.github/workflows/release.yml`, which refuses to publish if
362
+ the tag and `package.json` disagree, and runs the full suite on Linux first.
363
+ The OIDC path is confirmed working: a run against an already-published version
364
+ authenticated and got as far as `cannot publish over the previously published
365
+ versions`, which is the failure you want to see — it means the credential
366
+ exchange succeeded and only the version was wrong.
367
+
368
+ Two things to know if you are wiring this up on a fork or a new package:
369
+
370
+ - The trusted publisher on npmjs.com names the **workflow filename**, so
371
+ renaming `release.yml` breaks publishing until the setting is updated.
372
+ - npm cannot publish a package's **first** version this way — a trusted
373
+ publisher can only be configured on a package that already exists
374
+ ([npm/cli#8544](https://github.com/npm/cli/issues/8544)). That one publish
375
+ needs a token; every release after it is token-free.
376
+
377
+ ## Platform support
378
+
379
+ | | |
380
+ |---|---|
381
+ | **Linux** | supported — CI runs the full suite on Node 20 and 22 |
382
+ | **macOS** | supported — CI runs the full suite on Node 20 and 22 |
383
+ | **Windows** | **not supported** |
384
+
385
+ On Windows 448 of 458 tests pass, so measurement itself works. What does not
386
+ work is stopping: Node cannot deliver SIGINT to a child process group there
387
+ the way it does on POSIX, so an interrupted `eval` never reaches the `ABORTED`
388
+ path — it exits with a null code instead of 2 and can leave its claim behind.
389
+ An unattended harness that cannot be reliably stopped is not something to be
390
+ quiet about, so `doctor` says so on Windows rather than letting you find out
391
+ at 3am. Two further failures are the test suite's own POSIX assumptions
392
+ (a `chmod`-unreadable directory, and path separators) rather than product bugs.
393
+
394
+ WSL reports as Linux and is unaffected. Node 20 or newer is required
395
+ everywhere.
396
+
397
+ ## Limitations
398
+
399
+ Stated here rather than left for you to discover:
400
+
401
+ - **A `KEEP` is evidence, not proof.** Any significance threshold admits
402
+ false positives by construction — `alpha = 0.05` means a true no-op change
403
+ still looks "significant" one time in twenty, by design, however the
404
+ harness is implemented.
405
+ - **Laptops are noisy.** Background processes, thermal throttling, and on
406
+ Apple Silicon specifically, the scheduler moving work between performance
407
+ and efficiency cores, all show up as measurement noise, not signal.
408
+ `doctor` reports load, power source and thermal state and warns when they
409
+ look bad — it cannot fix any of them. If experiments look erratic on a
410
+ laptop, raise `min_effect_pct` or measure on a quiet machine instead.
411
+ - **No benchmarks, no value.** `init` refuses outright on a repository with
412
+ no `*.bench.*` file, rather than accepting a config with nothing to gate
413
+ on and pretending a verdict means something.
414
+ - **A small measurement asymmetry survives within a round.** Rounds
415
+ alternate which side runs first, cancelling drift *between* rounds, but
416
+ within a single round one side always runs first and the other second —
417
+ any bias monotonic across just that round (a CPU still ramping up, a
418
+ background job starting mid-measurement) lands on whichever side ran
419
+ second. Averaging over rounds does not remove this, because it is a
420
+ constant offset, not noise; that's why round counts are kept even.
421
+ - **Microbenchmarks are not your application.** A `bench()` that measures a
422
+ cold path or a trivial helper produces numbers that are entirely real and
423
+ entirely useless. Benchmark what actually dominates your workload.
424
+ - **`count` below 4 can never reach significance**, whatever the size of the
425
+ improvement — with 2 or 3 rounds per side the best achievable two-sided
426
+ p-value (0.333 or 0.1) is already above the default `alpha`. `config`
427
+ refuses a `count` under 4 rather than silently running a harness that can
428
+ never `KEEP`.
429
+ - **`bytes/op` is an approximate hint, never scored.** V8 exposes no
430
+ allocation counter, so it is a heap-size delta measured across a
431
+ forced-GC window — noisy by construction, since it can only see whatever
432
+ the GC left uncollected at the sampling point, not what was actually
433
+ allocated. It can also be entirely absent: the driver child needs
434
+ `--expose-gc`, and a bench file that fails to import or throws while
435
+ running is dropped from the hint rather than measured. Either way it is
436
+ silently omitted from the eval output rather than failing the run — a
437
+ hint that couldn't be measured must never fail a real, correctly-measured
438
+ experiment. It exists to point at allocation-heavy code, nothing more.
439
+ - **There is no `benchtime` setting.** Vitest exposes no global "run this
440
+ benchmark for N seconds" option, so per-benchmark duration lives in the
441
+ `bench()` call itself (via
442
+ Vitest's own `time`/`iterations` options in the bench file), not in
443
+ `.autor3search/config.yaml`. `count` controls how many measured rounds
444
+ the harness runs, not how long any one of them takes.
445
+ - **This harness is not a compiled binary.** It is an npm-installed
446
+ JavaScript program, running as the same OS user as the agent it's grading
447
+ — so an agent that decided to, could in principle edit this package's own
448
+ installed files. A harness the agent cannot reach at all would not have
449
+ that hole; this one has to close it by other means. Out-of-tree state (the frozen copies,
450
+ the baseline record, the pinned worktree, all living under the user
451
+ cache rather than the repository), together with rejecting dependency
452
+ files and known Vitest/Vite config filenames outright, protects the
453
+ metric from *most* of what the agent could do inside the repository being
454
+ optimized: editing its own tests or benchmarks, weakening the config that
455
+ constrains it, or redirecting the bench runner's import through a config
456
+ file. It does not close every such route — only the config filenames
457
+ Vitest 2.1.9 is known to load at the repository root are rejected, so a
458
+ bench runner reconfigured to load from somewhere else, a workspace-globbed
459
+ config in a subdirectory, or some other toolchain file this project has
460
+ not enumerated could still retarget what gets measured. That residual gap
461
+ is stated here rather than left for you to find out the hard way.
462
+ - **A narrow TOCTOU window exists in the freeze module.** `src/freeze.js`
463
+ checks that a frozen path (and every directory on the way to it) is not a
464
+ symlink or hard link, then reads or writes it. Between that check and the
465
+ write, an agent racing the harness in the same process tree could in
466
+ principle swap the path. This is theoretical under this project's threat
467
+ model — an unattended overnight agent isn't racing filesystem syscalls
468
+ against itself — but it is real, not closed by design, and is disclosed
469
+ here rather than implied away.
470
+
471
+ ## Repos with no benchmarks
472
+
473
+ `autor3search-javascript init` refuses to write a config for a repository
474
+ with no `*.bench.*` file:
475
+
476
+ ```
477
+ no benchmarks found in <repo>.
478
+
479
+ autor3search-javascript optimizes what it can measure, and refuses to guess. Write at least one
480
+ Vitest benchmark covering the code you want made faster, in a *.bench.js file:
481
+
482
+ import { bench } from 'vitest'
483
+ import { thing } from './thing.js'
484
+
485
+ bench('thing', () => { thing() })
486
+
487
+ Benchmark the path that actually dominates your workload — one that exercises a cold path or a
488
+ trivial helper produces numbers that are entirely real and entirely useless. Then run init again.
489
+ ```
490
+
491
+ Write a `bench()` around the function that shows up at the top of a real
492
+ profile of your application, not around whatever is easiest to isolate. Once
493
+ `init` finds at least one, it discovers it by *parsing* the file (via
494
+ `@babel/parser`), not by running it — so this works even on a tree that does
495
+ not currently build, which matters because that is also when a candidate
496
+ most needs to fail loudly rather than be silently skipped.
497
+
498
+ ## License
499
+
500
+ MIT © 2026 Gal Be
@@ -0,0 +1,4 @@
1
+ #!/usr/bin/env node
2
+ import { dispatch } from '../src/cli/main.js'
3
+
4
+ process.exitCode = await dispatch(process.argv.slice(2), { out: process.stdout, err: process.stderr })
package/package.json ADDED
@@ -0,0 +1,50 @@
1
+ {
2
+ "name": "@autor3search/javascript",
3
+ "version": "0.2.0",
4
+ "description": "Autonomous AI-driven performance optimization for any JavaScript repository",
5
+ "type": "module",
6
+ "license": "MIT",
7
+ "author": "Gal Be <galevgi@gmail.com>",
8
+ "repository": {
9
+ "type": "git",
10
+ "url": "git+https://github.com/autor3search/javascript.git"
11
+ },
12
+ "keywords": [
13
+ "benchmark",
14
+ "performance",
15
+ "optimization",
16
+ "vitest",
17
+ "ai-agent",
18
+ "harness"
19
+ ],
20
+ "bin": {
21
+ "autor3search-javascript": "bin/autor3search-javascript.js"
22
+ },
23
+ "engines": {
24
+ "node": ">=20"
25
+ },
26
+ "files": [
27
+ "bin",
28
+ "src",
29
+ "templates"
30
+ ],
31
+ "scripts": {
32
+ "test": "vitest run",
33
+ "test:watch": "vitest"
34
+ },
35
+ "dependencies": {
36
+ "@babel/parser": "^7.25.0",
37
+ "picomatch": "^4.0.2",
38
+ "yaml": "^2.5.0"
39
+ },
40
+ "devDependencies": {
41
+ "vitest": "^2.1.0"
42
+ },
43
+ "homepage": "https://github.com/autor3search/javascript#readme",
44
+ "bugs": {
45
+ "url": "https://github.com/autor3search/javascript/issues"
46
+ },
47
+ "publishConfig": {
48
+ "access": "public"
49
+ }
50
+ }
@@ -0,0 +1,31 @@
1
+ /**
2
+ * The BenchRunner seam.
3
+ *
4
+ * Everything downstream of here — measure, stats, verdict, pipeline — is
5
+ * language-neutral and knows nothing about Vitest. Adding a second runner
6
+ * means adding a module here, not touching the scoring core.
7
+ *
8
+ * @typedef {object} BenchRunner
9
+ * @property {string} name
10
+ * @property {(dir: string, opts: object) => Promise<import('../../bench/set.js').BenchSet>} run
11
+ * Runs ONE measured round and returns one observation per benchmark.
12
+ */
13
+ import { vitestRunner } from './vitest.js'
14
+
15
+ export { vitestRunner }
16
+
17
+ const REGISTRY = new Map([[vitestRunner.name, vitestRunner]])
18
+
19
+ /**
20
+ * @param {string} name
21
+ * @returns {BenchRunner}
22
+ */
23
+ export function getBenchRunner(name) {
24
+ const runner = REGISTRY.get(name)
25
+ if (!runner) {
26
+ throw new Error(
27
+ `${JSON.stringify(name)} is not a registered bench adapter (have: ${[...REGISTRY.keys()].join(', ')})`,
28
+ )
29
+ }
30
+ return runner
31
+ }