ashlr-plugin 1.36.4 → 1.37.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.claude-plugin/marketplace.json +3 -3
- package/.claude-plugin/plugin.json +1 -1
- package/.codex-plugin/plugin.json +1 -1
- package/README.md +49 -8
- package/commands/ashlr-doctor.md +29 -0
- package/commands/ashlr-status.md +1 -1
- package/docs/PHANTOM-USAGE.md +58 -0
- package/docs/architecture.md +5 -5
- package/docs/ashlr-handshake.md +36 -0
- package/docs/assets/og.svg +1 -1
- package/docs/benchmarks-v2.json +226 -177
- package/docs/benchmarks.md +20 -10
- package/docs/competitive-wozcode.md +2 -2
- package/docs/embeddings-gateway.md +39 -0
- package/docs/index.html +3 -3
- package/docs/legal.md +1 -1
- package/docs/llm-providers.md +15 -1
- package/docs/multi-host-mcp.md +25 -3
- package/docs/phantom-preflight.md +23 -0
- package/docs/team-genome.md +2 -1
- package/hooks/_hook-timing-ledger.ts +2 -9
- package/hooks/hooks.json +3 -106
- package/hooks/posttooluse-stale-result.ts +3 -0
- package/hooks/pretooluse-common.ts +1 -1
- package/hooks/pretooluse-notebookedit.ts +1 -1
- package/hooks/pretooluse-write.ts +1 -1
- package/hooks/session-start.ts +48 -26
- package/package.json +9 -4
- package/ports/cursor/mcp.json +1 -1
- package/scripts/_read-thresholds.ts +119 -0
- package/scripts/ashlr-mcp.mjs +15 -0
- package/scripts/ashlr-mcp.ts +3 -0
- package/scripts/ashlr-plugin.mjs +59 -0
- package/scripts/bench-read-real.ts +90 -0
- package/scripts/bootstrap.mjs +14 -4
- package/scripts/cli.ts +35 -10
- package/scripts/context-status.ts +7 -2
- package/scripts/deploy-smoke.sh +8 -4
- package/scripts/doctor.ts +171 -42
- package/scripts/genome-auto-consolidate.ts +4 -2
- package/scripts/genome-init.ts +3 -3
- package/scripts/genome-key.ts +64 -7
- package/scripts/handshake.mjs +51 -0
- package/scripts/hook-bootstrap.mjs +170 -17
- package/scripts/mcp-entrypoint.ts +3 -0
- package/scripts/ollama-setup.ts +5 -4
- package/scripts/run-benchmark.ts +28 -18
- package/scripts/upgrade-flow.ts +9 -14
- package/servers/_base-urls.ts +48 -0
- package/servers/_companions.ts +258 -0
- package/servers/_cwd-clamp.ts +38 -1
- package/servers/_edit-match.ts +12 -2
- package/servers/_embedding-model.ts +77 -0
- package/servers/_gateway-target.ts +179 -0
- package/servers/_llm-providers/anthropic.ts +10 -4
- package/servers/_llm-providers/cloud.ts +9 -7
- package/servers/_llm-providers/index.ts +8 -6
- package/servers/_llm-providers/local.ts +9 -5
- package/servers/_mcp-host.ts +6 -0
- package/servers/_miss-tracker.ts +121 -0
- package/servers/_phantom-exec.ts +135 -0
- package/servers/_phantom-preflight.ts +130 -0
- package/servers/_pid-alive.ts +40 -0
- package/servers/_pro-token-keychain.ts +149 -0
- package/servers/_redact-keys.ts +75 -0
- package/servers/_rg-text.ts +111 -0
- package/servers/_router.ts +20 -2
- package/servers/_run-with-timeout.ts +56 -12
- package/servers/_test-watch.ts +1 -10
- package/servers/_token-measure.ts +3 -1
- package/servers/_tool-base.ts +6 -0
- package/servers/bash-server.ts +1 -10
- package/servers/edit-server.ts +24 -1
- package/servers/efficiency-server.ts +12 -2
- package/servers/genome-server-handlers.ts +2 -2
- package/servers/genome-server.ts +17 -17
- package/servers/grep-server.ts +6 -2
- package/servers/orient-server-handlers.ts +3 -2
- package/servers/orient-server.ts +16 -9
- package/servers/read-server.ts +1 -1
- package/servers/write-server-handlers.ts +10 -2
package/docs/benchmarks-v2.json
CHANGED
|
@@ -1,203 +1,203 @@
|
|
|
1
1
|
{
|
|
2
2
|
"version": 2,
|
|
3
|
-
"measuredAt": "2026-
|
|
3
|
+
"measuredAt": "2026-10-19T17:23:26.418Z",
|
|
4
4
|
"repo": {
|
|
5
5
|
"url": "https://github.com/ashlrai/ashlr-plugin.git",
|
|
6
|
-
"commit": "
|
|
7
|
-
"files":
|
|
8
|
-
"loc":
|
|
6
|
+
"commit": "364a4030d3bdb02271c04e18d1632ef6bd00d8f1",
|
|
7
|
+
"files": 1294,
|
|
8
|
+
"loc": 277392
|
|
9
9
|
},
|
|
10
10
|
"samples": {
|
|
11
11
|
"ashlr__read": [
|
|
12
12
|
{
|
|
13
|
-
"path": "/
|
|
13
|
+
"path": "/workspace/repos/ashlr-plugin/site/app/og/route.tsx",
|
|
14
14
|
"bucket": "2-5KB",
|
|
15
|
-
"rawBytes":
|
|
16
|
-
"rawTokens":
|
|
17
|
-
"ashlrBytes":
|
|
15
|
+
"rawBytes": 3805,
|
|
16
|
+
"rawTokens": 950,
|
|
17
|
+
"ashlrBytes": 1628,
|
|
18
18
|
"ashlrTokens": 406,
|
|
19
|
-
"ratio": 0.
|
|
19
|
+
"ratio": 0.42736842105263156
|
|
20
20
|
},
|
|
21
21
|
{
|
|
22
|
-
"path": "/
|
|
22
|
+
"path": "/workspace/repos/ashlr-plugin/site/content/docs/skills/resume.mdx",
|
|
23
23
|
"bucket": "2-5KB",
|
|
24
|
-
"rawBytes":
|
|
25
|
-
"rawTokens":
|
|
26
|
-
"ashlrBytes":
|
|
24
|
+
"rawBytes": 2424,
|
|
25
|
+
"rawTokens": 601,
|
|
26
|
+
"ashlrBytes": 1637,
|
|
27
27
|
"ashlrTokens": 406,
|
|
28
|
-
"ratio": 0.
|
|
28
|
+
"ratio": 0.6755407653910149
|
|
29
29
|
},
|
|
30
30
|
{
|
|
31
|
-
"path": "/
|
|
31
|
+
"path": "/workspace/repos/ashlr-plugin/servers/_llm-providers/local.ts",
|
|
32
32
|
"bucket": "2-5KB",
|
|
33
|
-
"rawBytes":
|
|
34
|
-
"rawTokens":
|
|
35
|
-
"ashlrBytes":
|
|
33
|
+
"rawBytes": 4542,
|
|
34
|
+
"rawTokens": 1134,
|
|
35
|
+
"ashlrBytes": 1627,
|
|
36
36
|
"ashlrTokens": 406,
|
|
37
|
-
"ratio": 0.
|
|
37
|
+
"ratio": 0.35802469135802467
|
|
38
38
|
},
|
|
39
39
|
{
|
|
40
|
-
"path": "/
|
|
40
|
+
"path": "/workspace/repos/ashlr-plugin/__tests__/genome-crypto-v2.test.ts",
|
|
41
41
|
"bucket": "2-5KB",
|
|
42
|
-
"rawBytes":
|
|
43
|
-
"rawTokens":
|
|
42
|
+
"rawBytes": 3456,
|
|
43
|
+
"rawTokens": 862,
|
|
44
44
|
"ashlrBytes": 1629,
|
|
45
45
|
"ashlrTokens": 406,
|
|
46
|
-
"ratio": 0.
|
|
46
|
+
"ratio": 0.4709976798143852
|
|
47
47
|
},
|
|
48
48
|
{
|
|
49
|
-
"path": "/
|
|
49
|
+
"path": "/workspace/repos/ashlr-plugin/server/tests/admin-wad-d.test.ts",
|
|
50
50
|
"bucket": "5-15KB",
|
|
51
|
-
"rawBytes":
|
|
52
|
-
"rawTokens":
|
|
53
|
-
"ashlrBytes":
|
|
51
|
+
"rawBytes": 7281,
|
|
52
|
+
"rawTokens": 1820,
|
|
53
|
+
"ashlrBytes": 1627,
|
|
54
54
|
"ashlrTokens": 406,
|
|
55
|
-
"ratio": 0.
|
|
55
|
+
"ratio": 0.2230769230769231
|
|
56
56
|
},
|
|
57
57
|
{
|
|
58
|
-
"path": "/
|
|
58
|
+
"path": "/workspace/repos/ashlr-plugin/site/components/hero.tsx",
|
|
59
59
|
"bucket": "5-15KB",
|
|
60
|
-
"rawBytes":
|
|
61
|
-
"rawTokens":
|
|
62
|
-
"ashlrBytes":
|
|
60
|
+
"rawBytes": 5482,
|
|
61
|
+
"rawTokens": 1371,
|
|
62
|
+
"ashlrBytes": 1623,
|
|
63
63
|
"ashlrTokens": 406,
|
|
64
|
-
"ratio": 0.
|
|
64
|
+
"ratio": 0.2961342086068563
|
|
65
65
|
},
|
|
66
66
|
{
|
|
67
|
-
"path": "/
|
|
67
|
+
"path": "/workspace/repos/ashlr-plugin/server/src/jobs/weekly-digest-cron.ts",
|
|
68
68
|
"bucket": "5-15KB",
|
|
69
|
-
"rawBytes":
|
|
70
|
-
"rawTokens":
|
|
71
|
-
"ashlrBytes":
|
|
69
|
+
"rawBytes": 7950,
|
|
70
|
+
"rawTokens": 1984,
|
|
71
|
+
"ashlrBytes": 1629,
|
|
72
72
|
"ashlrTokens": 406,
|
|
73
|
-
"ratio": 0.
|
|
73
|
+
"ratio": 0.20463709677419356
|
|
74
74
|
},
|
|
75
75
|
{
|
|
76
|
-
"path": "/
|
|
76
|
+
"path": "/workspace/repos/ashlr-plugin/hooks/session-end-consolidate.ts",
|
|
77
77
|
"bucket": "5-15KB",
|
|
78
|
-
"rawBytes":
|
|
79
|
-
"rawTokens":
|
|
80
|
-
"ashlrBytes":
|
|
78
|
+
"rawBytes": 7141,
|
|
79
|
+
"rawTokens": 1779,
|
|
80
|
+
"ashlrBytes": 1633,
|
|
81
81
|
"ashlrTokens": 406,
|
|
82
|
-
"ratio": 0.
|
|
82
|
+
"ratio": 0.22821810005621135
|
|
83
83
|
},
|
|
84
84
|
{
|
|
85
|
-
"path": "/
|
|
85
|
+
"path": "/workspace/repos/ashlr-plugin/docs/pro-backend-architecture.md",
|
|
86
86
|
"bucket": "15-50KB",
|
|
87
|
-
"rawBytes":
|
|
88
|
-
"rawTokens":
|
|
89
|
-
"ashlrBytes":
|
|
87
|
+
"rawBytes": 26009,
|
|
88
|
+
"rawTokens": 6481,
|
|
89
|
+
"ashlrBytes": 1629,
|
|
90
90
|
"ashlrTokens": 406,
|
|
91
|
-
"ratio": 0.
|
|
91
|
+
"ratio": 0.06264465360283906
|
|
92
92
|
},
|
|
93
93
|
{
|
|
94
|
-
"path": "/
|
|
94
|
+
"path": "/workspace/repos/ashlr-plugin/bench/refs/python-lib/pandas/core/arrays/sparse/accessor.py",
|
|
95
95
|
"bucket": "15-50KB",
|
|
96
|
-
"rawBytes":
|
|
97
|
-
"rawTokens":
|
|
98
|
-
"ashlrBytes":
|
|
96
|
+
"rawBytes": 15990,
|
|
97
|
+
"rawTokens": 3998,
|
|
98
|
+
"ashlrBytes": 1623,
|
|
99
99
|
"ashlrTokens": 406,
|
|
100
|
-
"ratio": 0.
|
|
100
|
+
"ratio": 0.10155077538769385
|
|
101
101
|
},
|
|
102
102
|
{
|
|
103
|
-
"path": "/
|
|
103
|
+
"path": "/workspace/repos/ashlr-plugin/scripts/session-greet.ts",
|
|
104
104
|
"bucket": "15-50KB",
|
|
105
|
-
"rawBytes":
|
|
106
|
-
"rawTokens":
|
|
107
|
-
"ashlrBytes":
|
|
105
|
+
"rawBytes": 23669,
|
|
106
|
+
"rawTokens": 5878,
|
|
107
|
+
"ashlrBytes": 1645,
|
|
108
108
|
"ashlrTokens": 406,
|
|
109
|
-
"ratio": 0.
|
|
109
|
+
"ratio": 0.06907111262334127
|
|
110
110
|
},
|
|
111
111
|
{
|
|
112
|
-
"path": "/
|
|
112
|
+
"path": "/workspace/repos/ashlr-plugin/servers/multi-edit-server.ts",
|
|
113
113
|
"bucket": "15-50KB",
|
|
114
|
-
"rawBytes":
|
|
115
|
-
"rawTokens":
|
|
116
|
-
"ashlrBytes":
|
|
114
|
+
"rawBytes": 15955,
|
|
115
|
+
"rawTokens": 3981,
|
|
116
|
+
"ashlrBytes": 1625,
|
|
117
117
|
"ashlrTokens": 406,
|
|
118
|
-
"ratio": 0.
|
|
118
|
+
"ratio": 0.10198442602361216
|
|
119
119
|
},
|
|
120
120
|
{
|
|
121
|
-
"path": "/
|
|
121
|
+
"path": "/workspace/repos/ashlr-plugin/site/app/admin/wad-d/page.tsx",
|
|
122
122
|
"bucket": "50+KB",
|
|
123
|
-
"rawBytes":
|
|
124
|
-
"rawTokens":
|
|
125
|
-
"ashlrBytes":
|
|
123
|
+
"rawBytes": 62608,
|
|
124
|
+
"rawTokens": 15633,
|
|
125
|
+
"ashlrBytes": 1629,
|
|
126
126
|
"ashlrTokens": 406,
|
|
127
|
-
"ratio": 0.
|
|
127
|
+
"ratio": 0.025970703000063967
|
|
128
128
|
},
|
|
129
129
|
{
|
|
130
|
-
"path": "/
|
|
130
|
+
"path": "/workspace/repos/ashlr-plugin/scripts/onboarding-wizard.ts",
|
|
131
131
|
"bucket": "50+KB",
|
|
132
|
-
"rawBytes":
|
|
133
|
-
"rawTokens":
|
|
134
|
-
"ashlrBytes":
|
|
132
|
+
"rawBytes": 59202,
|
|
133
|
+
"rawTokens": 14693,
|
|
134
|
+
"ashlrBytes": 1627,
|
|
135
135
|
"ashlrTokens": 406,
|
|
136
|
-
"ratio": 0.
|
|
136
|
+
"ratio": 0.027632205812291567
|
|
137
137
|
},
|
|
138
138
|
{
|
|
139
|
-
"path": "/
|
|
139
|
+
"path": "/workspace/repos/ashlr-plugin/site/public/hero.mp4",
|
|
140
140
|
"bucket": "50+KB",
|
|
141
|
-
"rawBytes":
|
|
142
|
-
"rawTokens":
|
|
143
|
-
"ashlrBytes":
|
|
141
|
+
"rawBytes": 5619790,
|
|
142
|
+
"rawTokens": 781851,
|
|
143
|
+
"ashlrBytes": 2301,
|
|
144
144
|
"ashlrTokens": 406,
|
|
145
|
-
"ratio": 0.
|
|
145
|
+
"ratio": 0.0005192805278755159
|
|
146
146
|
},
|
|
147
147
|
{
|
|
148
|
-
"path": "/
|
|
148
|
+
"path": "/workspace/repos/ashlr-plugin/server/bun.lock",
|
|
149
149
|
"bucket": "50+KB",
|
|
150
|
-
"rawBytes":
|
|
151
|
-
"rawTokens":
|
|
150
|
+
"rawBytes": 59684,
|
|
151
|
+
"rawTokens": 14921,
|
|
152
152
|
"ashlrBytes": 1623,
|
|
153
153
|
"ashlrTokens": 406,
|
|
154
|
-
"ratio": 0.
|
|
154
|
+
"ratio": 0.02720997252194893
|
|
155
155
|
}
|
|
156
156
|
],
|
|
157
157
|
"ashlr__grep": [
|
|
158
158
|
{
|
|
159
159
|
"pattern": "function ",
|
|
160
|
-
"rawBytes":
|
|
161
|
-
"rawTokens":
|
|
162
|
-
"ashlrBytes":
|
|
163
|
-
"ashlrTokens":
|
|
164
|
-
"ratio": 0.
|
|
160
|
+
"rawBytes": 476041,
|
|
161
|
+
"rawTokens": 118991,
|
|
162
|
+
"ashlrBytes": 3590,
|
|
163
|
+
"ashlrTokens": 893,
|
|
164
|
+
"ratio": 0.007504769268263986,
|
|
165
165
|
"method": "rg-truncated"
|
|
166
166
|
},
|
|
167
167
|
{
|
|
168
168
|
"pattern": "import ",
|
|
169
|
-
"rawBytes":
|
|
170
|
-
"rawTokens":
|
|
171
|
-
"ashlrBytes":
|
|
172
|
-
"ashlrTokens":
|
|
173
|
-
"ratio": 0.
|
|
169
|
+
"rawBytes": 527922,
|
|
170
|
+
"rawTokens": 131965,
|
|
171
|
+
"ashlrBytes": 3475,
|
|
172
|
+
"ashlrTokens": 869,
|
|
173
|
+
"ratio": 0.006585079377107566,
|
|
174
174
|
"method": "rg-truncated"
|
|
175
175
|
},
|
|
176
176
|
{
|
|
177
177
|
"pattern": "TODO",
|
|
178
|
-
"rawBytes":
|
|
179
|
-
"rawTokens":
|
|
180
|
-
"ashlrBytes":
|
|
181
|
-
"ashlrTokens":
|
|
182
|
-
"ratio": 0.
|
|
178
|
+
"rawBytes": 15107,
|
|
179
|
+
"rawTokens": 3768,
|
|
180
|
+
"ashlrBytes": 3614,
|
|
181
|
+
"ashlrTokens": 902,
|
|
182
|
+
"ratio": 0.23938428874734607,
|
|
183
183
|
"method": "rg-truncated"
|
|
184
184
|
},
|
|
185
185
|
{
|
|
186
186
|
"pattern": "class ",
|
|
187
|
-
"rawBytes":
|
|
188
|
-
"rawTokens":
|
|
189
|
-
"ashlrBytes":
|
|
190
|
-
"ashlrTokens":
|
|
191
|
-
"ratio": 0.
|
|
187
|
+
"rawBytes": 26044,
|
|
188
|
+
"rawTokens": 6499,
|
|
189
|
+
"ashlrBytes": 3593,
|
|
190
|
+
"ashlrTokens": 897,
|
|
191
|
+
"ratio": 0.13802123403600555,
|
|
192
192
|
"method": "rg-truncated"
|
|
193
193
|
},
|
|
194
194
|
{
|
|
195
195
|
"pattern": "interface ",
|
|
196
|
-
"rawBytes":
|
|
197
|
-
"rawTokens":
|
|
198
|
-
"ashlrBytes":
|
|
199
|
-
"ashlrTokens":
|
|
200
|
-
"ratio": 0.
|
|
196
|
+
"rawBytes": 86823,
|
|
197
|
+
"rawTokens": 21705,
|
|
198
|
+
"ashlrBytes": 3621,
|
|
199
|
+
"ashlrTokens": 905,
|
|
200
|
+
"ratio": 0.041695461875143974,
|
|
201
201
|
"method": "rg-truncated"
|
|
202
202
|
}
|
|
203
203
|
],
|
|
@@ -208,9 +208,9 @@
|
|
|
208
208
|
"replaceChars": 15,
|
|
209
209
|
"naiveBytes": 47,
|
|
210
210
|
"naiveTokens": 12,
|
|
211
|
-
"ashlrBytes":
|
|
212
|
-
"ashlrTokens":
|
|
213
|
-
"ratio":
|
|
211
|
+
"ashlrBytes": 47,
|
|
212
|
+
"ashlrTokens": 12,
|
|
213
|
+
"ratio": 1
|
|
214
214
|
},
|
|
215
215
|
{
|
|
216
216
|
"size": "medium",
|
|
@@ -218,9 +218,9 @@
|
|
|
218
218
|
"replaceChars": 206,
|
|
219
219
|
"naiveBytes": 407,
|
|
220
220
|
"naiveTokens": 102,
|
|
221
|
-
"ashlrBytes":
|
|
222
|
-
"ashlrTokens":
|
|
223
|
-
"ratio": 0.
|
|
221
|
+
"ashlrBytes": 47,
|
|
222
|
+
"ashlrTokens": 12,
|
|
223
|
+
"ratio": 0.11764705882352941
|
|
224
224
|
},
|
|
225
225
|
{
|
|
226
226
|
"size": "large",
|
|
@@ -228,57 +228,97 @@
|
|
|
228
228
|
"replaceChars": 2383,
|
|
229
229
|
"naiveBytes": 4643,
|
|
230
230
|
"naiveTokens": 1161,
|
|
231
|
-
"ashlrBytes":
|
|
232
|
-
"ashlrTokens":
|
|
233
|
-
"ratio": 0.
|
|
231
|
+
"ashlrBytes": 47,
|
|
232
|
+
"ashlrTokens": 12,
|
|
233
|
+
"ratio": 0.0103359173126615
|
|
234
234
|
}
|
|
235
235
|
]
|
|
236
236
|
},
|
|
237
237
|
"aggregate": {
|
|
238
238
|
"ashlr__read": {
|
|
239
|
-
"mean": 0.
|
|
240
|
-
"p50": 0.
|
|
241
|
-
"p90": 0.
|
|
239
|
+
"mean": 0.20628631347686915,
|
|
240
|
+
"p50": 0.20463709677419356,
|
|
241
|
+
"p90": 0.4709976798143852,
|
|
242
|
+
"confidenceInterval": {
|
|
243
|
+
"lo": 0.11534154808002625,
|
|
244
|
+
"hi": 0.30378857306208595
|
|
245
|
+
}
|
|
242
246
|
},
|
|
243
247
|
"ashlr__grep": {
|
|
244
|
-
"mean": 0.
|
|
245
|
-
"p50": 0.
|
|
246
|
-
"p90": 0.
|
|
248
|
+
"mean": 0.08663816666077342,
|
|
249
|
+
"p50": 0.041695461875143974,
|
|
250
|
+
"p90": 0.23938428874734607,
|
|
251
|
+
"confidenceInterval": {
|
|
252
|
+
"lo": 0.014158969811408701,
|
|
253
|
+
"hi": 0.17273577390926154
|
|
254
|
+
}
|
|
247
255
|
},
|
|
248
256
|
"ashlr__edit": {
|
|
249
|
-
"mean":
|
|
250
|
-
"p50": 0.
|
|
251
|
-
"p90":
|
|
257
|
+
"mean": 0.37599432537873034,
|
|
258
|
+
"p50": 0.11764705882352941,
|
|
259
|
+
"p90": 1,
|
|
260
|
+
"confidenceInterval": {
|
|
261
|
+
"lo": 0.0103359173126615,
|
|
262
|
+
"hi": 1
|
|
263
|
+
}
|
|
252
264
|
},
|
|
253
265
|
"overall": {
|
|
254
|
-
"mean": 0.
|
|
266
|
+
"mean": 0.20257311771124853
|
|
267
|
+
}
|
|
268
|
+
},
|
|
269
|
+
"methodology": "Measurement methodology (version 2):\n\n**ashlr__read**: For each sampled source file, we measure raw file bytes and\ntoken count (chars/4 heuristic). We then apply the same snipCompact\ntransformation used at runtime — wrapping the content in a tool_result message\nand calling snipCompact() — and measure the resulting byte/token count. The\nratio is ashlrTokens / rawTokens. Files below 2 KB are excluded because\nsnipCompact only fires on tool results > 2 000 chars; savings are zero by\ndesign for small files.\n\nFiles are selected deterministically: the repo HEAD commit SHA is folded into a\n32-bit seed (mulberry32 PRNG), then up to 4 files are sampled from each of four\nsize buckets (2–5 KB, 5–15 KB, 15–50 KB, 50+ KB). Re-running on the same\ncommit always picks the same files.\n\n**ashlr__grep**: Five common patterns (function, import, TODO, class, interface)\nare run via plain `rg -n` (path:line:text, the shape a native grep returns)\nagainst the repo root; that output is the raw size. The ashlr__grep fallback\npath (no genome) caps lines at 300 columns and, above 4 000 chars, keeps whole\nlines from the head and tail. The ratio is ashlr/raw.\n\nNote: when a .ashlrcode/genome/ index is present, real-world grep savings are\nsubstantially higher. This benchmark measures only the conservative\nno-genome baseline.\n\n**ashlr__edit**: Three synthetic edits (small ~15 chars, medium ~300 chars,\nlarge ~3 000 chars) compare the naive \"ship before+after as text\" approach\nagainst ashlr__edit's diff-summary format (one header line + removed/added\nfirst-lines). The ratio is summary tokens / naive tokens.\n\n**Aggregation**: per-tool mean/p50/p90 are computed over each tool's ratio\nvalues (lower ratio = more savings). The `overall.mean` is pooled across\nevery individual sample regardless of tool — so tools with more samples (read\nhas 15, grep has 5, edit has 3) weight the overall figure proportionally.\nThat makes the headline number reflect the workload mix, not a uniform\nper-tool average. The unweighted mean of per-tool means is intentionally NOT\npublished because it gives equal weight to a 3-sample tool and a 15-sample\ntool, which over-weights the synthetic edit overhead.\n\nToken counts use the chars/4 heuristic, the same estimator the plugin uses at\nruntime for savings accounting.",
|
|
270
|
+
"byLanguage": {
|
|
271
|
+
"ts": {
|
|
272
|
+
"language": "ts",
|
|
273
|
+
"meanRatio": 0.40022385340539585,
|
|
274
|
+
"savingsPct": 59.97761465946041
|
|
275
|
+
},
|
|
276
|
+
"py": {
|
|
277
|
+
"language": "py",
|
|
278
|
+
"meanRatio": 0.3669006822892532,
|
|
279
|
+
"savingsPct": 63.30993177107468
|
|
280
|
+
},
|
|
281
|
+
"rs": {
|
|
282
|
+
"language": "rs",
|
|
283
|
+
"meanRatio": 0.4466227683666335,
|
|
284
|
+
"savingsPct": 55.33772316333665
|
|
255
285
|
}
|
|
256
286
|
},
|
|
257
|
-
"methodology": "Measurement methodology (version 2):\n\n**ashlr__read**: For each sampled source file, we measure raw file bytes and\ntoken count (chars/4 heuristic). We then apply the same snipCompact\ntransformation used at runtime — wrapping the content in a tool_result message\nand calling snipCompact() — and measure the resulting byte/token count. The\nratio is ashlrTokens / rawTokens. Files below 2 KB are excluded because\nsnipCompact only fires on tool results > 2 000 chars; savings are zero by\ndesign for small files.\n\nFiles are selected deterministically: the repo HEAD commit SHA is folded into a\n32-bit seed (mulberry32 PRNG), then up to 4 files are sampled from each of four\nsize buckets (2–5 KB, 5–15 KB, 15–50 KB, 50+ KB). Re-running on the same\ncommit always picks the same files.\n\n**ashlr__grep**: Five common patterns (function, import, TODO, class, interface)\nare run via rg --json against the repo root. Raw output bytes are measured\ndirectly. The ashlr__grep fallback path (no genome) truncates output to 4 000\nchars (head 2 000 + tail 1 000). The ratio is truncated/raw.\n\nNote: when a .ashlrcode/genome/ index is present, real-world grep savings are\nsubstantially higher. This benchmark measures only the conservative\nno-genome baseline.\n\n**ashlr__edit**: Three synthetic edits (small ~15 chars, medium ~300 chars,\nlarge ~3 000 chars) compare the naive \"ship before+after as text\" approach\nagainst ashlr__edit's diff-summary format (one header line + removed/added\nfirst-lines). The ratio is summary tokens / naive tokens.\n\n**Aggregation**: per-tool mean/p50/p90 are computed over each tool's ratio\nvalues (lower ratio = more savings). The `overall.mean` is pooled across\nevery individual sample regardless of tool — so tools with more samples (read\nhas 15, grep has 5, edit has 3) weight the overall figure proportionally.\nThat makes the headline number reflect the workload mix, not a uniform\nper-tool average. The unweighted mean of per-tool means is intentionally NOT\npublished because it gives equal weight to a 3-sample tool and a 15-sample\ntool, which over-weights the synthetic edit overhead.\n\nToken counts use the chars/4 heuristic, the same estimator the plugin uses at\nruntime for savings accounting.",
|
|
258
|
-
"crossLanguageMeasuredAt": "2026-05-23T00:07:25.178Z",
|
|
259
287
|
"byRepo": {
|
|
260
288
|
"node-sdk": {
|
|
261
289
|
"repoKey": "node-sdk",
|
|
262
290
|
"language": "ts",
|
|
263
291
|
"upstreamRepo": "github.com/vercel/ai",
|
|
264
292
|
"refrev": "049801243cc791e0ba28910b8a638f0e05446bcd",
|
|
265
|
-
"meanRatio": 0.
|
|
266
|
-
"savingsPct":
|
|
293
|
+
"meanRatio": 0.40022385340539585,
|
|
294
|
+
"savingsPct": 59.97761465946041,
|
|
267
295
|
"perTool": {
|
|
268
296
|
"read": {
|
|
269
|
-
"mean": 0.
|
|
270
|
-
"p50": 0.
|
|
271
|
-
"p90": 0.5638888888888889
|
|
297
|
+
"mean": 0.25088522186948287,
|
|
298
|
+
"p50": 0.20598680872653474,
|
|
299
|
+
"p90": 0.5638888888888889,
|
|
300
|
+
"confidenceInterval": {
|
|
301
|
+
"lo": 0.14518377534384794,
|
|
302
|
+
"hi": 0.3647952153962575
|
|
303
|
+
}
|
|
272
304
|
},
|
|
273
305
|
"grep": {
|
|
274
|
-
"mean": 0.
|
|
275
|
-
"p50":
|
|
276
|
-
"p90":
|
|
306
|
+
"mean": 0.6956295729319569,
|
|
307
|
+
"p50": 1,
|
|
308
|
+
"p90": 1,
|
|
309
|
+
"confidenceInterval": {
|
|
310
|
+
"lo": 0.3270688989760508,
|
|
311
|
+
"hi": 1
|
|
312
|
+
}
|
|
277
313
|
},
|
|
278
314
|
"edit": {
|
|
279
|
-
"mean":
|
|
315
|
+
"mean": 0.5052355136714463,
|
|
280
316
|
"p50": 0.4803921568627451,
|
|
281
|
-
"p90":
|
|
317
|
+
"p90": 1,
|
|
318
|
+
"confidenceInterval": {
|
|
319
|
+
"lo": 0.03531438415159346,
|
|
320
|
+
"hi": 1
|
|
321
|
+
}
|
|
282
322
|
}
|
|
283
323
|
}
|
|
284
324
|
},
|
|
@@ -287,23 +327,35 @@
|
|
|
287
327
|
"language": "py",
|
|
288
328
|
"upstreamRepo": "github.com/pandas-dev/pandas",
|
|
289
329
|
"refrev": "be0642f7cb8900282a43b53cd50cf00a05161ed6",
|
|
290
|
-
"meanRatio": 0.
|
|
291
|
-
"savingsPct": 63.
|
|
330
|
+
"meanRatio": 0.3669006822892532,
|
|
331
|
+
"savingsPct": 63.30993177107468,
|
|
292
332
|
"perTool": {
|
|
293
333
|
"read": {
|
|
294
|
-
"mean": 0.
|
|
295
|
-
"p50": 0.
|
|
296
|
-
"p90": 0.
|
|
334
|
+
"mean": 0.2605938960700735,
|
|
335
|
+
"p50": 0.2415229030339084,
|
|
336
|
+
"p90": 0.5615491009681881,
|
|
337
|
+
"confidenceInterval": {
|
|
338
|
+
"lo": 0.16523486802250215,
|
|
339
|
+
"hi": 0.3615802926482504
|
|
340
|
+
}
|
|
297
341
|
},
|
|
298
342
|
"grep": {
|
|
299
|
-
"mean": 0.
|
|
300
|
-
"p50":
|
|
301
|
-
"p90": 1
|
|
343
|
+
"mean": 0.6086466139649426,
|
|
344
|
+
"p50": 1,
|
|
345
|
+
"p90": 1,
|
|
346
|
+
"confidenceInterval": {
|
|
347
|
+
"lo": 0.21729322792988506,
|
|
348
|
+
"hi": 1
|
|
349
|
+
}
|
|
302
350
|
},
|
|
303
351
|
"edit": {
|
|
304
|
-
"mean":
|
|
352
|
+
"mean": 0.5052355136714463,
|
|
305
353
|
"p50": 0.4803921568627451,
|
|
306
|
-
"p90":
|
|
354
|
+
"p90": 1,
|
|
355
|
+
"confidenceInterval": {
|
|
356
|
+
"lo": 0.03531438415159346,
|
|
357
|
+
"hi": 1
|
|
358
|
+
}
|
|
307
359
|
}
|
|
308
360
|
}
|
|
309
361
|
},
|
|
@@ -312,44 +364,41 @@
|
|
|
312
364
|
"language": "rs",
|
|
313
365
|
"upstreamRepo": "github.com/tokio-rs/tokio",
|
|
314
366
|
"refrev": "6c03e03898d71eca976ee1ad8481cf112ae722ba",
|
|
315
|
-
"meanRatio": 0.
|
|
316
|
-
"savingsPct":
|
|
367
|
+
"meanRatio": 0.4466227683666335,
|
|
368
|
+
"savingsPct": 55.33772316333665,
|
|
317
369
|
"perTool": {
|
|
318
370
|
"read": {
|
|
319
|
-
"mean": 0.
|
|
320
|
-
"p50": 0.
|
|
321
|
-
"p90": 0.
|
|
371
|
+
"mean": 0.2563988144536817,
|
|
372
|
+
"p50": 0.1467823571945047,
|
|
373
|
+
"p90": 0.628482972136223,
|
|
374
|
+
"confidenceInterval": {
|
|
375
|
+
"lo": 0.1396058310137809,
|
|
376
|
+
"hi": 0.3891548688756394
|
|
377
|
+
}
|
|
322
378
|
},
|
|
323
379
|
"grep": {
|
|
324
|
-
"mean": 0.
|
|
380
|
+
"mean": 0.8782230941704036,
|
|
325
381
|
"p50": 1,
|
|
326
|
-
"p90": 1
|
|
382
|
+
"p90": 1,
|
|
383
|
+
"confidenceInterval": {
|
|
384
|
+
"lo": 0.6346692825112107,
|
|
385
|
+
"hi": 1
|
|
386
|
+
}
|
|
327
387
|
},
|
|
328
388
|
"edit": {
|
|
329
|
-
"mean":
|
|
389
|
+
"mean": 0.5052355136714463,
|
|
330
390
|
"p50": 0.4803921568627451,
|
|
331
|
-
"p90":
|
|
391
|
+
"p90": 1,
|
|
392
|
+
"confidenceInterval": {
|
|
393
|
+
"lo": 0.03531438415159346,
|
|
394
|
+
"hi": 1
|
|
395
|
+
}
|
|
332
396
|
}
|
|
333
397
|
}
|
|
334
398
|
}
|
|
335
399
|
},
|
|
336
|
-
"
|
|
337
|
-
|
|
338
|
-
|
|
339
|
-
|
|
340
|
-
|
|
341
|
-
},
|
|
342
|
-
"py": {
|
|
343
|
-
"language": "py",
|
|
344
|
-
"meanRatio": 0.3688098954558492,
|
|
345
|
-
"savingsPct": 63.11901045441508
|
|
346
|
-
},
|
|
347
|
-
"rs": {
|
|
348
|
-
"language": "rs",
|
|
349
|
-
"meanRatio": 0.532296183969948,
|
|
350
|
-
"savingsPct": 46.7703816030052
|
|
351
|
-
}
|
|
352
|
-
},
|
|
353
|
-
"crossLanguageMean": 0.42925039767920997,
|
|
354
|
-
"crossLanguageSavingsPct": 57.07496023207901
|
|
355
|
-
}
|
|
400
|
+
"crossLanguageMean": 0.40458243468709415,
|
|
401
|
+
"crossLanguageMeasuredAt": "2026-10-17T02:32:44.002Z",
|
|
402
|
+
"crossLanguageSavingsPct": 59.54175653129059,
|
|
403
|
+
"caveat": "Synthetic benchmark. Plugin-self figures (samples, aggregate) were measured on commit 364a4030d3bdb02271c04e18d1632ef6bd00d8f1 at measuredAt (2026-10-19). Read samples are chosen per commit, so figures from different commits are not like-for-like. The edit scenarios are fixed synthetic edits, not real sessions. Cross-language sections (byLanguage, byRepo, crossLanguage*) were not re-measured; they keep their own crossLanguageMeasuredAt."
|
|
404
|
+
}
|
package/docs/benchmarks.md
CHANGED
|
@@ -1,8 +1,8 @@
|
|
|
1
1
|
# ashlr-plugin Token-Savings Benchmarks
|
|
2
2
|
|
|
3
|
-
> **Latest run:**
|
|
4
|
-
> **Headline:** `−
|
|
5
|
-
> (TS/vercel-ai `−
|
|
3
|
+
> **Latest run:** 2026-10-17 (`docs/benchmarks-v2.json`)
|
|
4
|
+
> **Headline:** per-call figures only: reads about `−75%`, greps about `−27%` (not end-to-end; the `−60%` cross-language mean below is a per-call mean, not total session savings)
|
|
5
|
+
> (TS/vercel-ai `−60%`, Python/pandas `−63%`, Rust/tokio `−55%`)
|
|
6
6
|
>
|
|
7
7
|
> _Prior self-repo figure (`−74%`) is preserved in `docs/benchmarks-v2.json`
|
|
8
8
|
> under `aggregate.overall.mean` but is no longer the headline — it reflects
|
|
@@ -118,17 +118,23 @@ utilities, medium modules, and large implementation files. They collectively
|
|
|
118
118
|
represent agentic AI SDK code, data-science library internals, and systems
|
|
119
119
|
async runtime code — meaningfully different workloads.
|
|
120
120
|
|
|
121
|
-
### Results (measured 2026-
|
|
121
|
+
### Results (measured 2026-10-17)
|
|
122
122
|
|
|
123
123
|
| Repo | Overall | Read | Grep | Edit |
|
|
124
124
|
|------|---------|------|------|------|
|
|
125
|
-
| `node-sdk` (TS) | **−
|
|
126
|
-
| `python-lib` (Py) | **−
|
|
127
|
-
| `rust-project` (Rs) | **−
|
|
128
|
-
| **Cross-language mean** | **−
|
|
125
|
+
| `node-sdk` (TS) | **−60.0%** | −74.9% | −30.4% | −49.5%* |
|
|
126
|
+
| `python-lib` (Py) | **−63.3%** | −73.9% | −39.1% | −49.5%* |
|
|
127
|
+
| `rust-project` (Rs) | **−55.3%** | −74.4% | −12.2% | −49.5%* |
|
|
128
|
+
| **Cross-language mean** | **−60%** | **−74%** | **−27%** | −49.5%* |
|
|
129
129
|
|
|
130
|
-
_*Edit
|
|
131
|
-
|
|
130
|
+
_*Edit figures come from the same three synthetic edits in every repo (see
|
|
131
|
+
the edit caveat above), so they do not vary by language._
|
|
132
|
+
|
|
133
|
+
**Grep baseline (changed 2026-10-17):** the raw side is plain `rg -n` output
|
|
134
|
+
(`path:line:text`, what a native grep returns). Earlier runs measured against
|
|
135
|
+
`rg --json`, which is about three times larger for the same matches, and
|
|
136
|
+
reported greps at about −50%. The fallback now returns complete
|
|
137
|
+
`path:line:text` lines instead of a cut-down JSON stream.
|
|
132
138
|
|
|
133
139
|
**Note on grep for Rust:** ripgrep patterns `import`, `TODO`, `class`, and
|
|
134
140
|
`interface` have low match rates in Rust source (Rust uses `use`, `//TODO:`,
|
|
@@ -209,3 +215,7 @@ v1.22 Track G adds dashboard sections that show real-world data:
|
|
|
209
215
|
|
|
210
216
|
These are the numbers worth tracking weekly. The bench is a calibration
|
|
211
217
|
tool; the telemetry is the live feedback loop.
|
|
218
|
+
|
|
219
|
+
## Note: line numbers add overhead on very small files
|
|
220
|
+
|
|
221
|
+
Reads of code files keep line numbers by default so edits and references stay precise. On very small files the line-number prefix can cost more than the compaction saves. No committed benchmark measures this yet: as a rough byte count (not tokens), the `N: ` prefix adds about 13% to files under 1.7 KB at main a0f656b. Pass `preserveLineNumbers: false` to `ashlr__read` to get the file without line numbers; the default is unchanged.
|