@sliday/tamp 0.8.15 → 0.8.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/compress.js +8 -2
- package/lib/bm25.js +334 -44
- package/lib/code-chunks.js +269 -0
- package/package.json +3 -1
package/compress.js
CHANGED
|
@@ -167,14 +167,20 @@ function bm25TrimTargets(targets, body, provider) {
|
|
|
167
167
|
if (!userText) return 0
|
|
168
168
|
if (isDangerousTask(userText)) return 0
|
|
169
169
|
|
|
170
|
+
// File paths let the trimmer pick a block detector by extension so dropped
|
|
171
|
+
// lines respect code structure. Anthropic-only (see lib/path-extract.js);
|
|
172
|
+
// elsewhere `path` is null and the trimmer sniffs the body instead.
|
|
173
|
+
const paths = extractTargetPaths(body, targets)
|
|
174
|
+
|
|
170
175
|
let trimmed = 0
|
|
171
|
-
for (
|
|
176
|
+
for (let i = 0; i < targets.length; i++) {
|
|
177
|
+
const target = targets[i]
|
|
172
178
|
if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.compressed) continue
|
|
173
179
|
if (typeof target.text !== 'string') continue
|
|
174
180
|
const byteLen = Buffer.byteLength(target.text, 'utf8')
|
|
175
181
|
if (byteLen <= 64 * 1024) continue
|
|
176
182
|
|
|
177
|
-
const result = trimLinesByRelevance(target.text, userText)
|
|
183
|
+
const result = trimLinesByRelevance(target.text, userText, { path: paths[i] })
|
|
178
184
|
if (!result) continue
|
|
179
185
|
if (result.text.length >= target.text.length) continue
|
|
180
186
|
target.compressed = result.text
|
package/lib/bm25.js
CHANGED
|
@@ -10,28 +10,123 @@
|
|
|
10
10
|
// This primitive is reusable: any future stage that wants "keep the most
|
|
11
11
|
// relevant N lines" can import tokenize/bm25Scores directly.
|
|
12
12
|
|
|
13
|
-
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
13
|
+
import { countTokens as rawCountTokens } from '@anthropic-ai/tokenizer'
|
|
14
|
+
import { detectBlocks, buildStructure, indentWidth } from './code-chunks.js'
|
|
15
|
+
|
|
16
|
+
// --- Safe token counting (C5) ---
|
|
17
|
+
//
|
|
18
|
+
// `countTokens` is NOT the fixed-cost call an earlier comment in this file
|
|
19
|
+
// claimed. It has a ~22ms floor on small inputs, but it is superlinear on
|
|
20
|
+
// large low-entropy input and it HARD CRASHES on some of it. Measured on
|
|
21
|
+
// whitespace-heavy bodies (400-char space runs):
|
|
22
|
+
//
|
|
23
|
+
// 80KB -> 8.7s 200KB -> 22.4s 400KB -> 88.9s
|
|
24
|
+
// 1MB -> RuntimeError: unreachable (tiktoken WASM trap)
|
|
25
|
+
//
|
|
26
|
+
// `bm25-trim` only runs on bodies over 64KB, so every attacker-controlled
|
|
27
|
+
// tool_result on this path paid it. A WASM trap is not catchable as a normal
|
|
28
|
+
// error in every runtime, so the primary defence is the size cap; the
|
|
29
|
+
// try/catch is only the backstop.
|
|
30
|
+
//
|
|
31
|
+
// The cap is 4KB, not 64KB, because the cost is driven by CONTENT, not size.
|
|
32
|
+
// Measured:
|
|
33
|
+
//
|
|
34
|
+
// spaces: 1KB 22ms 4KB 31ms 8KB 58ms 16KB 167ms 32KB 609ms
|
|
35
|
+
// prose: 8KB 22ms 32KB 24ms 64KB 27ms
|
|
36
|
+
//
|
|
37
|
+
// Prose is flat at the ~22ms floor whatever its length; pathological runs go
|
|
38
|
+
// quadratic. 4KB bounds the worst case to ~31ms. A 64KB cap still measured
|
|
39
|
+
// 44s on a megabyte of spaces, which is why the first attempt at this fix
|
|
40
|
+
// did not work.
|
|
41
|
+
const COUNT_CAP = 4 * 1024
|
|
42
|
+
const SAMPLES = 3
|
|
43
|
+
|
|
44
|
+
export function safeCountTokens(text) {
|
|
45
|
+
if (typeof text !== 'string' || text.length === 0) return 0
|
|
46
|
+
try {
|
|
47
|
+
if (text.length <= COUNT_CAP) return rawCountTokens(text)
|
|
48
|
+
// Three windows spread across the body rather than one prefix — same cost,
|
|
49
|
+
// better representativeness on a body whose head is not typical of its tail
|
|
50
|
+
// (a log with a JSON blob at the end, a file with a licence header).
|
|
51
|
+
const each = Math.floor(COUNT_CAP / SAMPLES)
|
|
52
|
+
const stride = Math.floor((text.length - each) / (SAMPLES - 1))
|
|
53
|
+
let sampled = 0
|
|
54
|
+
let counted = 0
|
|
55
|
+
for (let s = 0; s < SAMPLES; s++) {
|
|
56
|
+
const start = Math.min(s * stride, text.length - each)
|
|
57
|
+
const window = text.slice(start, start + each)
|
|
58
|
+
counted += rawCountTokens(window)
|
|
59
|
+
sampled += window.length
|
|
60
|
+
}
|
|
61
|
+
if (sampled === 0) return Math.max(1, Math.ceil(text.length / 4))
|
|
62
|
+
return Math.max(1, Math.round(counted * (text.length / sampled)))
|
|
63
|
+
} catch {
|
|
64
|
+
// Tokenizer refused this input. Fall back to a coarse estimate rather than
|
|
65
|
+
// aborting compression for the whole request.
|
|
66
|
+
return Math.max(1, Math.ceil(text.length / 4))
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const countTokens = safeCountTokens
|
|
14
71
|
|
|
15
72
|
// --- Tokenizer ---
|
|
16
73
|
// Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
|
|
17
74
|
// `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
|
|
18
75
|
// separators to underscore before splitting on non-word characters. Single
|
|
19
76
|
// character tokens are dropped (too noisy for BM25 on short queries).
|
|
77
|
+
//
|
|
78
|
+
// Emitting ONLY the collapsed form silently destroys recall on the queries
|
|
79
|
+
// agents actually ask. `parse config` tokenizes to ['parse','config'], while
|
|
80
|
+
// `function parseConfig(` yields ['parseconfig'] and `def parse_config(`
|
|
81
|
+
// yields ['parse_config'] — no shared term, so BM25 scores the definition
|
|
82
|
+
// line the user asked about at exactly 0 and `bm25-trim` drops it. So we also
|
|
83
|
+
// emit the identifier's parts, splitting on camelCase, PascalCase, acronym
|
|
84
|
+
// runs, and underscores. Case must survive until the split, hence the
|
|
85
|
+
// lowercase happens per-token at the end rather than up front.
|
|
20
86
|
const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
|
|
21
87
|
const SPLIT_RE = /[^A-Za-z0-9_]+/
|
|
88
|
+
// `HTTPServer` -> `HTTP Server`. Runs before CAMEL_LOWER_UPPER so that
|
|
89
|
+
// `parseHTTPResponse` splits three ways rather than swallowing the acronym.
|
|
90
|
+
const CAMEL_ACRONYM = /([A-Z]+)([A-Z][a-z])/g
|
|
91
|
+
const CAMEL_LOWER_UPPER = /([a-z0-9])([A-Z])/g
|
|
92
|
+
// No cap on parts per identifier. An earlier version capped at 4 and truncated
|
|
93
|
+
// from the TAIL, which reintroduced the exact bug this file exists to fix:
|
|
94
|
+
// `get_or_create_default_workspace_settings` emitted get/or/create/default and
|
|
95
|
+
// dropped `workspace` and `settings`, so a query for "workspace settings"
|
|
96
|
+
// scored the definition line at 0 and deleted it. The cap was motivated by an
|
|
97
|
+
// IDF-dilution worry that was never measured. If dilution is real,
|
|
98
|
+
// bench/retrieval-eval.js is where it should show up.
|
|
99
|
+
|
|
100
|
+
// Split one raw token into its component words. Returns [] when the token has
|
|
101
|
+
// no internal structure, so an already-atomic `config` is not emitted twice
|
|
102
|
+
// (which would inflate its term frequency and skew BM25).
|
|
103
|
+
function subTokens(rawTok) {
|
|
104
|
+
const spaced = rawTok
|
|
105
|
+
.replace(CAMEL_ACRONYM, '$1 $2')
|
|
106
|
+
.replace(CAMEL_LOWER_UPPER, '$1 $2')
|
|
107
|
+
.replace(/_+/g, ' ')
|
|
108
|
+
const parts = spaced.split(' ')
|
|
109
|
+
if (parts.length < 2) return []
|
|
110
|
+
const out = []
|
|
111
|
+
for (const part of parts) {
|
|
112
|
+
if (part.length < 2) continue
|
|
113
|
+
out.push(part.toLowerCase())
|
|
114
|
+
}
|
|
115
|
+
return out
|
|
116
|
+
}
|
|
22
117
|
|
|
23
118
|
export function tokenize(text) {
|
|
24
119
|
if (typeof text !== 'string' || text.length === 0) return []
|
|
25
|
-
let normalized = text.toLowerCase()
|
|
26
120
|
// Collapse intra-word separators. Run twice so chains like `a.b.c` fully
|
|
27
121
|
// collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
|
|
28
|
-
normalized =
|
|
122
|
+
let normalized = text.replace(INTRA_WORD_RE, '$1_$2')
|
|
29
123
|
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
30
124
|
const raw = normalized.split(SPLIT_RE)
|
|
31
125
|
const out = []
|
|
32
126
|
for (const tok of raw) {
|
|
33
|
-
if (tok.length
|
|
34
|
-
out.push(tok)
|
|
127
|
+
if (tok.length === 0) continue
|
|
128
|
+
if (tok.length >= 2) out.push(tok.toLowerCase())
|
|
129
|
+
for (const part of subTokens(tok)) out.push(part)
|
|
35
130
|
}
|
|
36
131
|
return out
|
|
37
132
|
}
|
|
@@ -39,60 +134,76 @@ export function tokenize(text) {
|
|
|
39
134
|
// --- BM25 ---
|
|
40
135
|
// Standard Okapi BM25. Each document is a line; query is the user's last
|
|
41
136
|
// message. Returns Float64Array of scores aligned with the documents array.
|
|
42
|
-
|
|
137
|
+
// `tokenizer` is injectable so bench/retrieval-eval.js can score the same
|
|
138
|
+
// corpus with the pre-subtoken tokenizer and report a real before/after.
|
|
139
|
+
export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75, tokenizer = tokenize } = {}) {
|
|
43
140
|
const N = documents.length
|
|
44
141
|
const scores = new Float64Array(N)
|
|
45
142
|
if (N === 0) return scores
|
|
46
143
|
|
|
47
|
-
const qTokens =
|
|
144
|
+
const qTokens = tokenizer(query)
|
|
48
145
|
if (qTokens.length === 0) return scores
|
|
49
146
|
|
|
50
|
-
//
|
|
51
|
-
|
|
147
|
+
// ONE pass over the documents (C7 + C8).
|
|
148
|
+
//
|
|
149
|
+
// C7: the old version kept `docTokens` — the full tokenization of every line
|
|
150
|
+
// — alive for the whole call, though only query terms are ever read from it.
|
|
151
|
+
// A 4.6MB body (60k lines) retained 47.4MB, roughly 10x amplification, on top
|
|
152
|
+
// of the original string, the split lines array and the parsed request body.
|
|
153
|
+
// The proxy serves concurrent requests with no per-request size cap on this
|
|
154
|
+
// stage, so that was an OOM path, not just memory pressure.
|
|
155
|
+
//
|
|
156
|
+
// C8: document frequency was computed as `docTokens[i].includes(term)` for
|
|
157
|
+
// every unique query term against every line — O(uniqQ x N x tokens/line) of
|
|
158
|
+
// linear scans. Both dimensions are hostile: N is the attacker-controlled
|
|
159
|
+
// line count, and the query is the user's last message, which routinely
|
|
160
|
+
// contains a pasted stack trace. Measured at N=20000: 70ms for a 10-word
|
|
161
|
+
// query, 163ms for 100 words, 676ms for 500.
|
|
162
|
+
//
|
|
163
|
+
// Now: tokenize each line once, keep only the per-line term frequencies for
|
|
164
|
+
// terms that appear in the query (empty for the vast majority of lines), and
|
|
165
|
+
// accumulate df in the same pass. Peak retention scales with the QUERY size,
|
|
166
|
+
// not the body size.
|
|
167
|
+
const qSet = new Set(qTokens)
|
|
52
168
|
const docLens = new Float64Array(N)
|
|
169
|
+
const docTf = new Array(N) // Map|null, only query terms, null when none present
|
|
170
|
+
const df = new Map()
|
|
53
171
|
let totalLen = 0
|
|
172
|
+
|
|
54
173
|
for (let i = 0; i < N; i++) {
|
|
55
|
-
const toks =
|
|
56
|
-
docTokens[i] = toks
|
|
174
|
+
const toks = tokenizer(documents[i])
|
|
57
175
|
docLens[i] = toks.length
|
|
58
176
|
totalLen += toks.length
|
|
177
|
+
let tf = null
|
|
178
|
+
for (const t of toks) {
|
|
179
|
+
if (!qSet.has(t)) continue
|
|
180
|
+
if (tf === null) tf = new Map()
|
|
181
|
+
tf.set(t, (tf.get(t) || 0) + 1)
|
|
182
|
+
}
|
|
183
|
+
docTf[i] = tf
|
|
184
|
+
if (tf !== null) for (const t of tf.keys()) df.set(t, (df.get(t) || 0) + 1)
|
|
185
|
+
// `toks` goes out of scope here and is collectable — nothing retains it.
|
|
59
186
|
}
|
|
187
|
+
|
|
60
188
|
const avgdl = N > 0 ? totalLen / N : 0
|
|
61
189
|
if (avgdl === 0) return scores
|
|
62
190
|
|
|
63
|
-
//
|
|
64
|
-
const uniqQ = Array.from(new Set(qTokens))
|
|
65
|
-
const df = new Map()
|
|
66
|
-
for (const term of uniqQ) {
|
|
67
|
-
let count = 0
|
|
68
|
-
for (let i = 0; i < N; i++) {
|
|
69
|
-
if (docTokens[i].includes(term)) count += 1
|
|
70
|
-
}
|
|
71
|
-
df.set(term, count)
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
|
|
191
|
+
// Okapi BM25 IDF with +1 smoothing.
|
|
75
192
|
const idf = new Map()
|
|
76
|
-
for (const term of
|
|
193
|
+
for (const term of qSet) {
|
|
77
194
|
const n = df.get(term) || 0
|
|
78
|
-
|
|
79
|
-
idf.set(term, v)
|
|
195
|
+
idf.set(term, Math.log(1 + (N - n + 0.5) / (n + 0.5)))
|
|
80
196
|
}
|
|
81
197
|
|
|
82
198
|
for (let i = 0; i < N; i++) {
|
|
199
|
+
const tf = docTf[i]
|
|
200
|
+
if (tf === null) continue // no query term on this line — scores 0, skip
|
|
83
201
|
const dl = docLens[i]
|
|
84
202
|
if (dl === 0) continue
|
|
85
|
-
// Term-frequency table for this doc (only for query terms we care about).
|
|
86
|
-
const tf = new Map()
|
|
87
|
-
const toks = docTokens[i]
|
|
88
|
-
for (const t of toks) {
|
|
89
|
-
if (!idf.has(t)) continue
|
|
90
|
-
tf.set(t, (tf.get(t) || 0) + 1)
|
|
91
|
-
}
|
|
92
203
|
let s = 0
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
204
|
+
// Iterate the terms actually PRESENT (usually 0-3) rather than every
|
|
205
|
+
// unique query term, so a long query costs nothing on unrelated lines.
|
|
206
|
+
for (const [term, f] of tf) {
|
|
96
207
|
const num = f * (k1 + 1)
|
|
97
208
|
const den = f + k1 * (1 - b + b * (dl / avgdl))
|
|
98
209
|
s += idf.get(term) * (num / den)
|
|
@@ -102,6 +213,55 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
|
102
213
|
return scores
|
|
103
214
|
}
|
|
104
215
|
|
|
216
|
+
// --- Rerank ---
|
|
217
|
+
// Raw BM25 treats every line as an equal bag of words. Two code-aware signals
|
|
218
|
+
// and one structural one fix its most visible failures.
|
|
219
|
+
//
|
|
220
|
+
// Note on scope: a path-based penalty (test files, .d.ts stubs) would be a
|
|
221
|
+
// no-op here. `bm25-trim` ranks lines WITHIN one tool_result, so a multiplier
|
|
222
|
+
// constant across the whole body cannot change any relative ordering. Path is
|
|
223
|
+
// used for language detection only.
|
|
224
|
+
|
|
225
|
+
// A line that DEFINES a query term beats a line that merely mentions it.
|
|
226
|
+
const DEF_RE = /\b(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function|class|def|const|let|var|type|interface|enum|struct|impl|trait|fn|func|module|namespace)\s+([A-Za-z_$][\w$]*)/
|
|
227
|
+
const COMMENT_RE = /^\s*(?:\/\/|#|\*|\/\*|--|<!--|;)/
|
|
228
|
+
const DEF_BOOST = 1.6
|
|
229
|
+
const COMMENT_PENALTY = 0.6
|
|
230
|
+
// A kept line lends a quarter of its score to each immediate neighbour, so the
|
|
231
|
+
// survivors read as contiguous code rather than confetti. Applied off the
|
|
232
|
+
// pre-coherence scores so boosts do not cascade down the file.
|
|
233
|
+
const COHERENCE = 0.25
|
|
234
|
+
|
|
235
|
+
export function rerankScores(scores, lines, queryTokens) {
|
|
236
|
+
const n = lines.length
|
|
237
|
+
const out = new Float64Array(n)
|
|
238
|
+
const qSet = new Set(queryTokens)
|
|
239
|
+
|
|
240
|
+
for (let i = 0; i < n; i++) {
|
|
241
|
+
let s = scores[i]
|
|
242
|
+
const line = lines[i]
|
|
243
|
+
if (typeof line !== 'string' || s === 0) { out[i] = s; continue }
|
|
244
|
+
|
|
245
|
+
const def = DEF_RE.exec(line)
|
|
246
|
+
if (def) {
|
|
247
|
+
for (const tok of tokenize(def[1])) {
|
|
248
|
+
if (qSet.has(tok)) { s *= DEF_BOOST; break }
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (COMMENT_RE.test(line)) s *= COMMENT_PENALTY
|
|
252
|
+
out[i] = s
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
if (n < 3) return out
|
|
256
|
+
const cohered = new Float64Array(n)
|
|
257
|
+
for (let i = 0; i < n; i++) {
|
|
258
|
+
const left = i > 0 ? out[i - 1] : 0
|
|
259
|
+
const right = i < n - 1 ? out[i + 1] : 0
|
|
260
|
+
cohered[i] = out[i] + COHERENCE * Math.max(left, right)
|
|
261
|
+
}
|
|
262
|
+
return cohered
|
|
263
|
+
}
|
|
264
|
+
|
|
105
265
|
// --- Line trimmer ---
|
|
106
266
|
// Keep top-K highest-scoring lines until `targetTokens` is consumed. First
|
|
107
267
|
// and last line are always preserved. Returns null when text is already
|
|
@@ -109,7 +269,16 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
|
109
269
|
//
|
|
110
270
|
// Output shape:
|
|
111
271
|
// { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
|
|
112
|
-
|
|
272
|
+
// `tokenizer` and `rerank` exist for bench/retrieval-eval.js, which replays the
|
|
273
|
+
// same fixtures through the pre-subtoken tokenizer to produce a real
|
|
274
|
+
// before/after. Production callers should leave both at their defaults.
|
|
275
|
+
export function trimLinesByRelevance(text, query, {
|
|
276
|
+
targetTokens = 4096,
|
|
277
|
+
minLines = 10,
|
|
278
|
+
path = null,
|
|
279
|
+
tokenizer = tokenize,
|
|
280
|
+
rerank = true,
|
|
281
|
+
} = {}) {
|
|
113
282
|
if (typeof text !== 'string' || text.length === 0) return null
|
|
114
283
|
|
|
115
284
|
const originalTokens = countTokens(text)
|
|
@@ -119,15 +288,44 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
119
288
|
const N = lines.length
|
|
120
289
|
if (N < minLines) return null
|
|
121
290
|
|
|
122
|
-
// Per-line token costs
|
|
123
|
-
//
|
|
291
|
+
// Per-line token costs. `countTokens` carries a fixed ~21ms cost per call
|
|
292
|
+
// regardless of input length, so counting each line separately turns a
|
|
293
|
+
// 1000-line tool_result into a ~20-second stall on the proxy hot path — for
|
|
294
|
+
// a budget that is approximate by nature. Instead calibrate a chars-per-token
|
|
295
|
+
// ratio from the whole-body count already taken above, and estimate per line.
|
|
296
|
+
// Two tokenizer calls total instead of N + 2. The reported
|
|
297
|
+
// original/trimmed token counts stay exact.
|
|
298
|
+
// Clamped (C4). A single global ratio is gameable: token density spans ~200x
|
|
299
|
+
// across content (400 spaces = 133 chars/token; emoji = 0.67), so a body
|
|
300
|
+
// whose bulk is low-density padding and whose payload is dense text made
|
|
301
|
+
// every payload line look ~20x cheaper than it is. Measured: a 762KB body
|
|
302
|
+
// reported budgetUsed 4050 against targetTokens 4096 while emitting 31,869
|
|
303
|
+
// real tokens — 7.8x over budget, with the only guard being "smaller than
|
|
304
|
+
// the (attacker-sized) input".
|
|
305
|
+
//
|
|
306
|
+
// Real chars-per-token tops out near 4.6 on ordinary prose. Capping the
|
|
307
|
+
// assumed ratio at 4 means a line can never be priced below len/4, so dense
|
|
308
|
+
// content cannot be smuggled in under a padded average. Over-estimating is
|
|
309
|
+
// safe here — it only makes the trim more conservative.
|
|
310
|
+
const MAX_CHARS_PER_TOKEN = 4
|
|
311
|
+
const rawRatio = originalTokens > 0 ? text.length / originalTokens : MAX_CHARS_PER_TOKEN
|
|
312
|
+
const charsPerToken = Math.min(rawRatio, MAX_CHARS_PER_TOKEN)
|
|
124
313
|
const lineTokens = new Array(N)
|
|
125
314
|
for (let i = 0; i < N; i++) {
|
|
126
|
-
lineTokens[i] =
|
|
315
|
+
lineTokens[i] = Math.max(1, Math.ceil((lines[i].length + 1) / charsPerToken))
|
|
127
316
|
}
|
|
128
317
|
const newlineCost = Math.max(0, N - 1)
|
|
129
318
|
|
|
130
|
-
const
|
|
319
|
+
const raw = bm25Scores(lines, query, { tokenizer })
|
|
320
|
+
const scores = rerank ? rerankScores(raw, lines, tokenizer(query)) : raw
|
|
321
|
+
|
|
322
|
+
// Block structure, when the body parses as code. Keeping a line without its
|
|
323
|
+
// enclosing signature and closing brace emits syntactically broken output;
|
|
324
|
+
// `structure` lets selection pull those ancestors in as part of the line's
|
|
325
|
+
// cost. Null (prose, logs, JSON, unbalanced source) falls back to plain
|
|
326
|
+
// per-line selection, exactly as before.
|
|
327
|
+
const blocks = detectBlocks(lines, { path })
|
|
328
|
+
const structure = blocks ? buildStructure(lines, blocks) : null
|
|
131
329
|
|
|
132
330
|
// Always-keep set: first + last as anchors.
|
|
133
331
|
const keep = new Uint8Array(N)
|
|
@@ -154,10 +352,99 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
154
352
|
}
|
|
155
353
|
candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
|
|
156
354
|
|
|
355
|
+
// Ancestors a line drags in with it: every enclosing block header not yet
|
|
356
|
+
// kept, plus that block's closing line. Charged as part of the candidate's
|
|
357
|
+
// cost so the budget stays honest instead of overflowing at assembly time.
|
|
358
|
+
// `inPending` is a reusable membership mark, cleared via the `pending` list
|
|
359
|
+
// at the top of each call (C6). The old version used `pending.includes(p)`,
|
|
360
|
+
// O(depth) per probe, and never short-circuited on an already-kept ancestor
|
|
361
|
+
// chain — so every REJECTED candidate refilled `pending` to full nesting
|
|
362
|
+
// depth, giving O(candidates x depth^2). Depth is attacker-controlled and
|
|
363
|
+
// costs no extra body bytes: at 20k lines, depth 200 measured 251ms total
|
|
364
|
+
// while depth 800 measured 4512ms, of which 4443ms was this function.
|
|
365
|
+
const pending = []
|
|
366
|
+
const inPending = new Uint8Array(N)
|
|
367
|
+
function closureFor(i) {
|
|
368
|
+
for (const p of pending) inPending[p] = 0
|
|
369
|
+
pending.length = 0
|
|
370
|
+
if (!structure) return 0
|
|
371
|
+
let cost = 0
|
|
372
|
+
const mark = (line) => {
|
|
373
|
+
if (line < 0 || line >= N || keep[line] || inPending[line]) return false
|
|
374
|
+
inPending[line] = 1
|
|
375
|
+
pending.push(line)
|
|
376
|
+
cost += lineTokens[line]
|
|
377
|
+
return true
|
|
378
|
+
}
|
|
379
|
+
// A line that OPENS a block must drag in its own closer, or a kept
|
|
380
|
+
// signature is left hanging with no `}`. Ancestors alone do not cover
|
|
381
|
+
// this: the header is not inside its own body.
|
|
382
|
+
//
|
|
383
|
+
// closer/headerOf are LISTS — one line can open or close several blocks
|
|
384
|
+
// (`function f() { if (x) {`). Taking only the first would strand the
|
|
385
|
+
// others, which is the defect these lists exist to fix.
|
|
386
|
+
const ownClosers = structure.closer[i]
|
|
387
|
+
if (ownClosers) for (const c of ownClosers) if (c !== i) mark(c)
|
|
388
|
+
|
|
389
|
+
// Conversely, a closing line's own headers are not ancestors either — the
|
|
390
|
+
// body sits between them — so seed the walk with them when the candidate
|
|
391
|
+
// IS a closer. Continue from the outermost of them; the inner ones are its
|
|
392
|
+
// descendants and get pulled in via their own closer lists.
|
|
393
|
+
const ownHeaders = structure.headerOf[i]
|
|
394
|
+
let p = structure.parent[i]
|
|
395
|
+
if (ownHeaders && ownHeaders.length > 0) {
|
|
396
|
+
let outermost = ownHeaders[0]
|
|
397
|
+
for (const h of ownHeaders) {
|
|
398
|
+
mark(h)
|
|
399
|
+
if (h < outermost) outermost = h
|
|
400
|
+
}
|
|
401
|
+
p = structure.parent[outermost]
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
while (p >= 0) {
|
|
405
|
+
// Short-circuit: if this ancestor is already kept, everything above it
|
|
406
|
+
// was kept with it, so the rest of the chain is redundant. This is what
|
|
407
|
+
// turns the rejected-candidate case from O(depth) into O(1).
|
|
408
|
+
if (keep[p]) break
|
|
409
|
+
if (!mark(p)) break // already pending — the rest of the chain is too
|
|
410
|
+
const closers = structure.closer[p]
|
|
411
|
+
// closer[] is a LIST since the C2 fix. Reading it as a scalar silently
|
|
412
|
+
// worked for single-closer blocks (`[3]` coerces to "3") and silently
|
|
413
|
+
// did nothing for multi-closer ones (`[2,3]` -> NaN).
|
|
414
|
+
if (closers) for (const c of closers) if (c !== i) mark(c)
|
|
415
|
+
p = structure.parent[p]
|
|
416
|
+
}
|
|
417
|
+
return cost
|
|
418
|
+
}
|
|
419
|
+
|
|
157
420
|
for (const c of candidates) {
|
|
158
|
-
if (
|
|
421
|
+
if (keep[c.i]) continue // already pulled in as an ancestor
|
|
422
|
+
const closureCost = closureFor(c.i)
|
|
423
|
+
if (budgetUsed + c.len + closureCost > targetTokens) continue
|
|
424
|
+
for (const a of pending) keep[a] = 1
|
|
159
425
|
keep[c.i] = 1
|
|
160
|
-
budgetUsed += c.len
|
|
426
|
+
budgetUsed += c.len + closureCost
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Blank lines cost zero tokens, so greedy selection always keeps them. Left
|
|
430
|
+
// alone they punch holes through dropped runs, and every hole costs another
|
|
431
|
+
// ~12-token marker — enough to overshoot the budget several times over on a
|
|
432
|
+
// fragmented body. A blank RUN stranded between two dropped lines carries
|
|
433
|
+
// nothing, so fold the whole run in.
|
|
434
|
+
//
|
|
435
|
+
// Must operate on runs, not single lines. An earlier version tested only
|
|
436
|
+
// `keep[i-1]` and `keep[i+1]`, which never fires on `\n\n` — the most common
|
|
437
|
+
// blank shape in source — because each blank sees the other as kept. Measured
|
|
438
|
+
// 72 of 148 surviving lines blank before this fix. Snapshotting `keep` does
|
|
439
|
+
// not help either; the run has to be considered as a unit.
|
|
440
|
+
for (let i = 1; i < N - 1; i++) {
|
|
441
|
+
if (!keep[i] || lines[i].trim() !== '') continue
|
|
442
|
+
let j = i
|
|
443
|
+
while (j < N - 1 && keep[j] && lines[j].trim() === '') j += 1
|
|
444
|
+
if (!keep[i - 1] && j < N && !keep[j]) {
|
|
445
|
+
for (let k = i; k < j; k += 1) keep[k] = 0
|
|
446
|
+
}
|
|
447
|
+
i = j - 1
|
|
161
448
|
}
|
|
162
449
|
|
|
163
450
|
// If even the anchors + marker exceed budget, still proceed — we need to
|
|
@@ -180,7 +467,10 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
180
467
|
let chars = 0
|
|
181
468
|
while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
|
|
182
469
|
const runLen = j - i
|
|
183
|
-
|
|
470
|
+
// Indent the marker to the dropped body's own depth, so a collapsed
|
|
471
|
+
// function body reads as part of the block it replaced.
|
|
472
|
+
const pad = lines[i].slice(0, indentWidth(lines[i]))
|
|
473
|
+
out.push(`${pad}[... ${runLen} lines omitted, ${chars} chars ...]`)
|
|
184
474
|
i = j
|
|
185
475
|
}
|
|
186
476
|
|
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Block structure detection for code bodies (Phase 7).
|
|
2
|
+
//
|
|
3
|
+
// `bm25-trim` drops individual lines. On code that produces syntactically
|
|
4
|
+
// broken output: a kept line from inside a function body, with its signature
|
|
5
|
+
// and closing brace gone, tells the model almost nothing. This module finds
|
|
6
|
+
// block ranges so the trimmer can enforce a simple invariant — keep a line,
|
|
7
|
+
// keep the headers and closers of every block enclosing it.
|
|
8
|
+
//
|
|
9
|
+
// Deliberately not tree-sitter. That is a native dependency, and Tamp ships a
|
|
10
|
+
// pure-JS package with a tight `files:` list. Two cheap detectors cover the
|
|
11
|
+
// languages coding agents actually read, and both refuse to guess: when the
|
|
12
|
+
// structure does not check out, `detectBlocks` returns null and the caller
|
|
13
|
+
// falls back to plain line trimming.
|
|
14
|
+
|
|
15
|
+
// --- Language selection ---
|
|
16
|
+
|
|
17
|
+
const BRACE_EXT = new Set([
|
|
18
|
+
'js', 'jsx', 'mjs', 'cjs', 'ts', 'tsx', 'mts', 'cts',
|
|
19
|
+
'java', 'c', 'h', 'cc', 'cpp', 'hpp', 'cs', 'go', 'rs',
|
|
20
|
+
'swift', 'kt', 'kts', 'scala', 'php', 'dart', 'zig',
|
|
21
|
+
])
|
|
22
|
+
const INDENT_EXT = new Set(['py', 'pyi', 'yaml', 'yml'])
|
|
23
|
+
|
|
24
|
+
export function languageForPath(path) {
|
|
25
|
+
if (typeof path !== 'string' || path.length === 0) return null
|
|
26
|
+
const dot = path.lastIndexOf('.')
|
|
27
|
+
if (dot < 0) return null
|
|
28
|
+
const ext = path.slice(dot + 1).toLowerCase()
|
|
29
|
+
if (BRACE_EXT.has(ext)) return 'brace'
|
|
30
|
+
if (INDENT_EXT.has(ext)) return 'indent'
|
|
31
|
+
return null
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Path-based detection only works on Anthropic requests (see
|
|
35
|
+
// lib/path-extract.js). Everywhere else we sniff the body itself.
|
|
36
|
+
const PY_BLOCK_RE = /^\s*(?:async\s+)?(?:def|class|if|for|while|with|try|elif|else|except|finally)\b.*:\s*$/
|
|
37
|
+
|
|
38
|
+
export function sniffLanguage(lines) {
|
|
39
|
+
if (!Array.isArray(lines) || lines.length < 10) return null
|
|
40
|
+
let braceOpens = 0
|
|
41
|
+
let pyBlocks = 0
|
|
42
|
+
const sample = Math.min(lines.length, 400)
|
|
43
|
+
for (let i = 0; i < sample; i++) {
|
|
44
|
+
const line = lines[i]
|
|
45
|
+
if (typeof line !== 'string') continue
|
|
46
|
+
if (/\{\s*$/.test(line)) braceOpens += 1
|
|
47
|
+
if (PY_BLOCK_RE.test(line)) pyBlocks += 1
|
|
48
|
+
}
|
|
49
|
+
// A low bar is safe: the brace detector independently returns null unless
|
|
50
|
+
// the whole body balances, so a wrong guess on a log file self-corrects.
|
|
51
|
+
if (braceOpens >= 3 && braceOpens > pyBlocks) return 'brace'
|
|
52
|
+
if (pyBlocks >= 3) return 'indent'
|
|
53
|
+
return null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// --- Shared helpers ---
|
|
57
|
+
|
|
58
|
+
export function indentWidth(line) {
|
|
59
|
+
if (typeof line !== 'string') return 0
|
|
60
|
+
let n = 0
|
|
61
|
+
while (n < line.length && (line[n] === ' ' || line[n] === '\t')) n += 1
|
|
62
|
+
return n
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// --- Brace detector ---
|
|
66
|
+
//
|
|
67
|
+
// One left-to-right character scan carrying string AND comment state (F2).
|
|
68
|
+
//
|
|
69
|
+
// The previous version detected `/*` on the RAW line before stripping strings,
|
|
70
|
+
// so any `/*` substring — a URL path, a glob, prose about comment syntax —
|
|
71
|
+
// flipped the scanner into block-comment state and swallowed every following
|
|
72
|
+
// line's braces until an unrelated `*/`. It fired on this repo's own
|
|
73
|
+
// compress.js:258 and config.js:220, silently disabling block-aware trimming
|
|
74
|
+
// for 2 of 6 source files. Injecting one line into a 214KB body flipped 465
|
|
75
|
+
// detected blocks to 0 and produced 72 unclosed braces in the output.
|
|
76
|
+
//
|
|
77
|
+
// It also missed a complete `/* } */` on one line, because it declined to
|
|
78
|
+
// enter comment state when `*/` followed on the same line, and the regex-based
|
|
79
|
+
// stripper never handled `/* */` at all.
|
|
80
|
+
//
|
|
81
|
+
// Scanning once, left to right, makes both cases fall out for free: a `/*`
|
|
82
|
+
// inside a string is never seen as a comment opener because we are in string
|
|
83
|
+
// state when we reach it.
|
|
84
|
+
//
|
|
85
|
+
// Returns null when braces do not balance, or when comment state is still open
|
|
86
|
+
// at end of input — the honest signal that the scan mis-parsed something
|
|
87
|
+
// (regex literals and JSX are still not modelled).
|
|
88
|
+
|
|
89
|
+
function detectBraceBlocks(lines) {
|
|
90
|
+
const blocks = []
|
|
91
|
+
const open = []
|
|
92
|
+
let inBlockComment = false
|
|
93
|
+
|
|
94
|
+
for (let i = 0; i < lines.length; i++) {
|
|
95
|
+
const line = lines[i]
|
|
96
|
+
if (typeof line !== 'string') return null
|
|
97
|
+
|
|
98
|
+
let quote = null // "'", '"' or '`' while inside a string literal
|
|
99
|
+
let escaped = false
|
|
100
|
+
|
|
101
|
+
for (let c = 0; c < line.length; c++) {
|
|
102
|
+
const ch = line[c]
|
|
103
|
+
|
|
104
|
+
if (inBlockComment) {
|
|
105
|
+
if (ch === '*' && line[c + 1] === '/') { inBlockComment = false; c += 1 }
|
|
106
|
+
continue
|
|
107
|
+
}
|
|
108
|
+
if (quote !== null) {
|
|
109
|
+
if (escaped) { escaped = false; continue }
|
|
110
|
+
if (ch === '\\') { escaped = true; continue }
|
|
111
|
+
if (ch === quote) quote = null
|
|
112
|
+
continue
|
|
113
|
+
}
|
|
114
|
+
if (ch === '"' || ch === "'" || ch === '`') { quote = ch; continue }
|
|
115
|
+
if (ch === '/' && line[c + 1] === '*') { inBlockComment = true; c += 1; continue }
|
|
116
|
+
if (ch === '/' && line[c + 1] === '/') break // line comment: rest of line is noise
|
|
117
|
+
if (ch === '#') break // ditto for shell/python-style comments
|
|
118
|
+
|
|
119
|
+
if (ch === '{') {
|
|
120
|
+
// The header is the line the brace opens on. A brace that opens and
|
|
121
|
+
// closes on the same line yields a zero-length body and is dropped
|
|
122
|
+
// below, so object literals inline in an argument list cost nothing.
|
|
123
|
+
open.push(i)
|
|
124
|
+
} else if (ch === '}') {
|
|
125
|
+
const headerLine = open.pop()
|
|
126
|
+
if (headerLine === undefined) return null // unbalanced — give up
|
|
127
|
+
if (i > headerLine + 1) {
|
|
128
|
+
blocks.push({ headerLine, startLine: headerLine + 1, endLine: i - 1, closeLine: i })
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Single-quoted and double-quoted strings do not span lines in any language
|
|
133
|
+
// this detector claims; reset so one stray quote cannot swallow the file.
|
|
134
|
+
// Backtick state deliberately carries over (template literals are multiline).
|
|
135
|
+
if (quote === '"' || quote === "'") quote = null
|
|
136
|
+
}
|
|
137
|
+
if (open.length > 0 || inBlockComment) return null
|
|
138
|
+
return blocks
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// --- Indent detector ---
|
|
142
|
+
//
|
|
143
|
+
// A block opens on a line ending in `:` and runs while indentation exceeds the
|
|
144
|
+
// header's. Blank lines belong to the block; they do not close it.
|
|
145
|
+
//
|
|
146
|
+
// Built in ONE pass with an indent stack (C3). The previous version scanned
|
|
147
|
+
// forward from every header to find its end, and because blank lines were
|
|
148
|
+
// skipped with `continue` rather than terminating the scan, every enclosing
|
|
149
|
+
// header re-walked the same blank field: O(headers x blank_lines), unbounded by
|
|
150
|
+
// body bytes since a blank line costs one byte but is visited once per open
|
|
151
|
+
// block. Measured 2000 headers over 200k blank lines (2.2MB) at 10783ms.
|
|
152
|
+
|
|
153
|
+
function detectIndentBlocks(lines) {
|
|
154
|
+
const blocks = []
|
|
155
|
+
const stack = [] // { headerLine, indent }, outermost first
|
|
156
|
+
let lastContentLine = -1
|
|
157
|
+
|
|
158
|
+
for (let i = 0; i < lines.length; i++) {
|
|
159
|
+
const line = lines[i]
|
|
160
|
+
if (typeof line !== 'string') return null
|
|
161
|
+
if (isBlank(line)) continue // blanks belong to whatever block is open
|
|
162
|
+
|
|
163
|
+
const indent = indentWidth(line)
|
|
164
|
+
while (stack.length > 0 && indent <= stack[stack.length - 1].indent) {
|
|
165
|
+
const done = stack.pop()
|
|
166
|
+
if (lastContentLine > done.headerLine) {
|
|
167
|
+
blocks.push({
|
|
168
|
+
headerLine: done.headerLine,
|
|
169
|
+
startLine: done.headerLine + 1,
|
|
170
|
+
endLine: lastContentLine,
|
|
171
|
+
closeLine: null,
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (PY_BLOCK_RE.test(line)) stack.push({ headerLine: i, indent })
|
|
176
|
+
lastContentLine = i
|
|
177
|
+
}
|
|
178
|
+
while (stack.length > 0) {
|
|
179
|
+
const done = stack.pop()
|
|
180
|
+
if (lastContentLine > done.headerLine) {
|
|
181
|
+
blocks.push({
|
|
182
|
+
headerLine: done.headerLine,
|
|
183
|
+
startLine: done.headerLine + 1,
|
|
184
|
+
endLine: lastContentLine,
|
|
185
|
+
closeLine: null,
|
|
186
|
+
})
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return blocks
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Non-allocating blank check. `line.trim() === ''` allocated a string per line,
|
|
193
|
+
// which the old per-header rescan paid once per open block per blank line.
|
|
194
|
+
function isBlank(line) {
|
|
195
|
+
for (let i = 0; i < line.length; i++) {
|
|
196
|
+
const ch = line[i]
|
|
197
|
+
if (ch !== ' ' && ch !== '\t' && ch !== '\r') return false
|
|
198
|
+
}
|
|
199
|
+
return true
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// --- Public entry ---
|
|
203
|
+
//
|
|
204
|
+
// `detectBlocks(lines, { path })` -> array of blocks, or null when no detector
|
|
205
|
+
// applies or the structure did not check out. Blocks are unordered with
|
|
206
|
+
// respect to nesting; use `buildStructure` to get the lookup maps.
|
|
207
|
+
|
|
208
|
+
export function detectBlocks(lines, { path = null } = {}) {
|
|
209
|
+
if (!Array.isArray(lines) || lines.length === 0) return null
|
|
210
|
+
const language = languageForPath(path) || sniffLanguage(lines)
|
|
211
|
+
if (language === 'brace') return detectBraceBlocks(lines)
|
|
212
|
+
if (language === 'indent') return detectIndentBlocks(lines)
|
|
213
|
+
return null
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Flatten blocks into per-line lookups the trimmer needs:
|
|
217
|
+
// parent[i] innermost block header enclosing line i, or -1
|
|
218
|
+
// closer[h] ARRAY of closing lines for blocks whose header is line h
|
|
219
|
+
// headerOf[c] ARRAY of header lines for blocks closing on line c
|
|
220
|
+
//
|
|
221
|
+
// `headerOf` is the inverse of `closer`: a bare `}` sits outside its own
|
|
222
|
+
// block's body, so without it a closing line looks parentless and can be kept
|
|
223
|
+
// on its own, stranding a `}` with no signature.
|
|
224
|
+
//
|
|
225
|
+
// Both are LISTS, not scalars (C2). One physical line can open or close more
|
|
226
|
+
// than one block — `function f() { if (x) {` opens two, `} }` closes two.
|
|
227
|
+
// Scalar maps keyed by line number silently kept only the LAST write, so the
|
|
228
|
+
// outer closer was lost and the trimmer emitted a signature with no matching
|
|
229
|
+
// brace. Verified before the fix: closer[0] === 2 (the inner `}`) not 3.
|
|
230
|
+
//
|
|
231
|
+
// `parent` is filled by one stack sweep rather than by painting each block's
|
|
232
|
+
// span (C1). Painting costs O(sum of block spans) = O(N x nesting depth),
|
|
233
|
+
// quadratic on deeply nested input: a body of N/2 `{` then N/2 `}` measured
|
|
234
|
+
// 200ms at 80KB, 1188ms at 200KB, 4741ms at 400KB. The sweep is O(N + blocks).
|
|
235
|
+
export function buildStructure(lines, blocks) {
|
|
236
|
+
const n = lines.length
|
|
237
|
+
const parent = new Int32Array(n).fill(-1)
|
|
238
|
+
const closer = new Array(n)
|
|
239
|
+
const headerOf = new Array(n)
|
|
240
|
+
if (!Array.isArray(blocks) || blocks.length === 0) return { parent, closer, headerOf }
|
|
241
|
+
|
|
242
|
+
for (const block of blocks) {
|
|
243
|
+
const h = block.headerLine
|
|
244
|
+
const c = block.closeLine
|
|
245
|
+
if (h < 0 || h >= n || c === null || c < 0 || c >= n) continue
|
|
246
|
+
;(closer[h] || (closer[h] = [])).push(c)
|
|
247
|
+
;(headerOf[c] || (headerOf[c] = [])).push(h)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Outer-first among blocks starting on the same line, so the narrower one
|
|
251
|
+
// lands on top of the stack and wins `parent` for the lines both cover.
|
|
252
|
+
const byStart = blocks
|
|
253
|
+
.slice()
|
|
254
|
+
.sort((a, b) => (a.startLine - b.startLine) || (b.endLine - a.endLine))
|
|
255
|
+
const stack = []
|
|
256
|
+
let bi = 0
|
|
257
|
+
for (let i = 0; i < n; i++) {
|
|
258
|
+
while (stack.length > 0 && stack[stack.length - 1].endLine < i) stack.pop()
|
|
259
|
+
while (bi < byStart.length && byStart[bi].startLine <= i) {
|
|
260
|
+
const b = byStart[bi]
|
|
261
|
+
bi += 1
|
|
262
|
+
if (b.endLine >= i) stack.push(b)
|
|
263
|
+
}
|
|
264
|
+
// A block never covers its own header (startLine is headerLine + 1), so
|
|
265
|
+
// parent[headerLine] keeps pointing at the enclosing block.
|
|
266
|
+
parent[i] = stack.length > 0 ? stack[stack.length - 1].headerLine : -1
|
|
267
|
+
}
|
|
268
|
+
return { parent, closer, headerOf }
|
|
269
|
+
}
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"sidecar/server.py",
|
|
15
15
|
"sidecar/requirements.txt"
|
|
16
16
|
],
|
|
17
|
-
"version": "0.8.
|
|
17
|
+
"version": "0.8.16",
|
|
18
18
|
"description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
|
|
19
19
|
"type": "module",
|
|
20
20
|
"main": "index.js",
|
|
@@ -23,9 +23,11 @@
|
|
|
23
23
|
},
|
|
24
24
|
"scripts": {
|
|
25
25
|
"start": "node bin/tamp.js",
|
|
26
|
+
"prepublishOnly": "node scripts/publish-guard.js",
|
|
26
27
|
"test": "node --test test/*.test.js",
|
|
27
28
|
"test:sidecar": "node test/capture-golden.js --verify",
|
|
28
29
|
"bench:semantic": "node bench/semantic-eval.js",
|
|
30
|
+
"bench:retrieval": "node bench/retrieval-eval.js",
|
|
29
31
|
"test:quality": "node bench/quality-eval.js",
|
|
30
32
|
"compat:probe": "node test/compat/probe-proxy.js",
|
|
31
33
|
"compat:run": "node test/compat/run.mjs"
|