@sliday/tamp 0.8.15 → 0.8.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/compress.js CHANGED
@@ -167,14 +167,20 @@ function bm25TrimTargets(targets, body, provider) {
167
167
  if (!userText) return 0
168
168
  if (isDangerousTask(userText)) return 0
169
169
 
170
+ // File paths let the trimmer pick a block detector by extension so dropped
171
+ // lines respect code structure. Anthropic-only (see lib/path-extract.js);
172
+ // elsewhere `path` is null and the trimmer sniffs the body instead.
173
+ const paths = extractTargetPaths(body, targets)
174
+
170
175
  let trimmed = 0
171
- for (const target of targets) {
176
+ for (let i = 0; i < targets.length; i++) {
177
+ const target = targets[i]
172
178
  if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.compressed) continue
173
179
  if (typeof target.text !== 'string') continue
174
180
  const byteLen = Buffer.byteLength(target.text, 'utf8')
175
181
  if (byteLen <= 64 * 1024) continue
176
182
 
177
- const result = trimLinesByRelevance(target.text, userText)
183
+ const result = trimLinesByRelevance(target.text, userText, { path: paths[i] })
178
184
  if (!result) continue
179
185
  if (result.text.length >= target.text.length) continue
180
186
  target.compressed = result.text
package/lib/bm25.js CHANGED
@@ -10,28 +10,123 @@
10
10
  // This primitive is reusable: any future stage that wants "keep the most
11
11
  // relevant N lines" can import tokenize/bm25Scores directly.
12
12
 
13
- import { countTokens } from '@anthropic-ai/tokenizer'
13
+ import { countTokens as rawCountTokens } from '@anthropic-ai/tokenizer'
14
+ import { detectBlocks, buildStructure, indentWidth } from './code-chunks.js'
15
+
16
+ // --- Safe token counting (C5) ---
17
+ //
18
+ // `countTokens` is NOT the fixed-cost call an earlier comment in this file
19
+ // claimed. It has a ~22ms floor on small inputs, but it is superlinear on
20
+ // large low-entropy input and it HARD CRASHES on some of it. Measured on
21
+ // whitespace-heavy bodies (400-char space runs):
22
+ //
23
+ // 80KB -> 8.7s 200KB -> 22.4s 400KB -> 88.9s
24
+ // 1MB -> RuntimeError: unreachable (tiktoken WASM trap)
25
+ //
26
+ // `bm25-trim` only runs on bodies over 64KB, so every attacker-controlled
27
+ // tool_result on this path paid it. A WASM trap is not catchable as a normal
28
+ // error in every runtime, so the primary defence is the size cap; the
29
+ // try/catch is only the backstop.
30
+ //
31
+ // The cap is 4KB, not 64KB, because the cost is driven by CONTENT, not size.
32
+ // Measured:
33
+ //
34
+ // spaces: 1KB 22ms 4KB 31ms 8KB 58ms 16KB 167ms 32KB 609ms
35
+ // prose: 8KB 22ms 32KB 24ms 64KB 27ms
36
+ //
37
+ // Prose is flat at the ~22ms floor whatever its length; pathological runs go
38
+ // quadratic. 4KB bounds the worst case to ~31ms. A 64KB cap still measured
39
+ // 44s on a megabyte of spaces, which is why the first attempt at this fix
40
+ // did not work.
41
+ const COUNT_CAP = 4 * 1024
42
+ const SAMPLES = 3
43
+
44
+ export function safeCountTokens(text) {
45
+ if (typeof text !== 'string' || text.length === 0) return 0
46
+ try {
47
+ if (text.length <= COUNT_CAP) return rawCountTokens(text)
48
+ // Three windows spread across the body rather than one prefix — same cost,
49
+ // better representativeness on a body whose head is not typical of its tail
50
+ // (a log with a JSON blob at the end, a file with a licence header).
51
+ const each = Math.floor(COUNT_CAP / SAMPLES)
52
+ const stride = Math.floor((text.length - each) / (SAMPLES - 1))
53
+ let sampled = 0
54
+ let counted = 0
55
+ for (let s = 0; s < SAMPLES; s++) {
56
+ const start = Math.min(s * stride, text.length - each)
57
+ const window = text.slice(start, start + each)
58
+ counted += rawCountTokens(window)
59
+ sampled += window.length
60
+ }
61
+ if (sampled === 0) return Math.max(1, Math.ceil(text.length / 4))
62
+ return Math.max(1, Math.round(counted * (text.length / sampled)))
63
+ } catch {
64
+ // Tokenizer refused this input. Fall back to a coarse estimate rather than
65
+ // aborting compression for the whole request.
66
+ return Math.max(1, Math.ceil(text.length / 4))
67
+ }
68
+ }
69
+
70
+ const countTokens = safeCountTokens
14
71
 
15
72
  // --- Tokenizer ---
16
73
  // Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
17
74
  // `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
18
75
  // separators to underscore before splitting on non-word characters. Single
19
76
  // character tokens are dropped (too noisy for BM25 on short queries).
77
+ //
78
+ // Emitting ONLY the collapsed form silently destroys recall on the queries
79
+ // agents actually ask. `parse config` tokenizes to ['parse','config'], while
80
+ // `function parseConfig(` yields ['parseconfig'] and `def parse_config(`
81
+ // yields ['parse_config'] — no shared term, so BM25 scores the definition
82
+ // line the user asked about at exactly 0 and `bm25-trim` drops it. So we also
83
+ // emit the identifier's parts, splitting on camelCase, PascalCase, acronym
84
+ // runs, and underscores. Case must survive until the split, hence the
85
+ // lowercase happens per-token at the end rather than up front.
20
86
  const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
21
87
  const SPLIT_RE = /[^A-Za-z0-9_]+/
88
+ // `HTTPServer` -> `HTTP Server`. Runs before CAMEL_LOWER_UPPER so that
89
+ // `parseHTTPResponse` splits three ways rather than swallowing the acronym.
90
+ const CAMEL_ACRONYM = /([A-Z]+)([A-Z][a-z])/g
91
+ const CAMEL_LOWER_UPPER = /([a-z0-9])([A-Z])/g
92
+ // No cap on parts per identifier. An earlier version capped at 4 and truncated
93
+ // from the TAIL, which reintroduced the exact bug this file exists to fix:
94
+ // `get_or_create_default_workspace_settings` emitted get/or/create/default and
95
+ // dropped `workspace` and `settings`, so a query for "workspace settings"
96
+ // scored the definition line at 0 and deleted it. The cap was motivated by an
97
+ // IDF-dilution worry that was never measured. If dilution is real,
98
+ // bench/retrieval-eval.js is where it should show up.
99
+
100
+ // Split one raw token into its component words. Returns [] when the token has
101
+ // no internal structure, so an already-atomic `config` is not emitted twice
102
+ // (which would inflate its term frequency and skew BM25).
103
+ function subTokens(rawTok) {
104
+ const spaced = rawTok
105
+ .replace(CAMEL_ACRONYM, '$1 $2')
106
+ .replace(CAMEL_LOWER_UPPER, '$1 $2')
107
+ .replace(/_+/g, ' ')
108
+ const parts = spaced.split(' ')
109
+ if (parts.length < 2) return []
110
+ const out = []
111
+ for (const part of parts) {
112
+ if (part.length < 2) continue
113
+ out.push(part.toLowerCase())
114
+ }
115
+ return out
116
+ }
22
117
 
23
118
  export function tokenize(text) {
24
119
  if (typeof text !== 'string' || text.length === 0) return []
25
- let normalized = text.toLowerCase()
26
120
  // Collapse intra-word separators. Run twice so chains like `a.b.c` fully
27
121
  // collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
28
- normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
122
+ let normalized = text.replace(INTRA_WORD_RE, '$1_$2')
29
123
  normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
30
124
  const raw = normalized.split(SPLIT_RE)
31
125
  const out = []
32
126
  for (const tok of raw) {
33
- if (tok.length < 2) continue
34
- out.push(tok)
127
+ if (tok.length === 0) continue
128
+ if (tok.length >= 2) out.push(tok.toLowerCase())
129
+ for (const part of subTokens(tok)) out.push(part)
35
130
  }
36
131
  return out
37
132
  }
@@ -39,60 +134,76 @@ export function tokenize(text) {
39
134
  // --- BM25 ---
40
135
  // Standard Okapi BM25. Each document is a line; query is the user's last
41
136
  // message. Returns Float64Array of scores aligned with the documents array.
42
- export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
137
+ // `tokenizer` is injectable so bench/retrieval-eval.js can score the same
138
+ // corpus with the pre-subtoken tokenizer and report a real before/after.
139
+ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75, tokenizer = tokenize } = {}) {
43
140
  const N = documents.length
44
141
  const scores = new Float64Array(N)
45
142
  if (N === 0) return scores
46
143
 
47
- const qTokens = tokenize(query)
144
+ const qTokens = tokenizer(query)
48
145
  if (qTokens.length === 0) return scores
49
146
 
50
- // Tokenize docs + build doc stats.
51
- const docTokens = new Array(N)
147
+ // ONE pass over the documents (C7 + C8).
148
+ //
149
+ // C7: the old version kept `docTokens` — the full tokenization of every line
150
+ // — alive for the whole call, though only query terms are ever read from it.
151
+ // A 4.6MB body (60k lines) retained 47.4MB, roughly 10x amplification, on top
152
+ // of the original string, the split lines array and the parsed request body.
153
+ // The proxy serves concurrent requests with no per-request size cap on this
154
+ // stage, so that was an OOM path, not just memory pressure.
155
+ //
156
+ // C8: document frequency was computed as `docTokens[i].includes(term)` for
157
+ // every unique query term against every line — O(uniqQ x N x tokens/line) of
158
+ // linear scans. Both dimensions are hostile: N is the attacker-controlled
159
+ // line count, and the query is the user's last message, which routinely
160
+ // contains a pasted stack trace. Measured at N=20000: 70ms for a 10-word
161
+ // query, 163ms for 100 words, 676ms for 500.
162
+ //
163
+ // Now: tokenize each line once, keep only the per-line term frequencies for
164
+ // terms that appear in the query (empty for the vast majority of lines), and
165
+ // accumulate df in the same pass. Peak retention scales with the QUERY size,
166
+ // not the body size.
167
+ const qSet = new Set(qTokens)
52
168
  const docLens = new Float64Array(N)
169
+ const docTf = new Array(N) // Map|null, only query terms, null when none present
170
+ const df = new Map()
53
171
  let totalLen = 0
172
+
54
173
  for (let i = 0; i < N; i++) {
55
- const toks = tokenize(documents[i])
56
- docTokens[i] = toks
174
+ const toks = tokenizer(documents[i])
57
175
  docLens[i] = toks.length
58
176
  totalLen += toks.length
177
+ let tf = null
178
+ for (const t of toks) {
179
+ if (!qSet.has(t)) continue
180
+ if (tf === null) tf = new Map()
181
+ tf.set(t, (tf.get(t) || 0) + 1)
182
+ }
183
+ docTf[i] = tf
184
+ if (tf !== null) for (const t of tf.keys()) df.set(t, (df.get(t) || 0) + 1)
185
+ // `toks` goes out of scope here and is collectable — nothing retains it.
59
186
  }
187
+
60
188
  const avgdl = N > 0 ? totalLen / N : 0
61
189
  if (avgdl === 0) return scores
62
190
 
63
- // Document-frequency per unique query term.
64
- const uniqQ = Array.from(new Set(qTokens))
65
- const df = new Map()
66
- for (const term of uniqQ) {
67
- let count = 0
68
- for (let i = 0; i < N; i++) {
69
- if (docTokens[i].includes(term)) count += 1
70
- }
71
- df.set(term, count)
72
- }
73
-
74
- // Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
191
+ // Okapi BM25 IDF with +1 smoothing.
75
192
  const idf = new Map()
76
- for (const term of uniqQ) {
193
+ for (const term of qSet) {
77
194
  const n = df.get(term) || 0
78
- const v = Math.log(1 + (N - n + 0.5) / (n + 0.5))
79
- idf.set(term, v)
195
+ idf.set(term, Math.log(1 + (N - n + 0.5) / (n + 0.5)))
80
196
  }
81
197
 
82
198
  for (let i = 0; i < N; i++) {
199
+ const tf = docTf[i]
200
+ if (tf === null) continue // no query term on this line — scores 0, skip
83
201
  const dl = docLens[i]
84
202
  if (dl === 0) continue
85
- // Term-frequency table for this doc (only for query terms we care about).
86
- const tf = new Map()
87
- const toks = docTokens[i]
88
- for (const t of toks) {
89
- if (!idf.has(t)) continue
90
- tf.set(t, (tf.get(t) || 0) + 1)
91
- }
92
203
  let s = 0
93
- for (const term of uniqQ) {
94
- const f = tf.get(term) || 0
95
- if (f === 0) continue
204
+ // Iterate the terms actually PRESENT (usually 0-3) rather than every
205
+ // unique query term, so a long query costs nothing on unrelated lines.
206
+ for (const [term, f] of tf) {
96
207
  const num = f * (k1 + 1)
97
208
  const den = f + k1 * (1 - b + b * (dl / avgdl))
98
209
  s += idf.get(term) * (num / den)
@@ -102,6 +213,55 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
102
213
  return scores
103
214
  }
104
215
 
216
+ // --- Rerank ---
217
+ // Raw BM25 treats every line as an equal bag of words. Two code-aware signals
218
+ // and one structural one fix its most visible failures.
219
+ //
220
+ // Note on scope: a path-based penalty (test files, .d.ts stubs) would be a
221
+ // no-op here. `bm25-trim` ranks lines WITHIN one tool_result, so a multiplier
222
+ // constant across the whole body cannot change any relative ordering. Path is
223
+ // used for language detection only.
224
+
225
+ // A line that DEFINES a query term beats a line that merely mentions it.
226
+ const DEF_RE = /\b(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function|class|def|const|let|var|type|interface|enum|struct|impl|trait|fn|func|module|namespace)\s+([A-Za-z_$][\w$]*)/
227
+ const COMMENT_RE = /^\s*(?:\/\/|#|\*|\/\*|--|<!--|;)/
228
+ const DEF_BOOST = 1.6
229
+ const COMMENT_PENALTY = 0.6
230
+ // A kept line lends a quarter of its score to each immediate neighbour, so the
231
+ // survivors read as contiguous code rather than confetti. Applied off the
232
+ // pre-coherence scores so boosts do not cascade down the file.
233
+ const COHERENCE = 0.25
234
+
235
+ export function rerankScores(scores, lines, queryTokens) {
236
+ const n = lines.length
237
+ const out = new Float64Array(n)
238
+ const qSet = new Set(queryTokens)
239
+
240
+ for (let i = 0; i < n; i++) {
241
+ let s = scores[i]
242
+ const line = lines[i]
243
+ if (typeof line !== 'string' || s === 0) { out[i] = s; continue }
244
+
245
+ const def = DEF_RE.exec(line)
246
+ if (def) {
247
+ for (const tok of tokenize(def[1])) {
248
+ if (qSet.has(tok)) { s *= DEF_BOOST; break }
249
+ }
250
+ }
251
+ if (COMMENT_RE.test(line)) s *= COMMENT_PENALTY
252
+ out[i] = s
253
+ }
254
+
255
+ if (n < 3) return out
256
+ const cohered = new Float64Array(n)
257
+ for (let i = 0; i < n; i++) {
258
+ const left = i > 0 ? out[i - 1] : 0
259
+ const right = i < n - 1 ? out[i + 1] : 0
260
+ cohered[i] = out[i] + COHERENCE * Math.max(left, right)
261
+ }
262
+ return cohered
263
+ }
264
+
105
265
  // --- Line trimmer ---
106
266
  // Keep top-K highest-scoring lines until `targetTokens` is consumed. First
107
267
  // and last line are always preserved. Returns null when text is already
@@ -109,7 +269,16 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
109
269
  //
110
270
  // Output shape:
111
271
  // { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
112
- export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLines = 10 } = {}) {
272
+ // `tokenizer` and `rerank` exist for bench/retrieval-eval.js, which replays the
273
+ // same fixtures through the pre-subtoken tokenizer to produce a real
274
+ // before/after. Production callers should leave both at their defaults.
275
+ export function trimLinesByRelevance(text, query, {
276
+ targetTokens = 4096,
277
+ minLines = 10,
278
+ path = null,
279
+ tokenizer = tokenize,
280
+ rerank = true,
281
+ } = {}) {
113
282
  if (typeof text !== 'string' || text.length === 0) return null
114
283
 
115
284
  const originalTokens = countTokens(text)
@@ -119,15 +288,44 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
119
288
  const N = lines.length
120
289
  if (N < minLines) return null
121
290
 
122
- // Per-line token costs (includes the trailing newline for all but the last
123
- // line; we approximate by counting each line once and adding N-1 tokens).
291
+ // Per-line token costs. `countTokens` carries a fixed ~21ms cost per call
292
+ // regardless of input length, so counting each line separately turns a
293
+ // 1000-line tool_result into a ~20-second stall on the proxy hot path — for
294
+ // a budget that is approximate by nature. Instead calibrate a chars-per-token
295
+ // ratio from the whole-body count already taken above, and estimate per line.
296
+ // Two tokenizer calls total instead of N + 2. The reported
297
+ // original/trimmed token counts stay exact.
298
+ // Clamped (C4). A single global ratio is gameable: token density spans ~200x
299
+ // across content (400 spaces = 133 chars/token; emoji = 0.67), so a body
300
+ // whose bulk is low-density padding and whose payload is dense text made
301
+ // every payload line look ~20x cheaper than it is. Measured: a 762KB body
302
+ // reported budgetUsed 4050 against targetTokens 4096 while emitting 31,869
303
+ // real tokens — 7.8x over budget, with the only guard being "smaller than
304
+ // the (attacker-sized) input".
305
+ //
306
+ // Real chars-per-token tops out near 4.6 on ordinary prose. Capping the
307
+ // assumed ratio at 4 means a line can never be priced below len/4, so dense
308
+ // content cannot be smuggled in under a padded average. Over-estimating is
309
+ // safe here — it only makes the trim more conservative.
310
+ const MAX_CHARS_PER_TOKEN = 4
311
+ const rawRatio = originalTokens > 0 ? text.length / originalTokens : MAX_CHARS_PER_TOKEN
312
+ const charsPerToken = Math.min(rawRatio, MAX_CHARS_PER_TOKEN)
124
313
  const lineTokens = new Array(N)
125
314
  for (let i = 0; i < N; i++) {
126
- lineTokens[i] = countTokens(lines[i])
315
+ lineTokens[i] = Math.max(1, Math.ceil((lines[i].length + 1) / charsPerToken))
127
316
  }
128
317
  const newlineCost = Math.max(0, N - 1)
129
318
 
130
- const scores = bm25Scores(lines, query)
319
+ const raw = bm25Scores(lines, query, { tokenizer })
320
+ const scores = rerank ? rerankScores(raw, lines, tokenizer(query)) : raw
321
+
322
+ // Block structure, when the body parses as code. Keeping a line without its
323
+ // enclosing signature and closing brace emits syntactically broken output;
324
+ // `structure` lets selection pull those ancestors in as part of the line's
325
+ // cost. Null (prose, logs, JSON, unbalanced source) falls back to plain
326
+ // per-line selection, exactly as before.
327
+ const blocks = detectBlocks(lines, { path })
328
+ const structure = blocks ? buildStructure(lines, blocks) : null
131
329
 
132
330
  // Always-keep set: first + last as anchors.
133
331
  const keep = new Uint8Array(N)
@@ -154,10 +352,99 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
154
352
  }
155
353
  candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
156
354
 
355
+ // Ancestors a line drags in with it: every enclosing block header not yet
356
+ // kept, plus that block's closing line. Charged as part of the candidate's
357
+ // cost so the budget stays honest instead of overflowing at assembly time.
358
+ // `inPending` is a reusable membership mark, cleared via the `pending` list
359
+ // at the top of each call (C6). The old version used `pending.includes(p)`,
360
+ // O(depth) per probe, and never short-circuited on an already-kept ancestor
361
+ // chain — so every REJECTED candidate refilled `pending` to full nesting
362
+ // depth, giving O(candidates x depth^2). Depth is attacker-controlled and
363
+ // costs no extra body bytes: at 20k lines, depth 200 measured 251ms total
364
+ // while depth 800 measured 4512ms, of which 4443ms was this function.
365
+ const pending = []
366
+ const inPending = new Uint8Array(N)
367
+ function closureFor(i) {
368
+ for (const p of pending) inPending[p] = 0
369
+ pending.length = 0
370
+ if (!structure) return 0
371
+ let cost = 0
372
+ const mark = (line) => {
373
+ if (line < 0 || line >= N || keep[line] || inPending[line]) return false
374
+ inPending[line] = 1
375
+ pending.push(line)
376
+ cost += lineTokens[line]
377
+ return true
378
+ }
379
+ // A line that OPENS a block must drag in its own closer, or a kept
380
+ // signature is left hanging with no `}`. Ancestors alone do not cover
381
+ // this: the header is not inside its own body.
382
+ //
383
+ // closer/headerOf are LISTS — one line can open or close several blocks
384
+ // (`function f() { if (x) {`). Taking only the first would strand the
385
+ // others, which is the defect these lists exist to fix.
386
+ const ownClosers = structure.closer[i]
387
+ if (ownClosers) for (const c of ownClosers) if (c !== i) mark(c)
388
+
389
+ // Conversely, a closing line's own headers are not ancestors either — the
390
+ // body sits between them — so seed the walk with them when the candidate
391
+ // IS a closer. Continue from the outermost of them; the inner ones are its
392
+ // descendants and get pulled in via their own closer lists.
393
+ const ownHeaders = structure.headerOf[i]
394
+ let p = structure.parent[i]
395
+ if (ownHeaders && ownHeaders.length > 0) {
396
+ let outermost = ownHeaders[0]
397
+ for (const h of ownHeaders) {
398
+ mark(h)
399
+ if (h < outermost) outermost = h
400
+ }
401
+ p = structure.parent[outermost]
402
+ }
403
+
404
+ while (p >= 0) {
405
+ // Short-circuit: if this ancestor is already kept, everything above it
406
+ // was kept with it, so the rest of the chain is redundant. This is what
407
+ // turns the rejected-candidate case from O(depth) into O(1).
408
+ if (keep[p]) break
409
+ if (!mark(p)) break // already pending — the rest of the chain is too
410
+ const closers = structure.closer[p]
411
+ // closer[] is a LIST since the C2 fix. Reading it as a scalar silently
412
+ // worked for single-closer blocks (`[3]` coerces to "3") and silently
413
+ // did nothing for multi-closer ones (`[2,3]` -> NaN).
414
+ if (closers) for (const c of closers) if (c !== i) mark(c)
415
+ p = structure.parent[p]
416
+ }
417
+ return cost
418
+ }
419
+
157
420
  for (const c of candidates) {
158
- if (budgetUsed + c.len > targetTokens) continue
421
+ if (keep[c.i]) continue // already pulled in as an ancestor
422
+ const closureCost = closureFor(c.i)
423
+ if (budgetUsed + c.len + closureCost > targetTokens) continue
424
+ for (const a of pending) keep[a] = 1
159
425
  keep[c.i] = 1
160
- budgetUsed += c.len
426
+ budgetUsed += c.len + closureCost
427
+ }
428
+
429
+ // Blank lines cost zero tokens, so greedy selection always keeps them. Left
430
+ // alone they punch holes through dropped runs, and every hole costs another
431
+ // ~12-token marker — enough to overshoot the budget several times over on a
432
+ // fragmented body. A blank RUN stranded between two dropped lines carries
433
+ // nothing, so fold the whole run in.
434
+ //
435
+ // Must operate on runs, not single lines. An earlier version tested only
436
+ // `keep[i-1]` and `keep[i+1]`, which never fires on `\n\n` — the most common
437
+ // blank shape in source — because each blank sees the other as kept. Measured
438
+ // 72 of 148 surviving lines blank before this fix. Snapshotting `keep` does
439
+ // not help either; the run has to be considered as a unit.
440
+ for (let i = 1; i < N - 1; i++) {
441
+ if (!keep[i] || lines[i].trim() !== '') continue
442
+ let j = i
443
+ while (j < N - 1 && keep[j] && lines[j].trim() === '') j += 1
444
+ if (!keep[i - 1] && j < N && !keep[j]) {
445
+ for (let k = i; k < j; k += 1) keep[k] = 0
446
+ }
447
+ i = j - 1
161
448
  }
162
449
 
163
450
  // If even the anchors + marker exceed budget, still proceed — we need to
@@ -180,7 +467,10 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
180
467
  let chars = 0
181
468
  while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
182
469
  const runLen = j - i
183
- out.push(`[... ${runLen} lines omitted, ${chars} chars ...]`)
470
+ // Indent the marker to the dropped body's own depth, so a collapsed
471
+ // function body reads as part of the block it replaced.
472
+ const pad = lines[i].slice(0, indentWidth(lines[i]))
473
+ out.push(`${pad}[... ${runLen} lines omitted, ${chars} chars ...]`)
184
474
  i = j
185
475
  }
186
476
 
@@ -0,0 +1,269 @@
1
+ // Block structure detection for code bodies (Phase 7).
2
+ //
3
+ // `bm25-trim` drops individual lines. On code that produces syntactically
4
+ // broken output: a kept line from inside a function body, with its signature
5
+ // and closing brace gone, tells the model almost nothing. This module finds
6
+ // block ranges so the trimmer can enforce a simple invariant — keep a line,
7
+ // keep the headers and closers of every block enclosing it.
8
+ //
9
+ // Deliberately not tree-sitter. That is a native dependency, and Tamp ships a
10
+ // pure-JS package with a tight `files:` list. Two cheap detectors cover the
11
+ // languages coding agents actually read, and both refuse to guess: when the
12
+ // structure does not check out, `detectBlocks` returns null and the caller
13
+ // falls back to plain line trimming.
14
+
15
+ // --- Language selection ---
16
+
17
+ const BRACE_EXT = new Set([
18
+ 'js', 'jsx', 'mjs', 'cjs', 'ts', 'tsx', 'mts', 'cts',
19
+ 'java', 'c', 'h', 'cc', 'cpp', 'hpp', 'cs', 'go', 'rs',
20
+ 'swift', 'kt', 'kts', 'scala', 'php', 'dart', 'zig',
21
+ ])
22
+ const INDENT_EXT = new Set(['py', 'pyi', 'yaml', 'yml'])
23
+
24
+ export function languageForPath(path) {
25
+ if (typeof path !== 'string' || path.length === 0) return null
26
+ const dot = path.lastIndexOf('.')
27
+ if (dot < 0) return null
28
+ const ext = path.slice(dot + 1).toLowerCase()
29
+ if (BRACE_EXT.has(ext)) return 'brace'
30
+ if (INDENT_EXT.has(ext)) return 'indent'
31
+ return null
32
+ }
33
+
34
+ // Path-based detection only works on Anthropic requests (see
35
+ // lib/path-extract.js). Everywhere else we sniff the body itself.
36
+ const PY_BLOCK_RE = /^\s*(?:async\s+)?(?:def|class|if|for|while|with|try|elif|else|except|finally)\b.*:\s*$/
37
+
38
+ export function sniffLanguage(lines) {
39
+ if (!Array.isArray(lines) || lines.length < 10) return null
40
+ let braceOpens = 0
41
+ let pyBlocks = 0
42
+ const sample = Math.min(lines.length, 400)
43
+ for (let i = 0; i < sample; i++) {
44
+ const line = lines[i]
45
+ if (typeof line !== 'string') continue
46
+ if (/\{\s*$/.test(line)) braceOpens += 1
47
+ if (PY_BLOCK_RE.test(line)) pyBlocks += 1
48
+ }
49
+ // A low bar is safe: the brace detector independently returns null unless
50
+ // the whole body balances, so a wrong guess on a log file self-corrects.
51
+ if (braceOpens >= 3 && braceOpens > pyBlocks) return 'brace'
52
+ if (pyBlocks >= 3) return 'indent'
53
+ return null
54
+ }
55
+
56
+ // --- Shared helpers ---
57
+
58
+ export function indentWidth(line) {
59
+ if (typeof line !== 'string') return 0
60
+ let n = 0
61
+ while (n < line.length && (line[n] === ' ' || line[n] === '\t')) n += 1
62
+ return n
63
+ }
64
+
65
+ // --- Brace detector ---
66
+ //
67
+ // One left-to-right character scan carrying string AND comment state (F2).
68
+ //
69
+ // The previous version detected `/*` on the RAW line before stripping strings,
70
+ // so any `/*` substring — a URL path, a glob, prose about comment syntax —
71
+ // flipped the scanner into block-comment state and swallowed every following
72
+ // line's braces until an unrelated `*/`. It fired on this repo's own
73
+ // compress.js:258 and config.js:220, silently disabling block-aware trimming
74
+ // for 2 of 6 source files. Injecting one line into a 214KB body flipped 465
75
+ // detected blocks to 0 and produced 72 unclosed braces in the output.
76
+ //
77
+ // It also missed a complete `/* } */` on one line, because it declined to
78
+ // enter comment state when `*/` followed on the same line, and the regex-based
79
+ // stripper never handled `/* */` at all.
80
+ //
81
+ // Scanning once, left to right, makes both cases fall out for free: a `/*`
82
+ // inside a string is never seen as a comment opener because we are in string
83
+ // state when we reach it.
84
+ //
85
+ // Returns null when braces do not balance, or when comment state is still open
86
+ // at end of input — the honest signal that the scan mis-parsed something
87
+ // (regex literals and JSX are still not modelled).
88
+
89
+ function detectBraceBlocks(lines) {
90
+ const blocks = []
91
+ const open = []
92
+ let inBlockComment = false
93
+
94
+ for (let i = 0; i < lines.length; i++) {
95
+ const line = lines[i]
96
+ if (typeof line !== 'string') return null
97
+
98
+ let quote = null // "'", '"' or '`' while inside a string literal
99
+ let escaped = false
100
+
101
+ for (let c = 0; c < line.length; c++) {
102
+ const ch = line[c]
103
+
104
+ if (inBlockComment) {
105
+ if (ch === '*' && line[c + 1] === '/') { inBlockComment = false; c += 1 }
106
+ continue
107
+ }
108
+ if (quote !== null) {
109
+ if (escaped) { escaped = false; continue }
110
+ if (ch === '\\') { escaped = true; continue }
111
+ if (ch === quote) quote = null
112
+ continue
113
+ }
114
+ if (ch === '"' || ch === "'" || ch === '`') { quote = ch; continue }
115
+ if (ch === '/' && line[c + 1] === '*') { inBlockComment = true; c += 1; continue }
116
+ if (ch === '/' && line[c + 1] === '/') break // line comment: rest of line is noise
117
+ if (ch === '#') break // ditto for shell/python-style comments
118
+
119
+ if (ch === '{') {
120
+ // The header is the line the brace opens on. A brace that opens and
121
+ // closes on the same line yields a zero-length body and is dropped
122
+ // below, so object literals inline in an argument list cost nothing.
123
+ open.push(i)
124
+ } else if (ch === '}') {
125
+ const headerLine = open.pop()
126
+ if (headerLine === undefined) return null // unbalanced — give up
127
+ if (i > headerLine + 1) {
128
+ blocks.push({ headerLine, startLine: headerLine + 1, endLine: i - 1, closeLine: i })
129
+ }
130
+ }
131
+ }
132
+ // Single-quoted and double-quoted strings do not span lines in any language
133
+ // this detector claims; reset so one stray quote cannot swallow the file.
134
+ // Backtick state deliberately carries over (template literals are multiline).
135
+ if (quote === '"' || quote === "'") quote = null
136
+ }
137
+ if (open.length > 0 || inBlockComment) return null
138
+ return blocks
139
+ }
140
+
141
+ // --- Indent detector ---
142
+ //
143
+ // A block opens on a line ending in `:` and runs while indentation exceeds the
144
+ // header's. Blank lines belong to the block; they do not close it.
145
+ //
146
+ // Built in ONE pass with an indent stack (C3). The previous version scanned
147
+ // forward from every header to find its end, and because blank lines were
148
+ // skipped with `continue` rather than terminating the scan, every enclosing
149
+ // header re-walked the same blank field: O(headers x blank_lines), unbounded by
150
+ // body bytes since a blank line costs one byte but is visited once per open
151
+ // block. Measured 2000 headers over 200k blank lines (2.2MB) at 10783ms.
152
+
153
+ function detectIndentBlocks(lines) {
154
+ const blocks = []
155
+ const stack = [] // { headerLine, indent }, outermost first
156
+ let lastContentLine = -1
157
+
158
+ for (let i = 0; i < lines.length; i++) {
159
+ const line = lines[i]
160
+ if (typeof line !== 'string') return null
161
+ if (isBlank(line)) continue // blanks belong to whatever block is open
162
+
163
+ const indent = indentWidth(line)
164
+ while (stack.length > 0 && indent <= stack[stack.length - 1].indent) {
165
+ const done = stack.pop()
166
+ if (lastContentLine > done.headerLine) {
167
+ blocks.push({
168
+ headerLine: done.headerLine,
169
+ startLine: done.headerLine + 1,
170
+ endLine: lastContentLine,
171
+ closeLine: null,
172
+ })
173
+ }
174
+ }
175
+ if (PY_BLOCK_RE.test(line)) stack.push({ headerLine: i, indent })
176
+ lastContentLine = i
177
+ }
178
+ while (stack.length > 0) {
179
+ const done = stack.pop()
180
+ if (lastContentLine > done.headerLine) {
181
+ blocks.push({
182
+ headerLine: done.headerLine,
183
+ startLine: done.headerLine + 1,
184
+ endLine: lastContentLine,
185
+ closeLine: null,
186
+ })
187
+ }
188
+ }
189
+ return blocks
190
+ }
191
+
192
+ // Non-allocating blank check. `line.trim() === ''` allocated a string per line,
193
+ // which the old per-header rescan paid once per open block per blank line.
194
+ function isBlank(line) {
195
+ for (let i = 0; i < line.length; i++) {
196
+ const ch = line[i]
197
+ if (ch !== ' ' && ch !== '\t' && ch !== '\r') return false
198
+ }
199
+ return true
200
+ }
201
+
202
+ // --- Public entry ---
203
+ //
204
+ // `detectBlocks(lines, { path })` -> array of blocks, or null when no detector
205
+ // applies or the structure did not check out. Blocks are unordered with
206
+ // respect to nesting; use `buildStructure` to get the lookup maps.
207
+
208
+ export function detectBlocks(lines, { path = null } = {}) {
209
+ if (!Array.isArray(lines) || lines.length === 0) return null
210
+ const language = languageForPath(path) || sniffLanguage(lines)
211
+ if (language === 'brace') return detectBraceBlocks(lines)
212
+ if (language === 'indent') return detectIndentBlocks(lines)
213
+ return null
214
+ }
215
+
216
+ // Flatten blocks into per-line lookups the trimmer needs:
217
+ // parent[i] innermost block header enclosing line i, or -1
218
+ // closer[h] ARRAY of closing lines for blocks whose header is line h
219
+ // headerOf[c] ARRAY of header lines for blocks closing on line c
220
+ //
221
+ // `headerOf` is the inverse of `closer`: a bare `}` sits outside its own
222
+ // block's body, so without it a closing line looks parentless and can be kept
223
+ // on its own, stranding a `}` with no signature.
224
+ //
225
+ // Both are LISTS, not scalars (C2). One physical line can open or close more
226
+ // than one block — `function f() { if (x) {` opens two, `} }` closes two.
227
+ // Scalar maps keyed by line number silently kept only the LAST write, so the
228
+ // outer closer was lost and the trimmer emitted a signature with no matching
229
+ // brace. Verified before the fix: closer[0] === 2 (the inner `}`) not 3.
230
+ //
231
+ // `parent` is filled by one stack sweep rather than by painting each block's
232
+ // span (C1). Painting costs O(sum of block spans) = O(N x nesting depth),
233
+ // quadratic on deeply nested input: a body of N/2 `{` then N/2 `}` measured
234
+ // 200ms at 80KB, 1188ms at 200KB, 4741ms at 400KB. The sweep is O(N + blocks).
235
+ export function buildStructure(lines, blocks) {
236
+ const n = lines.length
237
+ const parent = new Int32Array(n).fill(-1)
238
+ const closer = new Array(n)
239
+ const headerOf = new Array(n)
240
+ if (!Array.isArray(blocks) || blocks.length === 0) return { parent, closer, headerOf }
241
+
242
+ for (const block of blocks) {
243
+ const h = block.headerLine
244
+ const c = block.closeLine
245
+ if (h < 0 || h >= n || c === null || c < 0 || c >= n) continue
246
+ ;(closer[h] || (closer[h] = [])).push(c)
247
+ ;(headerOf[c] || (headerOf[c] = [])).push(h)
248
+ }
249
+
250
+ // Outer-first among blocks starting on the same line, so the narrower one
251
+ // lands on top of the stack and wins `parent` for the lines both cover.
252
+ const byStart = blocks
253
+ .slice()
254
+ .sort((a, b) => (a.startLine - b.startLine) || (b.endLine - a.endLine))
255
+ const stack = []
256
+ let bi = 0
257
+ for (let i = 0; i < n; i++) {
258
+ while (stack.length > 0 && stack[stack.length - 1].endLine < i) stack.pop()
259
+ while (bi < byStart.length && byStart[bi].startLine <= i) {
260
+ const b = byStart[bi]
261
+ bi += 1
262
+ if (b.endLine >= i) stack.push(b)
263
+ }
264
+ // A block never covers its own header (startLine is headerLine + 1), so
265
+ // parent[headerLine] keeps pointing at the enclosing block.
266
+ parent[i] = stack.length > 0 ? stack[stack.length - 1].headerLine : -1
267
+ }
268
+ return { parent, closer, headerOf }
269
+ }
package/package.json CHANGED
@@ -14,7 +14,7 @@
14
14
  "sidecar/server.py",
15
15
  "sidecar/requirements.txt"
16
16
  ],
17
- "version": "0.8.15",
17
+ "version": "0.8.16",
18
18
  "description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
19
19
  "type": "module",
20
20
  "main": "index.js",
@@ -23,9 +23,11 @@
23
23
  },
24
24
  "scripts": {
25
25
  "start": "node bin/tamp.js",
26
+ "prepublishOnly": "node scripts/publish-guard.js",
26
27
  "test": "node --test test/*.test.js",
27
28
  "test:sidecar": "node test/capture-golden.js --verify",
28
29
  "bench:semantic": "node bench/semantic-eval.js",
30
+ "bench:retrieval": "node bench/retrieval-eval.js",
29
31
  "test:quality": "node bench/quality-eval.js",
30
32
  "compat:probe": "node test/compat/probe-proxy.js",
31
33
  "compat:run": "node test/compat/run.mjs"