@sliday/tamp 0.8.7 → 0.8.16

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/lib/bm25.js CHANGED
@@ -10,28 +10,123 @@
10
10
  // This primitive is reusable: any future stage that wants "keep the most
11
11
  // relevant N lines" can import tokenize/bm25Scores directly.
12
12
 
13
- import { countTokens } from '@anthropic-ai/tokenizer'
13
+ import { countTokens as rawCountTokens } from '@anthropic-ai/tokenizer'
14
+ import { detectBlocks, buildStructure, indentWidth } from './code-chunks.js'
15
+
16
+ // --- Safe token counting (C5) ---
17
+ //
18
+ // `countTokens` is NOT the fixed-cost call an earlier comment in this file
19
+ // claimed. It has a ~22ms floor on small inputs, but it is superlinear on
20
+ // large low-entropy input and it HARD CRASHES on some of it. Measured on
21
+ // whitespace-heavy bodies (400-char space runs):
22
+ //
23
+ // 80KB -> 8.7s 200KB -> 22.4s 400KB -> 88.9s
24
+ // 1MB -> RuntimeError: unreachable (tiktoken WASM trap)
25
+ //
26
+ // `bm25-trim` only runs on bodies over 64KB, so every attacker-controlled
27
+ // tool_result on this path paid it. A WASM trap is not catchable as a normal
28
+ // error in every runtime, so the primary defence is the size cap; the
29
+ // try/catch is only the backstop.
30
+ //
31
+ // The cap is 4KB, not 64KB, because the cost is driven by CONTENT, not size.
32
+ // Measured:
33
+ //
34
+ // spaces: 1KB 22ms 4KB 31ms 8KB 58ms 16KB 167ms 32KB 609ms
35
+ // prose: 8KB 22ms 32KB 24ms 64KB 27ms
36
+ //
37
+ // Prose is flat at the ~22ms floor whatever its length; pathological runs go
38
+ // quadratic. 4KB bounds the worst case to ~31ms. A 64KB cap still measured
39
+ // 44s on a megabyte of spaces, which is why the first attempt at this fix
40
+ // did not work.
41
+ const COUNT_CAP = 4 * 1024
42
+ const SAMPLES = 3
43
+
44
+ export function safeCountTokens(text) {
45
+ if (typeof text !== 'string' || text.length === 0) return 0
46
+ try {
47
+ if (text.length <= COUNT_CAP) return rawCountTokens(text)
48
+ // Three windows spread across the body rather than one prefix — same cost,
49
+ // better representativeness on a body whose head is not typical of its tail
50
+ // (a log with a JSON blob at the end, a file with a licence header).
51
+ const each = Math.floor(COUNT_CAP / SAMPLES)
52
+ const stride = Math.floor((text.length - each) / (SAMPLES - 1))
53
+ let sampled = 0
54
+ let counted = 0
55
+ for (let s = 0; s < SAMPLES; s++) {
56
+ const start = Math.min(s * stride, text.length - each)
57
+ const window = text.slice(start, start + each)
58
+ counted += rawCountTokens(window)
59
+ sampled += window.length
60
+ }
61
+ if (sampled === 0) return Math.max(1, Math.ceil(text.length / 4))
62
+ return Math.max(1, Math.round(counted * (text.length / sampled)))
63
+ } catch {
64
+ // Tokenizer refused this input. Fall back to a coarse estimate rather than
65
+ // aborting compression for the whole request.
66
+ return Math.max(1, Math.ceil(text.length / 4))
67
+ }
68
+ }
69
+
70
+ const countTokens = safeCountTokens
14
71
 
15
72
  // --- Tokenizer ---
16
73
  // Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
17
74
  // `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
18
75
  // separators to underscore before splitting on non-word characters. Single
19
76
  // character tokens are dropped (too noisy for BM25 on short queries).
77
+ //
78
+ // Emitting ONLY the collapsed form silently destroys recall on the queries
79
+ // agents actually ask. `parse config` tokenizes to ['parse','config'], while
80
+ // `function parseConfig(` yields ['parseconfig'] and `def parse_config(`
81
+ // yields ['parse_config'] — no shared term, so BM25 scores the definition
82
+ // line the user asked about at exactly 0 and `bm25-trim` drops it. So we also
83
+ // emit the identifier's parts, splitting on camelCase, PascalCase, acronym
84
+ // runs, and underscores. Case must survive until the split, hence the
85
+ // lowercase happens per-token at the end rather than up front.
20
86
  const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
21
87
  const SPLIT_RE = /[^A-Za-z0-9_]+/
88
+ // `HTTPServer` -> `HTTP Server`. Runs before CAMEL_LOWER_UPPER so that
89
+ // `parseHTTPResponse` splits three ways rather than swallowing the acronym.
90
+ const CAMEL_ACRONYM = /([A-Z]+)([A-Z][a-z])/g
91
+ const CAMEL_LOWER_UPPER = /([a-z0-9])([A-Z])/g
92
+ // No cap on parts per identifier. An earlier version capped at 4 and truncated
93
+ // from the TAIL, which reintroduced the exact bug this file exists to fix:
94
+ // `get_or_create_default_workspace_settings` emitted get/or/create/default and
95
+ // dropped `workspace` and `settings`, so a query for "workspace settings"
96
+ // scored the definition line at 0 and deleted it. The cap was motivated by an
97
+ // IDF-dilution worry that was never measured. If dilution is real,
98
+ // bench/retrieval-eval.js is where it should show up.
99
+
100
+ // Split one raw token into its component words. Returns [] when the token has
101
+ // no internal structure, so an already-atomic `config` is not emitted twice
102
+ // (which would inflate its term frequency and skew BM25).
103
+ function subTokens(rawTok) {
104
+ const spaced = rawTok
105
+ .replace(CAMEL_ACRONYM, '$1 $2')
106
+ .replace(CAMEL_LOWER_UPPER, '$1 $2')
107
+ .replace(/_+/g, ' ')
108
+ const parts = spaced.split(' ')
109
+ if (parts.length < 2) return []
110
+ const out = []
111
+ for (const part of parts) {
112
+ if (part.length < 2) continue
113
+ out.push(part.toLowerCase())
114
+ }
115
+ return out
116
+ }
22
117
 
23
118
  export function tokenize(text) {
24
119
  if (typeof text !== 'string' || text.length === 0) return []
25
- let normalized = text.toLowerCase()
26
120
  // Collapse intra-word separators. Run twice so chains like `a.b.c` fully
27
121
  // collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
28
- normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
122
+ let normalized = text.replace(INTRA_WORD_RE, '$1_$2')
29
123
  normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
30
124
  const raw = normalized.split(SPLIT_RE)
31
125
  const out = []
32
126
  for (const tok of raw) {
33
- if (tok.length < 2) continue
34
- out.push(tok)
127
+ if (tok.length === 0) continue
128
+ if (tok.length >= 2) out.push(tok.toLowerCase())
129
+ for (const part of subTokens(tok)) out.push(part)
35
130
  }
36
131
  return out
37
132
  }
@@ -39,60 +134,76 @@ export function tokenize(text) {
39
134
  // --- BM25 ---
40
135
  // Standard Okapi BM25. Each document is a line; query is the user's last
41
136
  // message. Returns Float64Array of scores aligned with the documents array.
42
- export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
137
+ // `tokenizer` is injectable so bench/retrieval-eval.js can score the same
138
+ // corpus with the pre-subtoken tokenizer and report a real before/after.
139
+ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75, tokenizer = tokenize } = {}) {
43
140
  const N = documents.length
44
141
  const scores = new Float64Array(N)
45
142
  if (N === 0) return scores
46
143
 
47
- const qTokens = tokenize(query)
144
+ const qTokens = tokenizer(query)
48
145
  if (qTokens.length === 0) return scores
49
146
 
50
- // Tokenize docs + build doc stats.
51
- const docTokens = new Array(N)
147
+ // ONE pass over the documents (C7 + C8).
148
+ //
149
+ // C7: the old version kept `docTokens` — the full tokenization of every line
150
+ // — alive for the whole call, though only query terms are ever read from it.
151
+ // A 4.6MB body (60k lines) retained 47.4MB, roughly 10x amplification, on top
152
+ // of the original string, the split lines array and the parsed request body.
153
+ // The proxy serves concurrent requests with no per-request size cap on this
154
+ // stage, so that was an OOM path, not just memory pressure.
155
+ //
156
+ // C8: document frequency was computed as `docTokens[i].includes(term)` for
157
+ // every unique query term against every line — O(uniqQ x N x tokens/line) of
158
+ // linear scans. Both dimensions are hostile: N is the attacker-controlled
159
+ // line count, and the query is the user's last message, which routinely
160
+ // contains a pasted stack trace. Measured at N=20000: 70ms for a 10-word
161
+ // query, 163ms for 100 words, 676ms for 500.
162
+ //
163
+ // Now: tokenize each line once, keep only the per-line term frequencies for
164
+ // terms that appear in the query (empty for the vast majority of lines), and
165
+ // accumulate df in the same pass. Peak retention scales with the QUERY size,
166
+ // not the body size.
167
+ const qSet = new Set(qTokens)
52
168
  const docLens = new Float64Array(N)
169
+ const docTf = new Array(N) // Map|null, only query terms, null when none present
170
+ const df = new Map()
53
171
  let totalLen = 0
172
+
54
173
  for (let i = 0; i < N; i++) {
55
- const toks = tokenize(documents[i])
56
- docTokens[i] = toks
174
+ const toks = tokenizer(documents[i])
57
175
  docLens[i] = toks.length
58
176
  totalLen += toks.length
177
+ let tf = null
178
+ for (const t of toks) {
179
+ if (!qSet.has(t)) continue
180
+ if (tf === null) tf = new Map()
181
+ tf.set(t, (tf.get(t) || 0) + 1)
182
+ }
183
+ docTf[i] = tf
184
+ if (tf !== null) for (const t of tf.keys()) df.set(t, (df.get(t) || 0) + 1)
185
+ // `toks` goes out of scope here and is collectable — nothing retains it.
59
186
  }
187
+
60
188
  const avgdl = N > 0 ? totalLen / N : 0
61
189
  if (avgdl === 0) return scores
62
190
 
63
- // Document-frequency per unique query term.
64
- const uniqQ = Array.from(new Set(qTokens))
65
- const df = new Map()
66
- for (const term of uniqQ) {
67
- let count = 0
68
- for (let i = 0; i < N; i++) {
69
- if (docTokens[i].includes(term)) count += 1
70
- }
71
- df.set(term, count)
72
- }
73
-
74
- // Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
191
+ // Okapi BM25 IDF with +1 smoothing.
75
192
  const idf = new Map()
76
- for (const term of uniqQ) {
193
+ for (const term of qSet) {
77
194
  const n = df.get(term) || 0
78
- const v = Math.log(1 + (N - n + 0.5) / (n + 0.5))
79
- idf.set(term, v)
195
+ idf.set(term, Math.log(1 + (N - n + 0.5) / (n + 0.5)))
80
196
  }
81
197
 
82
198
  for (let i = 0; i < N; i++) {
199
+ const tf = docTf[i]
200
+ if (tf === null) continue // no query term on this line — scores 0, skip
83
201
  const dl = docLens[i]
84
202
  if (dl === 0) continue
85
- // Term-frequency table for this doc (only for query terms we care about).
86
- const tf = new Map()
87
- const toks = docTokens[i]
88
- for (const t of toks) {
89
- if (!idf.has(t)) continue
90
- tf.set(t, (tf.get(t) || 0) + 1)
91
- }
92
203
  let s = 0
93
- for (const term of uniqQ) {
94
- const f = tf.get(term) || 0
95
- if (f === 0) continue
204
+ // Iterate the terms actually PRESENT (usually 0-3) rather than every
205
+ // unique query term, so a long query costs nothing on unrelated lines.
206
+ for (const [term, f] of tf) {
96
207
  const num = f * (k1 + 1)
97
208
  const den = f + k1 * (1 - b + b * (dl / avgdl))
98
209
  s += idf.get(term) * (num / den)
@@ -102,6 +213,55 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
102
213
  return scores
103
214
  }
104
215
 
216
+ // --- Rerank ---
217
+ // Raw BM25 treats every line as an equal bag of words. Two code-aware signals
218
+ // and one structural one fix its most visible failures.
219
+ //
220
+ // Note on scope: a path-based penalty (test files, .d.ts stubs) would be a
221
+ // no-op here. `bm25-trim` ranks lines WITHIN one tool_result, so a multiplier
222
+ // constant across the whole body cannot change any relative ordering. Path is
223
+ // used for language detection only.
224
+
225
+ // A line that DEFINES a query term beats a line that merely mentions it.
226
+ const DEF_RE = /\b(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function|class|def|const|let|var|type|interface|enum|struct|impl|trait|fn|func|module|namespace)\s+([A-Za-z_$][\w$]*)/
227
+ const COMMENT_RE = /^\s*(?:\/\/|#|\*|\/\*|--|<!--|;)/
228
+ const DEF_BOOST = 1.6
229
+ const COMMENT_PENALTY = 0.6
230
+ // A kept line lends a quarter of its score to each immediate neighbour, so the
231
+ // survivors read as contiguous code rather than confetti. Applied off the
232
+ // pre-coherence scores so boosts do not cascade down the file.
233
+ const COHERENCE = 0.25
234
+
235
+ export function rerankScores(scores, lines, queryTokens) {
236
+ const n = lines.length
237
+ const out = new Float64Array(n)
238
+ const qSet = new Set(queryTokens)
239
+
240
+ for (let i = 0; i < n; i++) {
241
+ let s = scores[i]
242
+ const line = lines[i]
243
+ if (typeof line !== 'string' || s === 0) { out[i] = s; continue }
244
+
245
+ const def = DEF_RE.exec(line)
246
+ if (def) {
247
+ for (const tok of tokenize(def[1])) {
248
+ if (qSet.has(tok)) { s *= DEF_BOOST; break }
249
+ }
250
+ }
251
+ if (COMMENT_RE.test(line)) s *= COMMENT_PENALTY
252
+ out[i] = s
253
+ }
254
+
255
+ if (n < 3) return out
256
+ const cohered = new Float64Array(n)
257
+ for (let i = 0; i < n; i++) {
258
+ const left = i > 0 ? out[i - 1] : 0
259
+ const right = i < n - 1 ? out[i + 1] : 0
260
+ cohered[i] = out[i] + COHERENCE * Math.max(left, right)
261
+ }
262
+ return cohered
263
+ }
264
+
105
265
  // --- Line trimmer ---
106
266
  // Keep top-K highest-scoring lines until `targetTokens` is consumed. First
107
267
  // and last line are always preserved. Returns null when text is already
@@ -109,7 +269,16 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
109
269
  //
110
270
  // Output shape:
111
271
  // { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
112
- export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLines = 10 } = {}) {
272
+ // `tokenizer` and `rerank` exist for bench/retrieval-eval.js, which replays the
273
+ // same fixtures through the pre-subtoken tokenizer to produce a real
274
+ // before/after. Production callers should leave both at their defaults.
275
+ export function trimLinesByRelevance(text, query, {
276
+ targetTokens = 4096,
277
+ minLines = 10,
278
+ path = null,
279
+ tokenizer = tokenize,
280
+ rerank = true,
281
+ } = {}) {
113
282
  if (typeof text !== 'string' || text.length === 0) return null
114
283
 
115
284
  const originalTokens = countTokens(text)
@@ -119,21 +288,58 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
119
288
  const N = lines.length
120
289
  if (N < minLines) return null
121
290
 
122
- // Per-line token costs (includes the trailing newline for all but the last
123
- // line; we approximate by counting each line once and adding N-1 tokens).
291
+ // Per-line token costs. `countTokens` carries a fixed ~21ms cost per call
292
+ // regardless of input length, so counting each line separately turns a
293
+ // 1000-line tool_result into a ~20-second stall on the proxy hot path — for
294
+ // a budget that is approximate by nature. Instead calibrate a chars-per-token
295
+ // ratio from the whole-body count already taken above, and estimate per line.
296
+ // Two tokenizer calls total instead of N + 2. The reported
297
+ // original/trimmed token counts stay exact.
298
+ // Clamped (C4). A single global ratio is gameable: token density spans ~200x
299
+ // across content (400 spaces = 133 chars/token; emoji = 0.67), so a body
300
+ // whose bulk is low-density padding and whose payload is dense text made
301
+ // every payload line look ~20x cheaper than it is. Measured: a 762KB body
302
+ // reported budgetUsed 4050 against targetTokens 4096 while emitting 31,869
303
+ // real tokens — 7.8x over budget, with the only guard being "smaller than
304
+ // the (attacker-sized) input".
305
+ //
306
+ // Real chars-per-token tops out near 4.6 on ordinary prose. Capping the
307
+ // assumed ratio at 4 means a line can never be priced below len/4, so dense
308
+ // content cannot be smuggled in under a padded average. Over-estimating is
309
+ // safe here — it only makes the trim more conservative.
310
+ const MAX_CHARS_PER_TOKEN = 4
311
+ const rawRatio = originalTokens > 0 ? text.length / originalTokens : MAX_CHARS_PER_TOKEN
312
+ const charsPerToken = Math.min(rawRatio, MAX_CHARS_PER_TOKEN)
124
313
  const lineTokens = new Array(N)
125
314
  for (let i = 0; i < N; i++) {
126
- lineTokens[i] = countTokens(lines[i])
315
+ lineTokens[i] = Math.max(1, Math.ceil((lines[i].length + 1) / charsPerToken))
127
316
  }
128
317
  const newlineCost = Math.max(0, N - 1)
129
318
 
130
- const scores = bm25Scores(lines, query)
319
+ const raw = bm25Scores(lines, query, { tokenizer })
320
+ const scores = rerank ? rerankScores(raw, lines, tokenizer(query)) : raw
321
+
322
+ // Block structure, when the body parses as code. Keeping a line without its
323
+ // enclosing signature and closing brace emits syntactically broken output;
324
+ // `structure` lets selection pull those ancestors in as part of the line's
325
+ // cost. Null (prose, logs, JSON, unbalanced source) falls back to plain
326
+ // per-line selection, exactly as before.
327
+ const blocks = detectBlocks(lines, { path })
328
+ const structure = blocks ? buildStructure(lines, blocks) : null
131
329
 
132
330
  // Always-keep set: first + last as anchors.
133
331
  const keep = new Uint8Array(N)
134
332
  keep[0] = 1
135
333
  keep[N - 1] = 1
136
- let budgetUsed = lineTokens[0] + (N > 1 ? lineTokens[N - 1] : 0)
334
+ // Anchor the last NON-EMPTY line too. Tool output almost always ends with
335
+ // '\n', so lines[N-1] is '' — pinning only that empty terminator would let
336
+ // the real final line (often a trailing error or stack trace, exactly the
337
+ // "context anchor" this claims to preserve) be dropped as a candidate.
338
+ let lastReal = N - 1
339
+ while (lastReal > 0 && lines[lastReal] === '') lastReal--
340
+ keep[lastReal] = 1
341
+ let budgetUsed = 0
342
+ for (const idx of new Set([0, N - 1, lastReal])) budgetUsed += lineTokens[idx]
137
343
  // Rough marker cost reserved per collapsed gap; we'll reconcile below.
138
344
  const MARKER_RESERVE = 16
139
345
 
@@ -141,14 +347,104 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
141
347
  // context preferred on ties). We pick greedily until budget is hit.
142
348
  const candidates = []
143
349
  for (let i = 1; i < N - 1; i++) {
350
+ if (keep[i]) continue // skip already-anchored lines (e.g. lastReal)
144
351
  candidates.push({ i, score: scores[i], len: lineTokens[i] })
145
352
  }
146
353
  candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
147
354
 
355
+ // Ancestors a line drags in with it: every enclosing block header not yet
356
+ // kept, plus that block's closing line. Charged as part of the candidate's
357
+ // cost so the budget stays honest instead of overflowing at assembly time.
358
+ // `inPending` is a reusable membership mark, cleared via the `pending` list
359
+ // at the top of each call (C6). The old version used `pending.includes(p)`,
360
+ // O(depth) per probe, and never short-circuited on an already-kept ancestor
361
+ // chain — so every REJECTED candidate refilled `pending` to full nesting
362
+ // depth, giving O(candidates x depth^2). Depth is attacker-controlled and
363
+ // costs no extra body bytes: at 20k lines, depth 200 measured 251ms total
364
+ // while depth 800 measured 4512ms, of which 4443ms was this function.
365
+ const pending = []
366
+ const inPending = new Uint8Array(N)
367
+ function closureFor(i) {
368
+ for (const p of pending) inPending[p] = 0
369
+ pending.length = 0
370
+ if (!structure) return 0
371
+ let cost = 0
372
+ const mark = (line) => {
373
+ if (line < 0 || line >= N || keep[line] || inPending[line]) return false
374
+ inPending[line] = 1
375
+ pending.push(line)
376
+ cost += lineTokens[line]
377
+ return true
378
+ }
379
+ // A line that OPENS a block must drag in its own closer, or a kept
380
+ // signature is left hanging with no `}`. Ancestors alone do not cover
381
+ // this: the header is not inside its own body.
382
+ //
383
+ // closer/headerOf are LISTS — one line can open or close several blocks
384
+ // (`function f() { if (x) {`). Taking only the first would strand the
385
+ // others, which is the defect these lists exist to fix.
386
+ const ownClosers = structure.closer[i]
387
+ if (ownClosers) for (const c of ownClosers) if (c !== i) mark(c)
388
+
389
+ // Conversely, a closing line's own headers are not ancestors either — the
390
+ // body sits between them — so seed the walk with them when the candidate
391
+ // IS a closer. Continue from the outermost of them; the inner ones are its
392
+ // descendants and get pulled in via their own closer lists.
393
+ const ownHeaders = structure.headerOf[i]
394
+ let p = structure.parent[i]
395
+ if (ownHeaders && ownHeaders.length > 0) {
396
+ let outermost = ownHeaders[0]
397
+ for (const h of ownHeaders) {
398
+ mark(h)
399
+ if (h < outermost) outermost = h
400
+ }
401
+ p = structure.parent[outermost]
402
+ }
403
+
404
+ while (p >= 0) {
405
+ // Short-circuit: if this ancestor is already kept, everything above it
406
+ // was kept with it, so the rest of the chain is redundant. This is what
407
+ // turns the rejected-candidate case from O(depth) into O(1).
408
+ if (keep[p]) break
409
+ if (!mark(p)) break // already pending — the rest of the chain is too
410
+ const closers = structure.closer[p]
411
+ // closer[] is a LIST since the C2 fix. Reading it as a scalar silently
412
+ // worked for single-closer blocks (`[3]` coerces to "3") and silently
413
+ // did nothing for multi-closer ones (`[2,3]` -> NaN).
414
+ if (closers) for (const c of closers) if (c !== i) mark(c)
415
+ p = structure.parent[p]
416
+ }
417
+ return cost
418
+ }
419
+
148
420
  for (const c of candidates) {
149
- if (budgetUsed + c.len > targetTokens) continue
421
+ if (keep[c.i]) continue // already pulled in as an ancestor
422
+ const closureCost = closureFor(c.i)
423
+ if (budgetUsed + c.len + closureCost > targetTokens) continue
424
+ for (const a of pending) keep[a] = 1
150
425
  keep[c.i] = 1
151
- budgetUsed += c.len
426
+ budgetUsed += c.len + closureCost
427
+ }
428
+
429
+ // Blank lines cost zero tokens, so greedy selection always keeps them. Left
430
+ // alone they punch holes through dropped runs, and every hole costs another
431
+ // ~12-token marker — enough to overshoot the budget several times over on a
432
+ // fragmented body. A blank RUN stranded between two dropped lines carries
433
+ // nothing, so fold the whole run in.
434
+ //
435
+ // Must operate on runs, not single lines. An earlier version tested only
436
+ // `keep[i-1]` and `keep[i+1]`, which never fires on `\n\n` — the most common
437
+ // blank shape in source — because each blank sees the other as kept. Measured
438
+ // 72 of 148 surviving lines blank before this fix. Snapshotting `keep` does
439
+ // not help either; the run has to be considered as a unit.
440
+ for (let i = 1; i < N - 1; i++) {
441
+ if (!keep[i] || lines[i].trim() !== '') continue
442
+ let j = i
443
+ while (j < N - 1 && keep[j] && lines[j].trim() === '') j += 1
444
+ if (!keep[i - 1] && j < N && !keep[j]) {
445
+ for (let k = i; k < j; k += 1) keep[k] = 0
446
+ }
447
+ i = j - 1
152
448
  }
153
449
 
154
450
  // If even the anchors + marker exceed budget, still proceed — we need to
@@ -171,7 +467,10 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
171
467
  let chars = 0
172
468
  while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
173
469
  const runLen = j - i
174
- out.push(`[... ${runLen} lines omitted, ${chars} chars ...]`)
470
+ // Indent the marker to the dropped body's own depth, so a collapsed
471
+ // function body reads as part of the block it replaced.
472
+ const pad = lines[i].slice(0, indentWidth(lines[i]))
473
+ out.push(`${pad}[... ${runLen} lines omitted, ${chars} chars ...]`)
175
474
  i = j
176
475
  }
177
476