@sliday/tamp 0.8.7 → 0.8.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/bin/tamp.js +1 -1
- package/compress.js +92 -16
- package/config.js +23 -4
- package/detect.js +48 -1
- package/index.js +9 -2
- package/lib/bm25.js +344 -45
- package/lib/code-chunks.js +269 -0
- package/lib/redact.js +78 -0
- package/lib/rewriters/commands/docker.js +7 -5
- package/lib/rewriters/commands/npm.js +5 -3
- package/lib/rewriters/commands/pytest.js +10 -4
- package/metadata.js +1 -1
- package/package.json +3 -1
- package/providers.js +13 -15
- package/session-graph.js +8 -1
- package/stats.js +8 -3
package/lib/bm25.js
CHANGED
|
@@ -10,28 +10,123 @@
|
|
|
10
10
|
// This primitive is reusable: any future stage that wants "keep the most
|
|
11
11
|
// relevant N lines" can import tokenize/bm25Scores directly.
|
|
12
12
|
|
|
13
|
-
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
13
|
+
import { countTokens as rawCountTokens } from '@anthropic-ai/tokenizer'
|
|
14
|
+
import { detectBlocks, buildStructure, indentWidth } from './code-chunks.js'
|
|
15
|
+
|
|
16
|
+
// --- Safe token counting (C5) ---
|
|
17
|
+
//
|
|
18
|
+
// `countTokens` is NOT the fixed-cost call an earlier comment in this file
|
|
19
|
+
// claimed. It has a ~22ms floor on small inputs, but it is superlinear on
|
|
20
|
+
// large low-entropy input and it HARD CRASHES on some of it. Measured on
|
|
21
|
+
// whitespace-heavy bodies (400-char space runs):
|
|
22
|
+
//
|
|
23
|
+
// 80KB -> 8.7s 200KB -> 22.4s 400KB -> 88.9s
|
|
24
|
+
// 1MB -> RuntimeError: unreachable (tiktoken WASM trap)
|
|
25
|
+
//
|
|
26
|
+
// `bm25-trim` only runs on bodies over 64KB, so every attacker-controlled
|
|
27
|
+
// tool_result on this path paid it. A WASM trap is not catchable as a normal
|
|
28
|
+
// error in every runtime, so the primary defence is the size cap; the
|
|
29
|
+
// try/catch is only the backstop.
|
|
30
|
+
//
|
|
31
|
+
// The cap is 4KB, not 64KB, because the cost is driven by CONTENT, not size.
|
|
32
|
+
// Measured:
|
|
33
|
+
//
|
|
34
|
+
// spaces: 1KB 22ms 4KB 31ms 8KB 58ms 16KB 167ms 32KB 609ms
|
|
35
|
+
// prose: 8KB 22ms 32KB 24ms 64KB 27ms
|
|
36
|
+
//
|
|
37
|
+
// Prose is flat at the ~22ms floor whatever its length; pathological runs go
|
|
38
|
+
// quadratic. 4KB bounds the worst case to ~31ms. A 64KB cap still measured
|
|
39
|
+
// 44s on a megabyte of spaces, which is why the first attempt at this fix
|
|
40
|
+
// did not work.
|
|
41
|
+
const COUNT_CAP = 4 * 1024
|
|
42
|
+
const SAMPLES = 3
|
|
43
|
+
|
|
44
|
+
export function safeCountTokens(text) {
|
|
45
|
+
if (typeof text !== 'string' || text.length === 0) return 0
|
|
46
|
+
try {
|
|
47
|
+
if (text.length <= COUNT_CAP) return rawCountTokens(text)
|
|
48
|
+
// Three windows spread across the body rather than one prefix — same cost,
|
|
49
|
+
// better representativeness on a body whose head is not typical of its tail
|
|
50
|
+
// (a log with a JSON blob at the end, a file with a licence header).
|
|
51
|
+
const each = Math.floor(COUNT_CAP / SAMPLES)
|
|
52
|
+
const stride = Math.floor((text.length - each) / (SAMPLES - 1))
|
|
53
|
+
let sampled = 0
|
|
54
|
+
let counted = 0
|
|
55
|
+
for (let s = 0; s < SAMPLES; s++) {
|
|
56
|
+
const start = Math.min(s * stride, text.length - each)
|
|
57
|
+
const window = text.slice(start, start + each)
|
|
58
|
+
counted += rawCountTokens(window)
|
|
59
|
+
sampled += window.length
|
|
60
|
+
}
|
|
61
|
+
if (sampled === 0) return Math.max(1, Math.ceil(text.length / 4))
|
|
62
|
+
return Math.max(1, Math.round(counted * (text.length / sampled)))
|
|
63
|
+
} catch {
|
|
64
|
+
// Tokenizer refused this input. Fall back to a coarse estimate rather than
|
|
65
|
+
// aborting compression for the whole request.
|
|
66
|
+
return Math.max(1, Math.ceil(text.length / 4))
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
|
|
70
|
+
const countTokens = safeCountTokens
|
|
14
71
|
|
|
15
72
|
// --- Tokenizer ---
|
|
16
73
|
// Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
|
|
17
74
|
// `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
|
|
18
75
|
// separators to underscore before splitting on non-word characters. Single
|
|
19
76
|
// character tokens are dropped (too noisy for BM25 on short queries).
|
|
77
|
+
//
|
|
78
|
+
// Emitting ONLY the collapsed form silently destroys recall on the queries
|
|
79
|
+
// agents actually ask. `parse config` tokenizes to ['parse','config'], while
|
|
80
|
+
// `function parseConfig(` yields ['parseconfig'] and `def parse_config(`
|
|
81
|
+
// yields ['parse_config'] — no shared term, so BM25 scores the definition
|
|
82
|
+
// line the user asked about at exactly 0 and `bm25-trim` drops it. So we also
|
|
83
|
+
// emit the identifier's parts, splitting on camelCase, PascalCase, acronym
|
|
84
|
+
// runs, and underscores. Case must survive until the split, hence the
|
|
85
|
+
// lowercase happens per-token at the end rather than up front.
|
|
20
86
|
const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
|
|
21
87
|
const SPLIT_RE = /[^A-Za-z0-9_]+/
|
|
88
|
+
// `HTTPServer` -> `HTTP Server`. Runs before CAMEL_LOWER_UPPER so that
|
|
89
|
+
// `parseHTTPResponse` splits three ways rather than swallowing the acronym.
|
|
90
|
+
const CAMEL_ACRONYM = /([A-Z]+)([A-Z][a-z])/g
|
|
91
|
+
const CAMEL_LOWER_UPPER = /([a-z0-9])([A-Z])/g
|
|
92
|
+
// No cap on parts per identifier. An earlier version capped at 4 and truncated
|
|
93
|
+
// from the TAIL, which reintroduced the exact bug this file exists to fix:
|
|
94
|
+
// `get_or_create_default_workspace_settings` emitted get/or/create/default and
|
|
95
|
+
// dropped `workspace` and `settings`, so a query for "workspace settings"
|
|
96
|
+
// scored the definition line at 0 and deleted it. The cap was motivated by an
|
|
97
|
+
// IDF-dilution worry that was never measured. If dilution is real,
|
|
98
|
+
// bench/retrieval-eval.js is where it should show up.
|
|
99
|
+
|
|
100
|
+
// Split one raw token into its component words. Returns [] when the token has
|
|
101
|
+
// no internal structure, so an already-atomic `config` is not emitted twice
|
|
102
|
+
// (which would inflate its term frequency and skew BM25).
|
|
103
|
+
function subTokens(rawTok) {
|
|
104
|
+
const spaced = rawTok
|
|
105
|
+
.replace(CAMEL_ACRONYM, '$1 $2')
|
|
106
|
+
.replace(CAMEL_LOWER_UPPER, '$1 $2')
|
|
107
|
+
.replace(/_+/g, ' ')
|
|
108
|
+
const parts = spaced.split(' ')
|
|
109
|
+
if (parts.length < 2) return []
|
|
110
|
+
const out = []
|
|
111
|
+
for (const part of parts) {
|
|
112
|
+
if (part.length < 2) continue
|
|
113
|
+
out.push(part.toLowerCase())
|
|
114
|
+
}
|
|
115
|
+
return out
|
|
116
|
+
}
|
|
22
117
|
|
|
23
118
|
export function tokenize(text) {
|
|
24
119
|
if (typeof text !== 'string' || text.length === 0) return []
|
|
25
|
-
let normalized = text.toLowerCase()
|
|
26
120
|
// Collapse intra-word separators. Run twice so chains like `a.b.c` fully
|
|
27
121
|
// collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
|
|
28
|
-
normalized =
|
|
122
|
+
let normalized = text.replace(INTRA_WORD_RE, '$1_$2')
|
|
29
123
|
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
30
124
|
const raw = normalized.split(SPLIT_RE)
|
|
31
125
|
const out = []
|
|
32
126
|
for (const tok of raw) {
|
|
33
|
-
if (tok.length
|
|
34
|
-
out.push(tok)
|
|
127
|
+
if (tok.length === 0) continue
|
|
128
|
+
if (tok.length >= 2) out.push(tok.toLowerCase())
|
|
129
|
+
for (const part of subTokens(tok)) out.push(part)
|
|
35
130
|
}
|
|
36
131
|
return out
|
|
37
132
|
}
|
|
@@ -39,60 +134,76 @@ export function tokenize(text) {
|
|
|
39
134
|
// --- BM25 ---
|
|
40
135
|
// Standard Okapi BM25. Each document is a line; query is the user's last
|
|
41
136
|
// message. Returns Float64Array of scores aligned with the documents array.
|
|
42
|
-
|
|
137
|
+
// `tokenizer` is injectable so bench/retrieval-eval.js can score the same
|
|
138
|
+
// corpus with the pre-subtoken tokenizer and report a real before/after.
|
|
139
|
+
export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75, tokenizer = tokenize } = {}) {
|
|
43
140
|
const N = documents.length
|
|
44
141
|
const scores = new Float64Array(N)
|
|
45
142
|
if (N === 0) return scores
|
|
46
143
|
|
|
47
|
-
const qTokens =
|
|
144
|
+
const qTokens = tokenizer(query)
|
|
48
145
|
if (qTokens.length === 0) return scores
|
|
49
146
|
|
|
50
|
-
//
|
|
51
|
-
|
|
147
|
+
// ONE pass over the documents (C7 + C8).
|
|
148
|
+
//
|
|
149
|
+
// C7: the old version kept `docTokens` — the full tokenization of every line
|
|
150
|
+
// — alive for the whole call, though only query terms are ever read from it.
|
|
151
|
+
// A 4.6MB body (60k lines) retained 47.4MB, roughly 10x amplification, on top
|
|
152
|
+
// of the original string, the split lines array and the parsed request body.
|
|
153
|
+
// The proxy serves concurrent requests with no per-request size cap on this
|
|
154
|
+
// stage, so that was an OOM path, not just memory pressure.
|
|
155
|
+
//
|
|
156
|
+
// C8: document frequency was computed as `docTokens[i].includes(term)` for
|
|
157
|
+
// every unique query term against every line — O(uniqQ x N x tokens/line) of
|
|
158
|
+
// linear scans. Both dimensions are hostile: N is the attacker-controlled
|
|
159
|
+
// line count, and the query is the user's last message, which routinely
|
|
160
|
+
// contains a pasted stack trace. Measured at N=20000: 70ms for a 10-word
|
|
161
|
+
// query, 163ms for 100 words, 676ms for 500.
|
|
162
|
+
//
|
|
163
|
+
// Now: tokenize each line once, keep only the per-line term frequencies for
|
|
164
|
+
// terms that appear in the query (empty for the vast majority of lines), and
|
|
165
|
+
// accumulate df in the same pass. Peak retention scales with the QUERY size,
|
|
166
|
+
// not the body size.
|
|
167
|
+
const qSet = new Set(qTokens)
|
|
52
168
|
const docLens = new Float64Array(N)
|
|
169
|
+
const docTf = new Array(N) // Map|null, only query terms, null when none present
|
|
170
|
+
const df = new Map()
|
|
53
171
|
let totalLen = 0
|
|
172
|
+
|
|
54
173
|
for (let i = 0; i < N; i++) {
|
|
55
|
-
const toks =
|
|
56
|
-
docTokens[i] = toks
|
|
174
|
+
const toks = tokenizer(documents[i])
|
|
57
175
|
docLens[i] = toks.length
|
|
58
176
|
totalLen += toks.length
|
|
177
|
+
let tf = null
|
|
178
|
+
for (const t of toks) {
|
|
179
|
+
if (!qSet.has(t)) continue
|
|
180
|
+
if (tf === null) tf = new Map()
|
|
181
|
+
tf.set(t, (tf.get(t) || 0) + 1)
|
|
182
|
+
}
|
|
183
|
+
docTf[i] = tf
|
|
184
|
+
if (tf !== null) for (const t of tf.keys()) df.set(t, (df.get(t) || 0) + 1)
|
|
185
|
+
// `toks` goes out of scope here and is collectable — nothing retains it.
|
|
59
186
|
}
|
|
187
|
+
|
|
60
188
|
const avgdl = N > 0 ? totalLen / N : 0
|
|
61
189
|
if (avgdl === 0) return scores
|
|
62
190
|
|
|
63
|
-
//
|
|
64
|
-
const uniqQ = Array.from(new Set(qTokens))
|
|
65
|
-
const df = new Map()
|
|
66
|
-
for (const term of uniqQ) {
|
|
67
|
-
let count = 0
|
|
68
|
-
for (let i = 0; i < N; i++) {
|
|
69
|
-
if (docTokens[i].includes(term)) count += 1
|
|
70
|
-
}
|
|
71
|
-
df.set(term, count)
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
// Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
|
|
191
|
+
// Okapi BM25 IDF with +1 smoothing.
|
|
75
192
|
const idf = new Map()
|
|
76
|
-
for (const term of
|
|
193
|
+
for (const term of qSet) {
|
|
77
194
|
const n = df.get(term) || 0
|
|
78
|
-
|
|
79
|
-
idf.set(term, v)
|
|
195
|
+
idf.set(term, Math.log(1 + (N - n + 0.5) / (n + 0.5)))
|
|
80
196
|
}
|
|
81
197
|
|
|
82
198
|
for (let i = 0; i < N; i++) {
|
|
199
|
+
const tf = docTf[i]
|
|
200
|
+
if (tf === null) continue // no query term on this line — scores 0, skip
|
|
83
201
|
const dl = docLens[i]
|
|
84
202
|
if (dl === 0) continue
|
|
85
|
-
// Term-frequency table for this doc (only for query terms we care about).
|
|
86
|
-
const tf = new Map()
|
|
87
|
-
const toks = docTokens[i]
|
|
88
|
-
for (const t of toks) {
|
|
89
|
-
if (!idf.has(t)) continue
|
|
90
|
-
tf.set(t, (tf.get(t) || 0) + 1)
|
|
91
|
-
}
|
|
92
203
|
let s = 0
|
|
93
|
-
|
|
94
|
-
|
|
95
|
-
|
|
204
|
+
// Iterate the terms actually PRESENT (usually 0-3) rather than every
|
|
205
|
+
// unique query term, so a long query costs nothing on unrelated lines.
|
|
206
|
+
for (const [term, f] of tf) {
|
|
96
207
|
const num = f * (k1 + 1)
|
|
97
208
|
const den = f + k1 * (1 - b + b * (dl / avgdl))
|
|
98
209
|
s += idf.get(term) * (num / den)
|
|
@@ -102,6 +213,55 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
|
102
213
|
return scores
|
|
103
214
|
}
|
|
104
215
|
|
|
216
|
+
// --- Rerank ---
|
|
217
|
+
// Raw BM25 treats every line as an equal bag of words. Two code-aware signals
|
|
218
|
+
// and one structural one fix its most visible failures.
|
|
219
|
+
//
|
|
220
|
+
// Note on scope: a path-based penalty (test files, .d.ts stubs) would be a
|
|
221
|
+
// no-op here. `bm25-trim` ranks lines WITHIN one tool_result, so a multiplier
|
|
222
|
+
// constant across the whole body cannot change any relative ordering. Path is
|
|
223
|
+
// used for language detection only.
|
|
224
|
+
|
|
225
|
+
// A line that DEFINES a query term beats a line that merely mentions it.
|
|
226
|
+
const DEF_RE = /\b(?:export\s+(?:default\s+)?)?(?:async\s+)?(?:function|class|def|const|let|var|type|interface|enum|struct|impl|trait|fn|func|module|namespace)\s+([A-Za-z_$][\w$]*)/
|
|
227
|
+
const COMMENT_RE = /^\s*(?:\/\/|#|\*|\/\*|--|<!--|;)/
|
|
228
|
+
const DEF_BOOST = 1.6
|
|
229
|
+
const COMMENT_PENALTY = 0.6
|
|
230
|
+
// A kept line lends a quarter of its score to each immediate neighbour, so the
|
|
231
|
+
// survivors read as contiguous code rather than confetti. Applied off the
|
|
232
|
+
// pre-coherence scores so boosts do not cascade down the file.
|
|
233
|
+
const COHERENCE = 0.25
|
|
234
|
+
|
|
235
|
+
export function rerankScores(scores, lines, queryTokens) {
|
|
236
|
+
const n = lines.length
|
|
237
|
+
const out = new Float64Array(n)
|
|
238
|
+
const qSet = new Set(queryTokens)
|
|
239
|
+
|
|
240
|
+
for (let i = 0; i < n; i++) {
|
|
241
|
+
let s = scores[i]
|
|
242
|
+
const line = lines[i]
|
|
243
|
+
if (typeof line !== 'string' || s === 0) { out[i] = s; continue }
|
|
244
|
+
|
|
245
|
+
const def = DEF_RE.exec(line)
|
|
246
|
+
if (def) {
|
|
247
|
+
for (const tok of tokenize(def[1])) {
|
|
248
|
+
if (qSet.has(tok)) { s *= DEF_BOOST; break }
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
if (COMMENT_RE.test(line)) s *= COMMENT_PENALTY
|
|
252
|
+
out[i] = s
|
|
253
|
+
}
|
|
254
|
+
|
|
255
|
+
if (n < 3) return out
|
|
256
|
+
const cohered = new Float64Array(n)
|
|
257
|
+
for (let i = 0; i < n; i++) {
|
|
258
|
+
const left = i > 0 ? out[i - 1] : 0
|
|
259
|
+
const right = i < n - 1 ? out[i + 1] : 0
|
|
260
|
+
cohered[i] = out[i] + COHERENCE * Math.max(left, right)
|
|
261
|
+
}
|
|
262
|
+
return cohered
|
|
263
|
+
}
|
|
264
|
+
|
|
105
265
|
// --- Line trimmer ---
|
|
106
266
|
// Keep top-K highest-scoring lines until `targetTokens` is consumed. First
|
|
107
267
|
// and last line are always preserved. Returns null when text is already
|
|
@@ -109,7 +269,16 @@ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
|
109
269
|
//
|
|
110
270
|
// Output shape:
|
|
111
271
|
// { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
|
|
112
|
-
|
|
272
|
+
// `tokenizer` and `rerank` exist for bench/retrieval-eval.js, which replays the
|
|
273
|
+
// same fixtures through the pre-subtoken tokenizer to produce a real
|
|
274
|
+
// before/after. Production callers should leave both at their defaults.
|
|
275
|
+
export function trimLinesByRelevance(text, query, {
|
|
276
|
+
targetTokens = 4096,
|
|
277
|
+
minLines = 10,
|
|
278
|
+
path = null,
|
|
279
|
+
tokenizer = tokenize,
|
|
280
|
+
rerank = true,
|
|
281
|
+
} = {}) {
|
|
113
282
|
if (typeof text !== 'string' || text.length === 0) return null
|
|
114
283
|
|
|
115
284
|
const originalTokens = countTokens(text)
|
|
@@ -119,21 +288,58 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
119
288
|
const N = lines.length
|
|
120
289
|
if (N < minLines) return null
|
|
121
290
|
|
|
122
|
-
// Per-line token costs
|
|
123
|
-
//
|
|
291
|
+
// Per-line token costs. `countTokens` carries a fixed ~21ms cost per call
|
|
292
|
+
// regardless of input length, so counting each line separately turns a
|
|
293
|
+
// 1000-line tool_result into a ~20-second stall on the proxy hot path — for
|
|
294
|
+
// a budget that is approximate by nature. Instead calibrate a chars-per-token
|
|
295
|
+
// ratio from the whole-body count already taken above, and estimate per line.
|
|
296
|
+
// Two tokenizer calls total instead of N + 2. The reported
|
|
297
|
+
// original/trimmed token counts stay exact.
|
|
298
|
+
// Clamped (C4). A single global ratio is gameable: token density spans ~200x
|
|
299
|
+
// across content (400 spaces = 133 chars/token; emoji = 0.67), so a body
|
|
300
|
+
// whose bulk is low-density padding and whose payload is dense text made
|
|
301
|
+
// every payload line look ~20x cheaper than it is. Measured: a 762KB body
|
|
302
|
+
// reported budgetUsed 4050 against targetTokens 4096 while emitting 31,869
|
|
303
|
+
// real tokens — 7.8x over budget, with the only guard being "smaller than
|
|
304
|
+
// the (attacker-sized) input".
|
|
305
|
+
//
|
|
306
|
+
// Real chars-per-token tops out near 4.6 on ordinary prose. Capping the
|
|
307
|
+
// assumed ratio at 4 means a line can never be priced below len/4, so dense
|
|
308
|
+
// content cannot be smuggled in under a padded average. Over-estimating is
|
|
309
|
+
// safe here — it only makes the trim more conservative.
|
|
310
|
+
const MAX_CHARS_PER_TOKEN = 4
|
|
311
|
+
const rawRatio = originalTokens > 0 ? text.length / originalTokens : MAX_CHARS_PER_TOKEN
|
|
312
|
+
const charsPerToken = Math.min(rawRatio, MAX_CHARS_PER_TOKEN)
|
|
124
313
|
const lineTokens = new Array(N)
|
|
125
314
|
for (let i = 0; i < N; i++) {
|
|
126
|
-
lineTokens[i] =
|
|
315
|
+
lineTokens[i] = Math.max(1, Math.ceil((lines[i].length + 1) / charsPerToken))
|
|
127
316
|
}
|
|
128
317
|
const newlineCost = Math.max(0, N - 1)
|
|
129
318
|
|
|
130
|
-
const
|
|
319
|
+
const raw = bm25Scores(lines, query, { tokenizer })
|
|
320
|
+
const scores = rerank ? rerankScores(raw, lines, tokenizer(query)) : raw
|
|
321
|
+
|
|
322
|
+
// Block structure, when the body parses as code. Keeping a line without its
|
|
323
|
+
// enclosing signature and closing brace emits syntactically broken output;
|
|
324
|
+
// `structure` lets selection pull those ancestors in as part of the line's
|
|
325
|
+
// cost. Null (prose, logs, JSON, unbalanced source) falls back to plain
|
|
326
|
+
// per-line selection, exactly as before.
|
|
327
|
+
const blocks = detectBlocks(lines, { path })
|
|
328
|
+
const structure = blocks ? buildStructure(lines, blocks) : null
|
|
131
329
|
|
|
132
330
|
// Always-keep set: first + last as anchors.
|
|
133
331
|
const keep = new Uint8Array(N)
|
|
134
332
|
keep[0] = 1
|
|
135
333
|
keep[N - 1] = 1
|
|
136
|
-
|
|
334
|
+
// Anchor the last NON-EMPTY line too. Tool output almost always ends with
|
|
335
|
+
// '\n', so lines[N-1] is '' — pinning only that empty terminator would let
|
|
336
|
+
// the real final line (often a trailing error or stack trace, exactly the
|
|
337
|
+
// "context anchor" this claims to preserve) be dropped as a candidate.
|
|
338
|
+
let lastReal = N - 1
|
|
339
|
+
while (lastReal > 0 && lines[lastReal] === '') lastReal--
|
|
340
|
+
keep[lastReal] = 1
|
|
341
|
+
let budgetUsed = 0
|
|
342
|
+
for (const idx of new Set([0, N - 1, lastReal])) budgetUsed += lineTokens[idx]
|
|
137
343
|
// Rough marker cost reserved per collapsed gap; we'll reconcile below.
|
|
138
344
|
const MARKER_RESERVE = 16
|
|
139
345
|
|
|
@@ -141,14 +347,104 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
141
347
|
// context preferred on ties). We pick greedily until budget is hit.
|
|
142
348
|
const candidates = []
|
|
143
349
|
for (let i = 1; i < N - 1; i++) {
|
|
350
|
+
if (keep[i]) continue // skip already-anchored lines (e.g. lastReal)
|
|
144
351
|
candidates.push({ i, score: scores[i], len: lineTokens[i] })
|
|
145
352
|
}
|
|
146
353
|
candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
|
|
147
354
|
|
|
355
|
+
// Ancestors a line drags in with it: every enclosing block header not yet
|
|
356
|
+
// kept, plus that block's closing line. Charged as part of the candidate's
|
|
357
|
+
// cost so the budget stays honest instead of overflowing at assembly time.
|
|
358
|
+
// `inPending` is a reusable membership mark, cleared via the `pending` list
|
|
359
|
+
// at the top of each call (C6). The old version used `pending.includes(p)`,
|
|
360
|
+
// O(depth) per probe, and never short-circuited on an already-kept ancestor
|
|
361
|
+
// chain — so every REJECTED candidate refilled `pending` to full nesting
|
|
362
|
+
// depth, giving O(candidates x depth^2). Depth is attacker-controlled and
|
|
363
|
+
// costs no extra body bytes: at 20k lines, depth 200 measured 251ms total
|
|
364
|
+
// while depth 800 measured 4512ms, of which 4443ms was this function.
|
|
365
|
+
const pending = []
|
|
366
|
+
const inPending = new Uint8Array(N)
|
|
367
|
+
function closureFor(i) {
|
|
368
|
+
for (const p of pending) inPending[p] = 0
|
|
369
|
+
pending.length = 0
|
|
370
|
+
if (!structure) return 0
|
|
371
|
+
let cost = 0
|
|
372
|
+
const mark = (line) => {
|
|
373
|
+
if (line < 0 || line >= N || keep[line] || inPending[line]) return false
|
|
374
|
+
inPending[line] = 1
|
|
375
|
+
pending.push(line)
|
|
376
|
+
cost += lineTokens[line]
|
|
377
|
+
return true
|
|
378
|
+
}
|
|
379
|
+
// A line that OPENS a block must drag in its own closer, or a kept
|
|
380
|
+
// signature is left hanging with no `}`. Ancestors alone do not cover
|
|
381
|
+
// this: the header is not inside its own body.
|
|
382
|
+
//
|
|
383
|
+
// closer/headerOf are LISTS — one line can open or close several blocks
|
|
384
|
+
// (`function f() { if (x) {`). Taking only the first would strand the
|
|
385
|
+
// others, which is the defect these lists exist to fix.
|
|
386
|
+
const ownClosers = structure.closer[i]
|
|
387
|
+
if (ownClosers) for (const c of ownClosers) if (c !== i) mark(c)
|
|
388
|
+
|
|
389
|
+
// Conversely, a closing line's own headers are not ancestors either — the
|
|
390
|
+
// body sits between them — so seed the walk with them when the candidate
|
|
391
|
+
// IS a closer. Continue from the outermost of them; the inner ones are its
|
|
392
|
+
// descendants and get pulled in via their own closer lists.
|
|
393
|
+
const ownHeaders = structure.headerOf[i]
|
|
394
|
+
let p = structure.parent[i]
|
|
395
|
+
if (ownHeaders && ownHeaders.length > 0) {
|
|
396
|
+
let outermost = ownHeaders[0]
|
|
397
|
+
for (const h of ownHeaders) {
|
|
398
|
+
mark(h)
|
|
399
|
+
if (h < outermost) outermost = h
|
|
400
|
+
}
|
|
401
|
+
p = structure.parent[outermost]
|
|
402
|
+
}
|
|
403
|
+
|
|
404
|
+
while (p >= 0) {
|
|
405
|
+
// Short-circuit: if this ancestor is already kept, everything above it
|
|
406
|
+
// was kept with it, so the rest of the chain is redundant. This is what
|
|
407
|
+
// turns the rejected-candidate case from O(depth) into O(1).
|
|
408
|
+
if (keep[p]) break
|
|
409
|
+
if (!mark(p)) break // already pending — the rest of the chain is too
|
|
410
|
+
const closers = structure.closer[p]
|
|
411
|
+
// closer[] is a LIST since the C2 fix. Reading it as a scalar silently
|
|
412
|
+
// worked for single-closer blocks (`[3]` coerces to "3") and silently
|
|
413
|
+
// did nothing for multi-closer ones (`[2,3]` -> NaN).
|
|
414
|
+
if (closers) for (const c of closers) if (c !== i) mark(c)
|
|
415
|
+
p = structure.parent[p]
|
|
416
|
+
}
|
|
417
|
+
return cost
|
|
418
|
+
}
|
|
419
|
+
|
|
148
420
|
for (const c of candidates) {
|
|
149
|
-
if (
|
|
421
|
+
if (keep[c.i]) continue // already pulled in as an ancestor
|
|
422
|
+
const closureCost = closureFor(c.i)
|
|
423
|
+
if (budgetUsed + c.len + closureCost > targetTokens) continue
|
|
424
|
+
for (const a of pending) keep[a] = 1
|
|
150
425
|
keep[c.i] = 1
|
|
151
|
-
budgetUsed += c.len
|
|
426
|
+
budgetUsed += c.len + closureCost
|
|
427
|
+
}
|
|
428
|
+
|
|
429
|
+
// Blank lines cost zero tokens, so greedy selection always keeps them. Left
|
|
430
|
+
// alone they punch holes through dropped runs, and every hole costs another
|
|
431
|
+
// ~12-token marker — enough to overshoot the budget several times over on a
|
|
432
|
+
// fragmented body. A blank RUN stranded between two dropped lines carries
|
|
433
|
+
// nothing, so fold the whole run in.
|
|
434
|
+
//
|
|
435
|
+
// Must operate on runs, not single lines. An earlier version tested only
|
|
436
|
+
// `keep[i-1]` and `keep[i+1]`, which never fires on `\n\n` — the most common
|
|
437
|
+
// blank shape in source — because each blank sees the other as kept. Measured
|
|
438
|
+
// 72 of 148 surviving lines blank before this fix. Snapshotting `keep` does
|
|
439
|
+
// not help either; the run has to be considered as a unit.
|
|
440
|
+
for (let i = 1; i < N - 1; i++) {
|
|
441
|
+
if (!keep[i] || lines[i].trim() !== '') continue
|
|
442
|
+
let j = i
|
|
443
|
+
while (j < N - 1 && keep[j] && lines[j].trim() === '') j += 1
|
|
444
|
+
if (!keep[i - 1] && j < N && !keep[j]) {
|
|
445
|
+
for (let k = i; k < j; k += 1) keep[k] = 0
|
|
446
|
+
}
|
|
447
|
+
i = j - 1
|
|
152
448
|
}
|
|
153
449
|
|
|
154
450
|
// If even the anchors + marker exceed budget, still proceed — we need to
|
|
@@ -171,7 +467,10 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
171
467
|
let chars = 0
|
|
172
468
|
while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
|
|
173
469
|
const runLen = j - i
|
|
174
|
-
|
|
470
|
+
// Indent the marker to the dropped body's own depth, so a collapsed
|
|
471
|
+
// function body reads as part of the block it replaced.
|
|
472
|
+
const pad = lines[i].slice(0, indentWidth(lines[i]))
|
|
473
|
+
out.push(`${pad}[... ${runLen} lines omitted, ${chars} chars ...]`)
|
|
175
474
|
i = j
|
|
176
475
|
}
|
|
177
476
|
|