@sliday/tamp 0.5.9 → 0.7.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -39,6 +39,7 @@ Auto-detects API format, compresses tool output, forwards upstream. Error result
39
39
 
40
40
  | Stage | What |
41
41
  |-------|------|
42
+ | `cmd-strip` | Strip progress bars and spinners from command output (lossless) |
42
43
  | `minify` | Strip JSON whitespace |
43
44
  | `toon` | Columnar array encoding |
44
45
  | `strip-lines` | Remove line-number prefixes |
package/bin/lifecycle.js CHANGED
@@ -1,4 +1,4 @@
1
- import { readFileSync, writeFileSync, unlinkSync, mkdirSync } from 'node:fs'
1
+ import { readFileSync, writeFileSync, unlinkSync, mkdirSync, realpathSync, existsSync } from 'node:fs'
2
2
  import { join, dirname } from 'node:path'
3
3
  import http from 'node:http'
4
4
  import { CONFIG_PATH } from '../config.js'
@@ -9,16 +9,28 @@ export function pidFilePath(port) {
9
9
  return join(TAMP_DIR, `tamp-${port}.pid`)
10
10
  }
11
11
 
12
+ // Symlink-safe resolver. Returns the real filesystem path for an existing
13
+ // PID file, or the original path when the file doesn't exist yet (so write
14
+ // operations land on the expected path). This prevents a symlink-swap
15
+ // attacker from redirecting our read/write to a privileged file.
16
+ function resolvePidPath(port) {
17
+ const p = pidFilePath(port)
18
+ try {
19
+ if (existsSync(p)) return realpathSync(p)
20
+ } catch { /* fall through to raw path */ }
21
+ return p
22
+ }
23
+
12
24
  export function writePidFile(port) {
13
25
  mkdirSync(TAMP_DIR, { recursive: true })
14
- const file = pidFilePath(port)
26
+ const file = resolvePidPath(port)
15
27
  writeFileSync(file, `${process.pid}\n${Date.now()}\n`)
16
28
  return file
17
29
  }
18
30
 
19
31
  export function readPidFile(port) {
20
32
  try {
21
- const [pidStr, startedStr] = readFileSync(pidFilePath(port), 'utf8').split('\n')
33
+ const [pidStr, startedStr] = readFileSync(resolvePidPath(port), 'utf8').split('\n')
22
34
  const pid = Number(pidStr)
23
35
  if (!pid) return null
24
36
  return { pid, startedAt: Number(startedStr) || null }
@@ -26,7 +38,7 @@ export function readPidFile(port) {
26
38
  }
27
39
 
28
40
  export function clearPidFile(port) {
29
- try { unlinkSync(pidFilePath(port)) } catch {}
41
+ try { unlinkSync(resolvePidPath(port)) } catch {}
30
42
  }
31
43
 
32
44
  export function isProcessAlive(pid) {
package/compress.js CHANGED
@@ -4,9 +4,13 @@ import { encode } from '@toon-format/toon'
4
4
  import { countTokens } from '@anthropic-ai/tokenizer'
5
5
  import { createPatch } from 'diff'
6
6
  import { tryParseJSON, classifyContent, stripLineNumbers } from './detect.js'
7
+ import { rewriteCommandOutput } from './lib/rewriters/index.js'
7
8
  import { anthropic } from './providers.js'
8
9
  import { graphDeduplicateTargets } from './session-graph.js'
9
- import { detectTaskType, generateOutputRules } from './lib/rules-generator.js'
10
+ import { detectTaskType, generateOutputRules, isDangerousTask } from './lib/rules-generator.js'
11
+ import { extractTargetPaths } from './lib/path-extract.js'
12
+ import { summarize, rehydrateReferences } from './lib/disclosure.js'
13
+ import { trimLinesByRelevance } from './lib/bm25.js'
10
14
 
11
15
  const cache = new Map()
12
16
  const MAX_CACHE = 500
@@ -77,6 +81,101 @@ function quickSimilarity(a, b) {
77
81
  return union > 0 ? intersection / union : 0
78
82
  }
79
83
 
84
+ // --- Stage: read-diff ---
85
+ // Session-scoped unified diff for re-reads. For each target whose file path
86
+ // can be confidently attributed (via the preceding tool_use), compare against
87
+ // the last-seen body for the same (session, path) pair. If the patch is small
88
+ // enough, substitute it for the body. Always cache the fresh text afterwards.
89
+ function readDiffTargets(targets, body, sessionKey, readCache) {
90
+ if (!readCache || !sessionKey) return
91
+ const paths = extractTargetPaths(body, targets)
92
+ for (let i = 0; i < targets.length; i++) {
93
+ const target = targets[i]
94
+ const path = paths[i]
95
+ if (!path) continue
96
+ if (target.skip || target.dedup || target.diffed || target.compressed) {
97
+ // Even for already-compressed targets, refresh the cache with the
98
+ // original text so the next request can diff against the latest copy.
99
+ if (typeof target.text === 'string') readCache.put(sessionKey, path, target.text)
100
+ continue
101
+ }
102
+ if (typeof target.text !== 'string') continue
103
+
104
+ const prior = readCache.get(sessionKey, path)
105
+ if (prior && prior !== target.text) {
106
+ const patch = createPatch(path, prior, target.text, '', '')
107
+ if (patch.length < target.text.length * 0.5) {
108
+ target.compressed = `[read-diff from prior read of ${path}]:\n${patch}`
109
+ target.readDiffed = true
110
+ }
111
+ }
112
+ readCache.put(sessionKey, path, target.text)
113
+ }
114
+ }
115
+
116
+ // --- Stage: disclosure (Phase 5) ---
117
+ // For tool_result bodies >= 32 KB that haven't been handled by earlier stages,
118
+ // store the full body in br-cache and replace it with a 3-tier summary block.
119
+ // Bypassed when the latest user message classifies as a dangerous task.
120
+ function discloseTargets(targets, body, provider, brCache) {
121
+ if (!brCache || typeof brCache.put !== 'function') return 0
122
+ if (!provider || typeof provider.getLastUserText !== 'function') return 0
123
+ const userText = provider.getLastUserText(body) || ''
124
+ if (isDangerousTask(userText)) return 0
125
+
126
+ let disclosed = 0
127
+ for (const target of targets) {
128
+ if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.compressed) continue
129
+ if (typeof target.text !== 'string') continue
130
+ const byteLen = Buffer.byteLength(target.text, 'utf8')
131
+ if (byteLen < 32 * 1024) continue
132
+
133
+ const put = brCache.put(target.text)
134
+ if (!put) continue
135
+ const result = summarize(target.text, { hash: put.hash })
136
+ if (!result) continue
137
+ target.compressed = result.summary
138
+ target.disclosed = true
139
+ target.disclosedHash = result.hash
140
+ target.disclosedOriginalBytes = result.originalBytes
141
+ disclosed += 1
142
+ }
143
+ return disclosed
144
+ }
145
+
146
+ // --- Stage: bm25-trim (Phase 6) ---
147
+ // For tool_result bodies > 64 KB that survived earlier stages (including
148
+ // disclosure), rank lines by BM25 against the latest user message and drop
149
+ // low-relevance lines past a token budget. Bypassed when the latest user
150
+ // message classifies as a dangerous task — full fidelity is non-negotiable
151
+ // for security/debug/refactor/test flows.
152
+ function bm25TrimTargets(targets, body, provider) {
153
+ if (!provider || typeof provider.getLastUserText !== 'function') return 0
154
+ const userText = provider.getLastUserText(body) || ''
155
+ if (!userText) return 0
156
+ if (isDangerousTask(userText)) return 0
157
+
158
+ let trimmed = 0
159
+ for (const target of targets) {
160
+ if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.compressed) continue
161
+ if (typeof target.text !== 'string') continue
162
+ const byteLen = Buffer.byteLength(target.text, 'utf8')
163
+ if (byteLen <= 64 * 1024) continue
164
+
165
+ const result = trimLinesByRelevance(target.text, userText)
166
+ if (!result) continue
167
+ if (result.text.length >= target.text.length) continue
168
+ target.compressed = result.text
169
+ target.bm25Trimmed = true
170
+ target.bm25KeptLines = result.keptLines
171
+ target.bm25DroppedLines = result.droppedLines
172
+ target.bm25OriginalTokens = result.originalTokens
173
+ target.bm25TrimmedTokens = result.trimmedTokens
174
+ trimmed += 1
175
+ }
176
+ return trimmed
177
+ }
178
+
80
179
  // --- Stage: prune ---
81
180
  const PRUNE_KEYS = new Set(['integrity', 'shasum', '_id', '_from', '_resolved', '_integrity', '_nodeVersion', '_npmVersion', '_phantomChildren', '_requiredBy'])
82
181
 
@@ -218,9 +317,19 @@ export function compressText(text, config) {
218
317
 
219
318
  if (cls === 'text') {
220
319
  let processed = text
320
+ let cmdStripRewriter = null
321
+ let cmdStripLen = text.length
322
+ if (config.stages.includes('cmd-strip')) {
323
+ const out = rewriteCommandOutput(processed)
324
+ if (out.rewriter && out.text.length < processed.length) {
325
+ processed = out.text
326
+ cmdStripRewriter = out.rewriter
327
+ cmdStripLen = processed.length
328
+ }
329
+ }
221
330
  if (config.stages.includes('strip-lines')) {
222
- const stripped = stripLineNumbers(text)
223
- if (stripped !== text) processed = stripped
331
+ const stripped = stripLineNumbers(processed)
332
+ if (stripped !== processed) processed = stripped
224
333
  }
225
334
  if (config.stages.includes('whitespace')) {
226
335
  processed = normalizeWhitespace(processed)
@@ -229,7 +338,12 @@ export function compressText(text, config) {
229
338
  processed = stripComments(processed)
230
339
  }
231
340
  if (processed.length < text.length * 0.9) {
232
- return { text: processed, method: 'normalize', originalLen: text.length, compressedLen: processed.length, originalTokens: countTokens(text), compressedTokens: countTokens(processed) }
341
+ // If cmd-strip produced the savings and downstream stages didn't improve it further,
342
+ // surface the rewriter identifier in the method name so stats are legible.
343
+ const method = (cmdStripRewriter && processed.length === cmdStripLen)
344
+ ? `cmd-strip:${cmdStripRewriter}`
345
+ : 'normalize'
346
+ return { text: processed, method, originalLen: text.length, compressedLen: processed.length, originalTokens: countTokens(text), compressedTokens: countTokens(processed) }
233
347
  }
234
348
  if (config.stages.includes('llmlingua') && config.llmLinguaUrl) {
235
349
  return { async: true, asyncMethod: 'llmlingua', text: processed, cls }
@@ -440,7 +554,7 @@ function maybeInjectOutputHint(body, config, provider) {
440
554
  if (!userText) return null
441
555
 
442
556
  const taskType = config.autoDetectTaskType === false ? 'complex' : detectTaskType(userText)
443
- const rules = generateOutputRules(mode, taskType)
557
+ const rules = generateOutputRules(mode, taskType, config.agent)
444
558
  if (!rules) return null
445
559
 
446
560
  // Keep the injected hint compact and clearly marked so the model knows
@@ -451,10 +565,24 @@ function maybeInjectOutputHint(body, config, provider) {
451
565
  }
452
566
 
453
567
  export async function compressRequest(body, config, provider) {
568
+ // Phase 5: before any stage runs, rehydrate <tamp-ref:v1:...> markers the
569
+ // model quoted back from a prior turn. Must happen first so downstream
570
+ // stages see the expanded body if relevant.
571
+ let disclosureStats = null
572
+ if (config.stages?.includes('disclosure') && config.brCache) {
573
+ const rh = rehydrateReferences(body, provider, config.brCache)
574
+ if (rh.rehydrated || rh.missed) disclosureStats = { rehydrated: rh.rehydrated, missed: rh.missed, disclosed: 0 }
575
+ }
576
+
454
577
  const outputHint = maybeInjectOutputHint(body, config, provider)
455
578
 
456
579
  const targets = provider.extract(body, { ...config, cacheSafe: config.cacheSafe !== false })
457
580
 
581
+ // Read-diff: session-scoped unified diff for re-reads (lossless, opt-in)
582
+ if (config.stages.includes('read-diff') && config.readCache && config.sessionKey && provider.name === 'anthropic') {
583
+ readDiffTargets(targets, body, config.sessionKey, config.readCache)
584
+ }
585
+
458
586
  // Dedup: replace identical blocks with reference markers
459
587
  if (config.stages.includes('dedup')) deduplicateTargets(targets)
460
588
 
@@ -466,6 +594,20 @@ export async function compressRequest(body, config, provider) {
466
594
  graphDeduplicateTargets(targets, config.sessionBucket)
467
595
  }
468
596
 
597
+ // Disclosure: progressive 3-tier summary for huge tool_results (opt-in, lossy)
598
+ if (config.stages.includes('disclosure') && config.brCache && provider.name === 'anthropic') {
599
+ const n = discloseTargets(targets, body, provider, config.brCache)
600
+ if (n > 0) {
601
+ disclosureStats = disclosureStats || { rehydrated: 0, missed: 0, disclosed: 0 }
602
+ disclosureStats.disclosed += n
603
+ }
604
+ }
605
+
606
+ // BM25-trim: query-aware line drop for huge tool_results (opt-in, lossy)
607
+ if (config.stages.includes('bm25-trim')) {
608
+ bm25TrimTargets(targets, body, provider)
609
+ }
610
+
469
611
  const stats = []
470
612
  for (const target of targets) {
471
613
  if (target.skip) { stats.push({ index: target.index, skipped: target.skip }); continue }
@@ -479,10 +621,22 @@ export async function compressRequest(body, config, provider) {
479
621
  stats.push({ index: target.index, method: 'diff', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
480
622
  continue
481
623
  }
624
+ if (target.readDiffed) {
625
+ stats.push({ index: target.index, method: 'read-diff', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
626
+ continue
627
+ }
482
628
  if (target.graphed) {
483
629
  stats.push({ index: target.index, method: 'graph', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
484
630
  continue
485
631
  }
632
+ if (target.disclosed) {
633
+ stats.push({ index: target.index, method: 'disclosure', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
634
+ continue
635
+ }
636
+ if (target.bm25Trimmed) {
637
+ stats.push({ index: target.index, method: 'bm25-trim', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
638
+ continue
639
+ }
486
640
 
487
641
  const result = await compressBlock(target.text, config)
488
642
  if (result) {
@@ -493,7 +647,7 @@ export async function compressRequest(body, config, provider) {
493
647
  }
494
648
  }
495
649
  provider.apply(body, targets)
496
- return { body, stats, targetCount: targets.length, outputHint }
650
+ return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats }
497
651
  }
498
652
 
499
653
  export async function compressMessages(body, config) {
package/config.js CHANGED
@@ -3,6 +3,8 @@ import { homedir } from 'node:os'
3
3
  import { join } from 'node:path'
4
4
  import { DEFAULT_STAGES, VERSION, COMPRESSION_PRESETS } from './metadata.js'
5
5
 
6
+ const VALID_OUTPUT_MODES = new Set(['off', 'conservative', 'balanced', 'aggressive'])
7
+
6
8
  function parseBoolean(value, defaultValue) {
7
9
  if (value === undefined) return defaultValue
8
10
  if (value === 'true') return true
@@ -63,6 +65,15 @@ export function loadConfig(env = process.env) {
63
65
  // Resolve preset to stages (or use explicit stages if provided)
64
66
  const stages = resolvePreset(presetName, explicitStages)
65
67
 
68
+ // Output mode precedence (highest wins):
69
+ // 1. TAMP_OUTPUT_MODE — explicit per-session override
70
+ // 2. TAMP_OUTPUT_DEFAULT — env-level default seed (v1.5.0 parity)
71
+ // 3. 'off' — Caveman disabled unless opted in
72
+ const explicitOutputMode = get('TAMP_OUTPUT_MODE')
73
+ const defaultOutputMode = get('TAMP_OUTPUT_DEFAULT')
74
+ let outputMode = explicitOutputMode || defaultOutputMode || 'off'
75
+ if (!VALID_OUTPUT_MODES.has(outputMode)) outputMode = 'off'
76
+
66
77
  return Object.freeze({
67
78
  version: VERSION,
68
79
  port: parseInt(get('TAMP_PORT'), 10) || 7778,
@@ -76,7 +87,9 @@ export function loadConfig(env = process.env) {
76
87
  minSize: parseInt(get('TAMP_MIN_SIZE'), 10) || 200,
77
88
  stages,
78
89
  preset: presetName,
79
- outputMode: get('TAMP_OUTPUT_MODE') || 'off',
90
+ outputMode,
91
+ outputModeDefault: defaultOutputMode && VALID_OUTPUT_MODES.has(defaultOutputMode) ? defaultOutputMode : null,
92
+ agent: get('TAMP_AGENT') || null,
80
93
  autoDetectTaskType: parseBoolean(get('TAMP_AUTO_DETECT_TASK_TYPE'), true),
81
94
  log: get('TAMP_LOG') !== 'false',
82
95
  logFile: get('TAMP_LOG_FILE') || null,
@@ -109,12 +122,20 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
109
122
  # Explicit stages (overrides preset if set)
110
123
  # TAMP_STAGES=minify,toon,strip-lines,whitespace,llmlingua,dedup,diff,prune
111
124
 
112
- # Output compression mode (conservative | balanced | aggressive)
125
+ # Output compression mode (conservative | balanced | aggressive | off)
113
126
  # TAMP_OUTPUT_MODE=balanced
114
127
 
128
+ # Env-level default for output mode (used when TAMP_OUTPUT_MODE is unset).
129
+ # Valid: off | conservative | balanced | aggressive
130
+ # TAMP_OUTPUT_DEFAULT=off
131
+
115
132
  # Auto-detect task type for output compression (true | false)
116
133
  # TAMP_AUTO_DETECT_TASK_TYPE=true
117
134
 
135
+ # Agent identifier for per-agent output rule overrides
136
+ # (e.g. codex, cursor, cline, aider, claude-code)
137
+ # TAMP_AGENT=
138
+
118
139
  # TAMP_MIN_SIZE=200
119
140
  # TAMP_LOG=true
120
141
  # TAMP_LOG_FILE=
package/detect.js CHANGED
@@ -1,3 +1,5 @@
1
+ import { rewriteCommandOutput } from './lib/rewriters/index.js'
2
+
1
3
  const LINE_NUM_RE = /^ *\d+[\t→]/
2
4
 
3
5
  export function stripLineNumbers(str) {
@@ -41,3 +43,11 @@ export function classifyContent(str) {
41
43
  if (str.length > 0) return 'text'
42
44
  return 'unknown'
43
45
  }
46
+
47
+ // Thin wrapper around the rewriters module. Returns the rewriter name
48
+ // that would fire for this text, or null if no rewriter matches.
49
+ // Side-effect-free: discards the rewritten text and savedBytes.
50
+ export function detectCommandOutput(text) {
51
+ const out = rewriteCommandOutput(text)
52
+ return out.rewriter || null
53
+ }
package/index.js CHANGED
@@ -8,6 +8,33 @@ import { compressRequest } from './compress.js'
8
8
  import { detectProvider } from './providers.js'
9
9
  import { createSession, formatRequestLog } from './stats.js'
10
10
  import { createSessionStore, deriveSessionKey } from './session-graph.js'
11
+ import { createReadCache } from './lib/read-cache.js'
12
+ import { createBrCache } from './lib/br-cache.js'
13
+ import { generateOutputRules, PER_AGENT_OVERRIDES } from './lib/rules-generator.js'
14
+
15
+ // Regex source strings used by the task classifier — surfaced via
16
+ // /caveman-help so operators can see exactly how their inputs are scored.
17
+ const CLASSIFIER_PATTERNS = Object.freeze({
18
+ safe: [
19
+ '^(add|remove|update|set|unset)\\s+(env\\s+var|environment variable|config|configuration|\\w+=)',
20
+ '^fix\\s+typo',
21
+ '^update\\s+(README|readme|documentation|docs)',
22
+ '^(install|uninstall).*package',
23
+ '^add\\s+\\w+\\s+(as\\s+)?dependency',
24
+ '^update\\s+version',
25
+ '^(format|lint|fmt)|run linter',
26
+ ],
27
+ dangerous: [
28
+ 'security|vulnerability|exploit|attack',
29
+ 'debug|investigate|diagnose|troubleshoot',
30
+ 'memory leak|performance|optimization|optimize',
31
+ '^refactor|^architecture|^design',
32
+ '^fix\\s+bug',
33
+ '^explain|^why|^how\\s+(does|work)',
34
+ 'test|spec|coverage',
35
+ ],
36
+ complex: 'default when neither safe nor dangerous patterns match',
37
+ })
11
38
 
12
39
  function buildUpstreamUrl(reqPath, base) {
13
40
  const parsed = new URL(base)
@@ -31,12 +58,17 @@ export function createProxy(overrides = {}) {
31
58
  config.upstreams = { anthropic: overrides.upstream, openai: overrides.upstream, 'openai-responses': overrides.upstream, gemini: overrides.upstream }
32
59
  }
33
60
  const session = createSession()
34
- const sessionStore = createSessionStore()
35
- return { config, session, sessionStore, server: _createServer(config, session, sessionStore) }
61
+ const brCache = (config.stages?.includes('graph') || config.stages?.includes('br-cache') || config.stages?.includes('disclosure'))
62
+ ? createBrCache()
63
+ : null
64
+ const sessionStore = createSessionStore({ brCache })
65
+ const readCache = createReadCache()
66
+ return { config, session, sessionStore, readCache, brCache, server: _createServer(config, session, sessionStore, readCache, brCache) }
36
67
  }
37
68
 
38
- function _createServer(config, session, sessionStore) {
69
+ function _createServer(config, session, sessionStore, readCache, brCache) {
39
70
  const log = createRequestLogger(config)
71
+ const disclosureTotals = { rehydrated: 0, missed: 0, disclosed: 0 }
40
72
 
41
73
  function openUpstream(method, upstreamUrl, headers, res) {
42
74
  const mod = upstreamUrl.protocol === 'https:' ? https : http
@@ -120,11 +152,31 @@ return http.createServer(async (req, res) => {
120
152
  charsOriginal: totals.totalOriginal,
121
153
  blocksCompressed: totals.compressionCount,
122
154
  },
155
+ disclosure: disclosureTotals,
123
156
  })
124
157
  res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) })
125
158
  return res.end(req.method === 'HEAD' ? undefined : body)
126
159
  }
127
160
 
161
+ // Caveman Mode diagnostic endpoint (v1.5.0 parity)
162
+ if (req.url === '/caveman-help' && (req.method === 'GET' || req.method === 'HEAD')) {
163
+ const sampleSafe = generateOutputRules(config.outputMode, 'safe', config.agent)
164
+ const sampleDangerous = generateOutputRules(config.outputMode, 'dangerous', config.agent)
165
+ const body = JSON.stringify({
166
+ mode: config.outputMode,
167
+ defaultFromEnv: config.outputModeDefault || null,
168
+ agent: config.agent || null,
169
+ taskClassifierRules: CLASSIFIER_PATTERNS,
170
+ sampleInjection: {
171
+ safe: sampleSafe || null,
172
+ dangerous: sampleDangerous || null,
173
+ },
174
+ perAgent: Object.keys(PER_AGENT_OVERRIDES),
175
+ })
176
+ res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': Buffer.byteLength(body) })
177
+ return res.end(req.method === 'HEAD' ? undefined : body)
178
+ }
179
+
128
180
  const provider = detectProvider(req.method, req.url)
129
181
 
130
182
  if (!provider) {
@@ -211,16 +263,24 @@ return http.createServer(async (req, res) => {
211
263
  try {
212
264
  const parsed = JSON.parse(textBody.toString('utf-8'))
213
265
 
214
- const sessionBucket = config.stages?.includes('graph')
215
- ? sessionStore.getBucket(deriveSessionKey(req.headers))
266
+ const sessionKey = (config.stages?.includes('graph') || config.stages?.includes('read-diff'))
267
+ ? deriveSessionKey(req.headers)
216
268
  : null
217
- const { body, stats } = await compressRequest(parsed, { ...config, sessionBucket }, provider)
269
+ const sessionBucket = config.stages?.includes('graph') && sessionKey
270
+ ? sessionStore.getBucket(sessionKey)
271
+ : null
272
+ const { body, stats, disclosure } = await compressRequest(parsed, { ...config, sessionBucket, sessionKey, readCache, brCache }, provider)
218
273
  finalBody = Buffer.from(JSON.stringify(body), 'utf-8')
219
274
  // Send uncompressed — simpler and content-length is accurate
220
275
  if (decompressed) delete headers['content-encoding']
221
276
 
222
277
  session.record(stats)
223
- log(formatRequestLog(stats, session, provider.name, req.url, textBody.length, config.tokenCost))
278
+ if (disclosure) {
279
+ disclosureTotals.rehydrated += disclosure.rehydrated || 0
280
+ disclosureTotals.missed += disclosure.missed || 0
281
+ disclosureTotals.disclosed += disclosure.disclosed || 0
282
+ }
283
+ log(formatRequestLog(stats, session, provider.name, req.url, textBody.length, config.tokenCost, sessionBucket))
224
284
  config.onCompress?.(stats, session.getTotals(), { provider: provider.name, url: req.url, bodySize: textBody.length })
225
285
  } catch (err) {
226
286
  log(`[tamp] passthrough (parse error): ${err.message}`)
package/lib/bm25.js ADDED
@@ -0,0 +1,194 @@
1
+ // Query-aware line ranker for huge tool_result bodies (Phase 6).
2
+ //
3
+ // Pure-JS BM25 scorer, zero deps. Given a query (last user message) and a
4
+ // large body, score each line against the query using Okapi BM25 and keep
5
+ // the highest-scoring lines until a token budget is satisfied. First and
6
+ // last lines are always preserved as context anchors, and collapsed ranges
7
+ // are replaced by a "[... N lines omitted, N chars ...]" marker so the
8
+ // model can reason about the elision.
9
+ //
10
+ // This primitive is reusable: any future stage that wants "keep the most
11
+ // relevant N lines" can import tokenize/bm25Scores directly.
12
+
13
+ import { countTokens } from '@anthropic-ai/tokenizer'
14
+
15
+ // --- Tokenizer ---
16
+ // Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
17
+ // `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
18
+ // separators to underscore before splitting on non-word characters. Single
19
+ // character tokens are dropped (too noisy for BM25 on short queries).
20
+ const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
21
+ const SPLIT_RE = /[^A-Za-z0-9_]+/
22
+
23
+ export function tokenize(text) {
24
+ if (typeof text !== 'string' || text.length === 0) return []
25
+ let normalized = text.toLowerCase()
26
+ // Collapse intra-word separators. Run twice so chains like `a.b.c` fully
27
+ // collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
28
+ normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
29
+ normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
30
+ const raw = normalized.split(SPLIT_RE)
31
+ const out = []
32
+ for (const tok of raw) {
33
+ if (tok.length < 2) continue
34
+ out.push(tok)
35
+ }
36
+ return out
37
+ }
38
+
39
+ // --- BM25 ---
40
+ // Standard Okapi BM25. Each document is a line; query is the user's last
41
+ // message. Returns Float64Array of scores aligned with the documents array.
42
+ export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
43
+ const N = documents.length
44
+ const scores = new Float64Array(N)
45
+ if (N === 0) return scores
46
+
47
+ const qTokens = tokenize(query)
48
+ if (qTokens.length === 0) return scores
49
+
50
+ // Tokenize docs + build doc stats.
51
+ const docTokens = new Array(N)
52
+ const docLens = new Float64Array(N)
53
+ let totalLen = 0
54
+ for (let i = 0; i < N; i++) {
55
+ const toks = tokenize(documents[i])
56
+ docTokens[i] = toks
57
+ docLens[i] = toks.length
58
+ totalLen += toks.length
59
+ }
60
+ const avgdl = N > 0 ? totalLen / N : 0
61
+ if (avgdl === 0) return scores
62
+
63
+ // Document-frequency per unique query term.
64
+ const uniqQ = Array.from(new Set(qTokens))
65
+ const df = new Map()
66
+ for (const term of uniqQ) {
67
+ let count = 0
68
+ for (let i = 0; i < N; i++) {
69
+ if (docTokens[i].includes(term)) count += 1
70
+ }
71
+ df.set(term, count)
72
+ }
73
+
74
+ // Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
75
+ const idf = new Map()
76
+ for (const term of uniqQ) {
77
+ const n = df.get(term) || 0
78
+ const v = Math.log(1 + (N - n + 0.5) / (n + 0.5))
79
+ idf.set(term, v)
80
+ }
81
+
82
+ for (let i = 0; i < N; i++) {
83
+ const dl = docLens[i]
84
+ if (dl === 0) continue
85
+ // Term-frequency table for this doc (only for query terms we care about).
86
+ const tf = new Map()
87
+ const toks = docTokens[i]
88
+ for (const t of toks) {
89
+ if (!idf.has(t)) continue
90
+ tf.set(t, (tf.get(t) || 0) + 1)
91
+ }
92
+ let s = 0
93
+ for (const term of uniqQ) {
94
+ const f = tf.get(term) || 0
95
+ if (f === 0) continue
96
+ const num = f * (k1 + 1)
97
+ const den = f + k1 * (1 - b + b * (dl / avgdl))
98
+ s += idf.get(term) * (num / den)
99
+ }
100
+ scores[i] = s
101
+ }
102
+ return scores
103
+ }
104
+
105
+ // --- Line trimmer ---
106
+ // Keep top-K highest-scoring lines until `targetTokens` is consumed. First
107
+ // and last line are always preserved. Returns null when text is already
108
+ // under budget, has fewer than `minLines` lines, or trimming didn't help.
109
+ //
110
+ // Output shape:
111
+ // { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
112
+ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLines = 10 } = {}) {
113
+ if (typeof text !== 'string' || text.length === 0) return null
114
+
115
+ const originalTokens = countTokens(text)
116
+ if (originalTokens <= targetTokens) return null
117
+
118
+ const lines = text.split('\n')
119
+ const N = lines.length
120
+ if (N < minLines) return null
121
+
122
+ // Per-line token costs (includes the trailing newline for all but the last
123
+ // line; we approximate by counting each line once and adding N-1 tokens).
124
+ const lineTokens = new Array(N)
125
+ for (let i = 0; i < N; i++) {
126
+ lineTokens[i] = countTokens(lines[i])
127
+ }
128
+ const newlineCost = Math.max(0, N - 1)
129
+
130
+ const scores = bm25Scores(lines, query)
131
+
132
+ // Always-keep set: first + last as anchors.
133
+ const keep = new Uint8Array(N)
134
+ keep[0] = 1
135
+ keep[N - 1] = 1
136
+ let budgetUsed = lineTokens[0] + (N > 1 ? lineTokens[N - 1] : 0)
137
+ // Rough marker cost reserved per collapsed gap; we'll reconcile below.
138
+ const MARKER_RESERVE = 16
139
+
140
+ // Sort remaining lines by score desc, then by position asc (stable earlier
141
+ // context preferred on ties). We pick greedily until budget is hit.
142
+ const candidates = []
143
+ for (let i = 1; i < N - 1; i++) {
144
+ candidates.push({ i, score: scores[i], len: lineTokens[i] })
145
+ }
146
+ candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
147
+
148
+ for (const c of candidates) {
149
+ if (budgetUsed + c.len > targetTokens) continue
150
+ keep[c.i] = 1
151
+ budgetUsed += c.len
152
+ }
153
+
154
+ // If even the anchors + marker exceed budget, still proceed — we need to
155
+ // produce SOME trim. But if nothing was dropped at all, bail.
156
+ let keptLines = 0
157
+ for (let i = 0; i < N; i++) keptLines += keep[i]
158
+ const droppedLines = N - keptLines
159
+ if (droppedLines === 0) return null
160
+
161
+ // Assemble output, collapsing contiguous dropped runs into drop markers.
162
+ const out = []
163
+ let i = 0
164
+ while (i < N) {
165
+ if (keep[i]) {
166
+ out.push(lines[i])
167
+ i += 1
168
+ continue
169
+ }
170
+ let j = i
171
+ let chars = 0
172
+ while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
173
+ const runLen = j - i
174
+ out.push(`[... ${runLen} lines omitted, ${chars} chars ...]`)
175
+ i = j
176
+ }
177
+
178
+ const trimmedText = out.join('\n')
179
+ const trimmedTokens = countTokens(trimmedText)
180
+ // Defensive: if trimming somehow grew the text, don't return a trim.
181
+ if (trimmedTokens >= originalTokens) return null
182
+
183
+ return {
184
+ text: trimmedText,
185
+ keptLines,
186
+ droppedLines,
187
+ originalTokens,
188
+ trimmedTokens,
189
+ originalLines: N,
190
+ budgetUsed,
191
+ newlineCost,
192
+ markerReserve: MARKER_RESERVE,
193
+ }
194
+ }