@sliday/tamp 0.5.9 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/lifecycle.js +16 -4
- package/compress.js +160 -6
- package/config.js +23 -2
- package/detect.js +10 -0
- package/index.js +67 -7
- package/lib/bm25.js +194 -0
- package/lib/br-cache.js +157 -0
- package/lib/disclosure.js +124 -0
- package/lib/path-extract.js +49 -0
- package/lib/read-cache.js +55 -0
- package/lib/rewriters/commands/cargo.js +50 -0
- package/lib/rewriters/commands/docker.js +54 -0
- package/lib/rewriters/commands/git.js +81 -0
- package/lib/rewriters/commands/jest.js +49 -0
- package/lib/rewriters/commands/npm.js +82 -0
- package/lib/rewriters/commands/pip.js +58 -0
- package/lib/rewriters/commands/prisma.js +50 -0
- package/lib/rewriters/commands/pytest.js +55 -0
- package/lib/rewriters/commands/terraform.js +49 -0
- package/lib/rewriters/commands/wget-curl.js +67 -0
- package/lib/rewriters/index.js +77 -0
- package/lib/rules-generator.js +45 -2
- package/metadata.js +37 -5
- package/package.json +2 -2
- package/providers.js +117 -0
- package/session-graph.js +36 -3
- package/stats.js +12 -1
package/README.md
CHANGED
|
@@ -39,6 +39,7 @@ Auto-detects API format, compresses tool output, forwards upstream. Error result
|
|
|
39
39
|
|
|
40
40
|
| Stage | What |
|
|
41
41
|
|-------|------|
|
|
42
|
+
| `cmd-strip` | Strip progress bars and spinners from command output (lossless) |
|
|
42
43
|
| `minify` | Strip JSON whitespace |
|
|
43
44
|
| `toon` | Columnar array encoding |
|
|
44
45
|
| `strip-lines` | Remove line-number prefixes |
|
package/bin/lifecycle.js
CHANGED
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
import { readFileSync, writeFileSync, unlinkSync, mkdirSync } from 'node:fs'
|
|
1
|
+
import { readFileSync, writeFileSync, unlinkSync, mkdirSync, realpathSync, existsSync } from 'node:fs'
|
|
2
2
|
import { join, dirname } from 'node:path'
|
|
3
3
|
import http from 'node:http'
|
|
4
4
|
import { CONFIG_PATH } from '../config.js'
|
|
@@ -9,16 +9,28 @@ export function pidFilePath(port) {
|
|
|
9
9
|
return join(TAMP_DIR, `tamp-${port}.pid`)
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
+
// Symlink-safe resolver. Returns the real filesystem path for an existing
|
|
13
|
+
// PID file, or the original path when the file doesn't exist yet (so write
|
|
14
|
+
// operations land on the expected path). This prevents a symlink-swap
|
|
15
|
+
// attacker from redirecting our read/write to a privileged file.
|
|
16
|
+
function resolvePidPath(port) {
|
|
17
|
+
const p = pidFilePath(port)
|
|
18
|
+
try {
|
|
19
|
+
if (existsSync(p)) return realpathSync(p)
|
|
20
|
+
} catch { /* fall through to raw path */ }
|
|
21
|
+
return p
|
|
22
|
+
}
|
|
23
|
+
|
|
12
24
|
export function writePidFile(port) {
|
|
13
25
|
mkdirSync(TAMP_DIR, { recursive: true })
|
|
14
|
-
const file =
|
|
26
|
+
const file = resolvePidPath(port)
|
|
15
27
|
writeFileSync(file, `${process.pid}\n${Date.now()}\n`)
|
|
16
28
|
return file
|
|
17
29
|
}
|
|
18
30
|
|
|
19
31
|
export function readPidFile(port) {
|
|
20
32
|
try {
|
|
21
|
-
const [pidStr, startedStr] = readFileSync(
|
|
33
|
+
const [pidStr, startedStr] = readFileSync(resolvePidPath(port), 'utf8').split('\n')
|
|
22
34
|
const pid = Number(pidStr)
|
|
23
35
|
if (!pid) return null
|
|
24
36
|
return { pid, startedAt: Number(startedStr) || null }
|
|
@@ -26,7 +38,7 @@ export function readPidFile(port) {
|
|
|
26
38
|
}
|
|
27
39
|
|
|
28
40
|
export function clearPidFile(port) {
|
|
29
|
-
try { unlinkSync(
|
|
41
|
+
try { unlinkSync(resolvePidPath(port)) } catch {}
|
|
30
42
|
}
|
|
31
43
|
|
|
32
44
|
export function isProcessAlive(pid) {
|
package/compress.js
CHANGED
|
@@ -4,9 +4,13 @@ import { encode } from '@toon-format/toon'
|
|
|
4
4
|
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
5
5
|
import { createPatch } from 'diff'
|
|
6
6
|
import { tryParseJSON, classifyContent, stripLineNumbers } from './detect.js'
|
|
7
|
+
import { rewriteCommandOutput } from './lib/rewriters/index.js'
|
|
7
8
|
import { anthropic } from './providers.js'
|
|
8
9
|
import { graphDeduplicateTargets } from './session-graph.js'
|
|
9
|
-
import { detectTaskType, generateOutputRules } from './lib/rules-generator.js'
|
|
10
|
+
import { detectTaskType, generateOutputRules, isDangerousTask } from './lib/rules-generator.js'
|
|
11
|
+
import { extractTargetPaths } from './lib/path-extract.js'
|
|
12
|
+
import { summarize, rehydrateReferences } from './lib/disclosure.js'
|
|
13
|
+
import { trimLinesByRelevance } from './lib/bm25.js'
|
|
10
14
|
|
|
11
15
|
const cache = new Map()
|
|
12
16
|
const MAX_CACHE = 500
|
|
@@ -77,6 +81,101 @@ function quickSimilarity(a, b) {
|
|
|
77
81
|
return union > 0 ? intersection / union : 0
|
|
78
82
|
}
|
|
79
83
|
|
|
84
|
+
// --- Stage: read-diff ---
|
|
85
|
+
// Session-scoped unified diff for re-reads. For each target whose file path
|
|
86
|
+
// can be confidently attributed (via the preceding tool_use), compare against
|
|
87
|
+
// the last-seen body for the same (session, path) pair. If the patch is small
|
|
88
|
+
// enough, substitute it for the body. Always cache the fresh text afterwards.
|
|
89
|
+
function readDiffTargets(targets, body, sessionKey, readCache) {
|
|
90
|
+
if (!readCache || !sessionKey) return
|
|
91
|
+
const paths = extractTargetPaths(body, targets)
|
|
92
|
+
for (let i = 0; i < targets.length; i++) {
|
|
93
|
+
const target = targets[i]
|
|
94
|
+
const path = paths[i]
|
|
95
|
+
if (!path) continue
|
|
96
|
+
if (target.skip || target.dedup || target.diffed || target.compressed) {
|
|
97
|
+
// Even for already-compressed targets, refresh the cache with the
|
|
98
|
+
// original text so the next request can diff against the latest copy.
|
|
99
|
+
if (typeof target.text === 'string') readCache.put(sessionKey, path, target.text)
|
|
100
|
+
continue
|
|
101
|
+
}
|
|
102
|
+
if (typeof target.text !== 'string') continue
|
|
103
|
+
|
|
104
|
+
const prior = readCache.get(sessionKey, path)
|
|
105
|
+
if (prior && prior !== target.text) {
|
|
106
|
+
const patch = createPatch(path, prior, target.text, '', '')
|
|
107
|
+
if (patch.length < target.text.length * 0.5) {
|
|
108
|
+
target.compressed = `[read-diff from prior read of ${path}]:\n${patch}`
|
|
109
|
+
target.readDiffed = true
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
readCache.put(sessionKey, path, target.text)
|
|
113
|
+
}
|
|
114
|
+
}
|
|
115
|
+
|
|
116
|
+
// --- Stage: disclosure (Phase 5) ---
|
|
117
|
+
// For tool_result bodies >= 32 KB that haven't been handled by earlier stages,
|
|
118
|
+
// store the full body in br-cache and replace it with a 3-tier summary block.
|
|
119
|
+
// Bypassed when the latest user message classifies as a dangerous task.
|
|
120
|
+
function discloseTargets(targets, body, provider, brCache) {
|
|
121
|
+
if (!brCache || typeof brCache.put !== 'function') return 0
|
|
122
|
+
if (!provider || typeof provider.getLastUserText !== 'function') return 0
|
|
123
|
+
const userText = provider.getLastUserText(body) || ''
|
|
124
|
+
if (isDangerousTask(userText)) return 0
|
|
125
|
+
|
|
126
|
+
let disclosed = 0
|
|
127
|
+
for (const target of targets) {
|
|
128
|
+
if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.compressed) continue
|
|
129
|
+
if (typeof target.text !== 'string') continue
|
|
130
|
+
const byteLen = Buffer.byteLength(target.text, 'utf8')
|
|
131
|
+
if (byteLen < 32 * 1024) continue
|
|
132
|
+
|
|
133
|
+
const put = brCache.put(target.text)
|
|
134
|
+
if (!put) continue
|
|
135
|
+
const result = summarize(target.text, { hash: put.hash })
|
|
136
|
+
if (!result) continue
|
|
137
|
+
target.compressed = result.summary
|
|
138
|
+
target.disclosed = true
|
|
139
|
+
target.disclosedHash = result.hash
|
|
140
|
+
target.disclosedOriginalBytes = result.originalBytes
|
|
141
|
+
disclosed += 1
|
|
142
|
+
}
|
|
143
|
+
return disclosed
|
|
144
|
+
}
|
|
145
|
+
|
|
146
|
+
// --- Stage: bm25-trim (Phase 6) ---
|
|
147
|
+
// For tool_result bodies > 64 KB that survived earlier stages (including
|
|
148
|
+
// disclosure), rank lines by BM25 against the latest user message and drop
|
|
149
|
+
// low-relevance lines past a token budget. Bypassed when the latest user
|
|
150
|
+
// message classifies as a dangerous task — full fidelity is non-negotiable
|
|
151
|
+
// for security/debug/refactor/test flows.
|
|
152
|
+
function bm25TrimTargets(targets, body, provider) {
|
|
153
|
+
if (!provider || typeof provider.getLastUserText !== 'function') return 0
|
|
154
|
+
const userText = provider.getLastUserText(body) || ''
|
|
155
|
+
if (!userText) return 0
|
|
156
|
+
if (isDangerousTask(userText)) return 0
|
|
157
|
+
|
|
158
|
+
let trimmed = 0
|
|
159
|
+
for (const target of targets) {
|
|
160
|
+
if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.compressed) continue
|
|
161
|
+
if (typeof target.text !== 'string') continue
|
|
162
|
+
const byteLen = Buffer.byteLength(target.text, 'utf8')
|
|
163
|
+
if (byteLen <= 64 * 1024) continue
|
|
164
|
+
|
|
165
|
+
const result = trimLinesByRelevance(target.text, userText)
|
|
166
|
+
if (!result) continue
|
|
167
|
+
if (result.text.length >= target.text.length) continue
|
|
168
|
+
target.compressed = result.text
|
|
169
|
+
target.bm25Trimmed = true
|
|
170
|
+
target.bm25KeptLines = result.keptLines
|
|
171
|
+
target.bm25DroppedLines = result.droppedLines
|
|
172
|
+
target.bm25OriginalTokens = result.originalTokens
|
|
173
|
+
target.bm25TrimmedTokens = result.trimmedTokens
|
|
174
|
+
trimmed += 1
|
|
175
|
+
}
|
|
176
|
+
return trimmed
|
|
177
|
+
}
|
|
178
|
+
|
|
80
179
|
// --- Stage: prune ---
|
|
81
180
|
const PRUNE_KEYS = new Set(['integrity', 'shasum', '_id', '_from', '_resolved', '_integrity', '_nodeVersion', '_npmVersion', '_phantomChildren', '_requiredBy'])
|
|
82
181
|
|
|
@@ -218,9 +317,19 @@ export function compressText(text, config) {
|
|
|
218
317
|
|
|
219
318
|
if (cls === 'text') {
|
|
220
319
|
let processed = text
|
|
320
|
+
let cmdStripRewriter = null
|
|
321
|
+
let cmdStripLen = text.length
|
|
322
|
+
if (config.stages.includes('cmd-strip')) {
|
|
323
|
+
const out = rewriteCommandOutput(processed)
|
|
324
|
+
if (out.rewriter && out.text.length < processed.length) {
|
|
325
|
+
processed = out.text
|
|
326
|
+
cmdStripRewriter = out.rewriter
|
|
327
|
+
cmdStripLen = processed.length
|
|
328
|
+
}
|
|
329
|
+
}
|
|
221
330
|
if (config.stages.includes('strip-lines')) {
|
|
222
|
-
const stripped = stripLineNumbers(
|
|
223
|
-
if (stripped !==
|
|
331
|
+
const stripped = stripLineNumbers(processed)
|
|
332
|
+
if (stripped !== processed) processed = stripped
|
|
224
333
|
}
|
|
225
334
|
if (config.stages.includes('whitespace')) {
|
|
226
335
|
processed = normalizeWhitespace(processed)
|
|
@@ -229,7 +338,12 @@ export function compressText(text, config) {
|
|
|
229
338
|
processed = stripComments(processed)
|
|
230
339
|
}
|
|
231
340
|
if (processed.length < text.length * 0.9) {
|
|
232
|
-
|
|
341
|
+
// If cmd-strip produced the savings and downstream stages didn't improve it further,
|
|
342
|
+
// surface the rewriter identifier in the method name so stats are legible.
|
|
343
|
+
const method = (cmdStripRewriter && processed.length === cmdStripLen)
|
|
344
|
+
? `cmd-strip:${cmdStripRewriter}`
|
|
345
|
+
: 'normalize'
|
|
346
|
+
return { text: processed, method, originalLen: text.length, compressedLen: processed.length, originalTokens: countTokens(text), compressedTokens: countTokens(processed) }
|
|
233
347
|
}
|
|
234
348
|
if (config.stages.includes('llmlingua') && config.llmLinguaUrl) {
|
|
235
349
|
return { async: true, asyncMethod: 'llmlingua', text: processed, cls }
|
|
@@ -440,7 +554,7 @@ function maybeInjectOutputHint(body, config, provider) {
|
|
|
440
554
|
if (!userText) return null
|
|
441
555
|
|
|
442
556
|
const taskType = config.autoDetectTaskType === false ? 'complex' : detectTaskType(userText)
|
|
443
|
-
const rules = generateOutputRules(mode, taskType)
|
|
557
|
+
const rules = generateOutputRules(mode, taskType, config.agent)
|
|
444
558
|
if (!rules) return null
|
|
445
559
|
|
|
446
560
|
// Keep the injected hint compact and clearly marked so the model knows
|
|
@@ -451,10 +565,24 @@ function maybeInjectOutputHint(body, config, provider) {
|
|
|
451
565
|
}
|
|
452
566
|
|
|
453
567
|
export async function compressRequest(body, config, provider) {
|
|
568
|
+
// Phase 5: before any stage runs, rehydrate <tamp-ref:v1:...> markers the
|
|
569
|
+
// model quoted back from a prior turn. Must happen first so downstream
|
|
570
|
+
// stages see the expanded body if relevant.
|
|
571
|
+
let disclosureStats = null
|
|
572
|
+
if (config.stages?.includes('disclosure') && config.brCache) {
|
|
573
|
+
const rh = rehydrateReferences(body, provider, config.brCache)
|
|
574
|
+
if (rh.rehydrated || rh.missed) disclosureStats = { rehydrated: rh.rehydrated, missed: rh.missed, disclosed: 0 }
|
|
575
|
+
}
|
|
576
|
+
|
|
454
577
|
const outputHint = maybeInjectOutputHint(body, config, provider)
|
|
455
578
|
|
|
456
579
|
const targets = provider.extract(body, { ...config, cacheSafe: config.cacheSafe !== false })
|
|
457
580
|
|
|
581
|
+
// Read-diff: session-scoped unified diff for re-reads (lossless, opt-in)
|
|
582
|
+
if (config.stages.includes('read-diff') && config.readCache && config.sessionKey && provider.name === 'anthropic') {
|
|
583
|
+
readDiffTargets(targets, body, config.sessionKey, config.readCache)
|
|
584
|
+
}
|
|
585
|
+
|
|
458
586
|
// Dedup: replace identical blocks with reference markers
|
|
459
587
|
if (config.stages.includes('dedup')) deduplicateTargets(targets)
|
|
460
588
|
|
|
@@ -466,6 +594,20 @@ export async function compressRequest(body, config, provider) {
|
|
|
466
594
|
graphDeduplicateTargets(targets, config.sessionBucket)
|
|
467
595
|
}
|
|
468
596
|
|
|
597
|
+
// Disclosure: progressive 3-tier summary for huge tool_results (opt-in, lossy)
|
|
598
|
+
if (config.stages.includes('disclosure') && config.brCache && provider.name === 'anthropic') {
|
|
599
|
+
const n = discloseTargets(targets, body, provider, config.brCache)
|
|
600
|
+
if (n > 0) {
|
|
601
|
+
disclosureStats = disclosureStats || { rehydrated: 0, missed: 0, disclosed: 0 }
|
|
602
|
+
disclosureStats.disclosed += n
|
|
603
|
+
}
|
|
604
|
+
}
|
|
605
|
+
|
|
606
|
+
// BM25-trim: query-aware line drop for huge tool_results (opt-in, lossy)
|
|
607
|
+
if (config.stages.includes('bm25-trim')) {
|
|
608
|
+
bm25TrimTargets(targets, body, provider)
|
|
609
|
+
}
|
|
610
|
+
|
|
469
611
|
const stats = []
|
|
470
612
|
for (const target of targets) {
|
|
471
613
|
if (target.skip) { stats.push({ index: target.index, skipped: target.skip }); continue }
|
|
@@ -479,10 +621,22 @@ export async function compressRequest(body, config, provider) {
|
|
|
479
621
|
stats.push({ index: target.index, method: 'diff', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
|
|
480
622
|
continue
|
|
481
623
|
}
|
|
624
|
+
if (target.readDiffed) {
|
|
625
|
+
stats.push({ index: target.index, method: 'read-diff', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
|
|
626
|
+
continue
|
|
627
|
+
}
|
|
482
628
|
if (target.graphed) {
|
|
483
629
|
stats.push({ index: target.index, method: 'graph', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
|
|
484
630
|
continue
|
|
485
631
|
}
|
|
632
|
+
if (target.disclosed) {
|
|
633
|
+
stats.push({ index: target.index, method: 'disclosure', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
|
|
634
|
+
continue
|
|
635
|
+
}
|
|
636
|
+
if (target.bm25Trimmed) {
|
|
637
|
+
stats.push({ index: target.index, method: 'bm25-trim', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
|
|
638
|
+
continue
|
|
639
|
+
}
|
|
486
640
|
|
|
487
641
|
const result = await compressBlock(target.text, config)
|
|
488
642
|
if (result) {
|
|
@@ -493,7 +647,7 @@ export async function compressRequest(body, config, provider) {
|
|
|
493
647
|
}
|
|
494
648
|
}
|
|
495
649
|
provider.apply(body, targets)
|
|
496
|
-
return { body, stats, targetCount: targets.length, outputHint }
|
|
650
|
+
return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats }
|
|
497
651
|
}
|
|
498
652
|
|
|
499
653
|
export async function compressMessages(body, config) {
|
package/config.js
CHANGED
|
@@ -3,6 +3,8 @@ import { homedir } from 'node:os'
|
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import { DEFAULT_STAGES, VERSION, COMPRESSION_PRESETS } from './metadata.js'
|
|
5
5
|
|
|
6
|
+
const VALID_OUTPUT_MODES = new Set(['off', 'conservative', 'balanced', 'aggressive'])
|
|
7
|
+
|
|
6
8
|
function parseBoolean(value, defaultValue) {
|
|
7
9
|
if (value === undefined) return defaultValue
|
|
8
10
|
if (value === 'true') return true
|
|
@@ -63,6 +65,15 @@ export function loadConfig(env = process.env) {
|
|
|
63
65
|
// Resolve preset to stages (or use explicit stages if provided)
|
|
64
66
|
const stages = resolvePreset(presetName, explicitStages)
|
|
65
67
|
|
|
68
|
+
// Output mode precedence (highest wins):
|
|
69
|
+
// 1. TAMP_OUTPUT_MODE — explicit per-session override
|
|
70
|
+
// 2. TAMP_OUTPUT_DEFAULT — env-level default seed (v1.5.0 parity)
|
|
71
|
+
// 3. 'off' — Caveman disabled unless opted in
|
|
72
|
+
const explicitOutputMode = get('TAMP_OUTPUT_MODE')
|
|
73
|
+
const defaultOutputMode = get('TAMP_OUTPUT_DEFAULT')
|
|
74
|
+
let outputMode = explicitOutputMode || defaultOutputMode || 'off'
|
|
75
|
+
if (!VALID_OUTPUT_MODES.has(outputMode)) outputMode = 'off'
|
|
76
|
+
|
|
66
77
|
return Object.freeze({
|
|
67
78
|
version: VERSION,
|
|
68
79
|
port: parseInt(get('TAMP_PORT'), 10) || 7778,
|
|
@@ -76,7 +87,9 @@ export function loadConfig(env = process.env) {
|
|
|
76
87
|
minSize: parseInt(get('TAMP_MIN_SIZE'), 10) || 200,
|
|
77
88
|
stages,
|
|
78
89
|
preset: presetName,
|
|
79
|
-
outputMode
|
|
90
|
+
outputMode,
|
|
91
|
+
outputModeDefault: defaultOutputMode && VALID_OUTPUT_MODES.has(defaultOutputMode) ? defaultOutputMode : null,
|
|
92
|
+
agent: get('TAMP_AGENT') || null,
|
|
80
93
|
autoDetectTaskType: parseBoolean(get('TAMP_AUTO_DETECT_TASK_TYPE'), true),
|
|
81
94
|
log: get('TAMP_LOG') !== 'false',
|
|
82
95
|
logFile: get('TAMP_LOG_FILE') || null,
|
|
@@ -109,12 +122,20 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
|
|
|
109
122
|
# Explicit stages (overrides preset if set)
|
|
110
123
|
# TAMP_STAGES=minify,toon,strip-lines,whitespace,llmlingua,dedup,diff,prune
|
|
111
124
|
|
|
112
|
-
# Output compression mode (conservative | balanced | aggressive)
|
|
125
|
+
# Output compression mode (conservative | balanced | aggressive | off)
|
|
113
126
|
# TAMP_OUTPUT_MODE=balanced
|
|
114
127
|
|
|
128
|
+
# Env-level default for output mode (used when TAMP_OUTPUT_MODE is unset).
|
|
129
|
+
# Valid: off | conservative | balanced | aggressive
|
|
130
|
+
# TAMP_OUTPUT_DEFAULT=off
|
|
131
|
+
|
|
115
132
|
# Auto-detect task type for output compression (true | false)
|
|
116
133
|
# TAMP_AUTO_DETECT_TASK_TYPE=true
|
|
117
134
|
|
|
135
|
+
# Agent identifier for per-agent output rule overrides
|
|
136
|
+
# (e.g. codex, cursor, cline, aider, claude-code)
|
|
137
|
+
# TAMP_AGENT=
|
|
138
|
+
|
|
118
139
|
# TAMP_MIN_SIZE=200
|
|
119
140
|
# TAMP_LOG=true
|
|
120
141
|
# TAMP_LOG_FILE=
|
package/detect.js
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import { rewriteCommandOutput } from './lib/rewriters/index.js'
|
|
2
|
+
|
|
1
3
|
const LINE_NUM_RE = /^ *\d+[\t→]/
|
|
2
4
|
|
|
3
5
|
export function stripLineNumbers(str) {
|
|
@@ -41,3 +43,11 @@ export function classifyContent(str) {
|
|
|
41
43
|
if (str.length > 0) return 'text'
|
|
42
44
|
return 'unknown'
|
|
43
45
|
}
|
|
46
|
+
|
|
47
|
+
// Thin wrapper around the rewriters module. Returns the rewriter name
|
|
48
|
+
// that would fire for this text, or null if no rewriter matches.
|
|
49
|
+
// Side-effect-free: discards the rewritten text and savedBytes.
|
|
50
|
+
export function detectCommandOutput(text) {
|
|
51
|
+
const out = rewriteCommandOutput(text)
|
|
52
|
+
return out.rewriter || null
|
|
53
|
+
}
|
package/index.js
CHANGED
|
@@ -8,6 +8,33 @@ import { compressRequest } from './compress.js'
|
|
|
8
8
|
import { detectProvider } from './providers.js'
|
|
9
9
|
import { createSession, formatRequestLog } from './stats.js'
|
|
10
10
|
import { createSessionStore, deriveSessionKey } from './session-graph.js'
|
|
11
|
+
import { createReadCache } from './lib/read-cache.js'
|
|
12
|
+
import { createBrCache } from './lib/br-cache.js'
|
|
13
|
+
import { generateOutputRules, PER_AGENT_OVERRIDES } from './lib/rules-generator.js'
|
|
14
|
+
|
|
15
|
+
// Regex source strings used by the task classifier — surfaced via
|
|
16
|
+
// /caveman-help so operators can see exactly how their inputs are scored.
|
|
17
|
+
const CLASSIFIER_PATTERNS = Object.freeze({
|
|
18
|
+
safe: [
|
|
19
|
+
'^(add|remove|update|set|unset)\\s+(env\\s+var|environment variable|config|configuration|\\w+=)',
|
|
20
|
+
'^fix\\s+typo',
|
|
21
|
+
'^update\\s+(README|readme|documentation|docs)',
|
|
22
|
+
'^(install|uninstall).*package',
|
|
23
|
+
'^add\\s+\\w+\\s+(as\\s+)?dependency',
|
|
24
|
+
'^update\\s+version',
|
|
25
|
+
'^(format|lint|fmt)|run linter',
|
|
26
|
+
],
|
|
27
|
+
dangerous: [
|
|
28
|
+
'security|vulnerability|exploit|attack',
|
|
29
|
+
'debug|investigate|diagnose|troubleshoot',
|
|
30
|
+
'memory leak|performance|optimization|optimize',
|
|
31
|
+
'^refactor|^architecture|^design',
|
|
32
|
+
'^fix\\s+bug',
|
|
33
|
+
'^explain|^why|^how\\s+(does|work)',
|
|
34
|
+
'test|spec|coverage',
|
|
35
|
+
],
|
|
36
|
+
complex: 'default when neither safe nor dangerous patterns match',
|
|
37
|
+
})
|
|
11
38
|
|
|
12
39
|
function buildUpstreamUrl(reqPath, base) {
|
|
13
40
|
const parsed = new URL(base)
|
|
@@ -31,12 +58,17 @@ export function createProxy(overrides = {}) {
|
|
|
31
58
|
config.upstreams = { anthropic: overrides.upstream, openai: overrides.upstream, 'openai-responses': overrides.upstream, gemini: overrides.upstream }
|
|
32
59
|
}
|
|
33
60
|
const session = createSession()
|
|
34
|
-
const
|
|
35
|
-
|
|
61
|
+
const brCache = (config.stages?.includes('graph') || config.stages?.includes('br-cache') || config.stages?.includes('disclosure'))
|
|
62
|
+
? createBrCache()
|
|
63
|
+
: null
|
|
64
|
+
const sessionStore = createSessionStore({ brCache })
|
|
65
|
+
const readCache = createReadCache()
|
|
66
|
+
return { config, session, sessionStore, readCache, brCache, server: _createServer(config, session, sessionStore, readCache, brCache) }
|
|
36
67
|
}
|
|
37
68
|
|
|
38
|
-
function _createServer(config, session, sessionStore) {
|
|
69
|
+
function _createServer(config, session, sessionStore, readCache, brCache) {
|
|
39
70
|
const log = createRequestLogger(config)
|
|
71
|
+
const disclosureTotals = { rehydrated: 0, missed: 0, disclosed: 0 }
|
|
40
72
|
|
|
41
73
|
function openUpstream(method, upstreamUrl, headers, res) {
|
|
42
74
|
const mod = upstreamUrl.protocol === 'https:' ? https : http
|
|
@@ -120,11 +152,31 @@ return http.createServer(async (req, res) => {
|
|
|
120
152
|
charsOriginal: totals.totalOriginal,
|
|
121
153
|
blocksCompressed: totals.compressionCount,
|
|
122
154
|
},
|
|
155
|
+
disclosure: disclosureTotals,
|
|
123
156
|
})
|
|
124
157
|
res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) })
|
|
125
158
|
return res.end(req.method === 'HEAD' ? undefined : body)
|
|
126
159
|
}
|
|
127
160
|
|
|
161
|
+
// Caveman Mode diagnostic endpoint (v1.5.0 parity)
|
|
162
|
+
if (req.url === '/caveman-help' && (req.method === 'GET' || req.method === 'HEAD')) {
|
|
163
|
+
const sampleSafe = generateOutputRules(config.outputMode, 'safe', config.agent)
|
|
164
|
+
const sampleDangerous = generateOutputRules(config.outputMode, 'dangerous', config.agent)
|
|
165
|
+
const body = JSON.stringify({
|
|
166
|
+
mode: config.outputMode,
|
|
167
|
+
defaultFromEnv: config.outputModeDefault || null,
|
|
168
|
+
agent: config.agent || null,
|
|
169
|
+
taskClassifierRules: CLASSIFIER_PATTERNS,
|
|
170
|
+
sampleInjection: {
|
|
171
|
+
safe: sampleSafe || null,
|
|
172
|
+
dangerous: sampleDangerous || null,
|
|
173
|
+
},
|
|
174
|
+
perAgent: Object.keys(PER_AGENT_OVERRIDES),
|
|
175
|
+
})
|
|
176
|
+
res.writeHead(200, { 'Content-Type': 'application/json; charset=utf-8', 'Content-Length': Buffer.byteLength(body) })
|
|
177
|
+
return res.end(req.method === 'HEAD' ? undefined : body)
|
|
178
|
+
}
|
|
179
|
+
|
|
128
180
|
const provider = detectProvider(req.method, req.url)
|
|
129
181
|
|
|
130
182
|
if (!provider) {
|
|
@@ -211,16 +263,24 @@ return http.createServer(async (req, res) => {
|
|
|
211
263
|
try {
|
|
212
264
|
const parsed = JSON.parse(textBody.toString('utf-8'))
|
|
213
265
|
|
|
214
|
-
const
|
|
215
|
-
?
|
|
266
|
+
const sessionKey = (config.stages?.includes('graph') || config.stages?.includes('read-diff'))
|
|
267
|
+
? deriveSessionKey(req.headers)
|
|
216
268
|
: null
|
|
217
|
-
const
|
|
269
|
+
const sessionBucket = config.stages?.includes('graph') && sessionKey
|
|
270
|
+
? sessionStore.getBucket(sessionKey)
|
|
271
|
+
: null
|
|
272
|
+
const { body, stats, disclosure } = await compressRequest(parsed, { ...config, sessionBucket, sessionKey, readCache, brCache }, provider)
|
|
218
273
|
finalBody = Buffer.from(JSON.stringify(body), 'utf-8')
|
|
219
274
|
// Send uncompressed — simpler and content-length is accurate
|
|
220
275
|
if (decompressed) delete headers['content-encoding']
|
|
221
276
|
|
|
222
277
|
session.record(stats)
|
|
223
|
-
|
|
278
|
+
if (disclosure) {
|
|
279
|
+
disclosureTotals.rehydrated += disclosure.rehydrated || 0
|
|
280
|
+
disclosureTotals.missed += disclosure.missed || 0
|
|
281
|
+
disclosureTotals.disclosed += disclosure.disclosed || 0
|
|
282
|
+
}
|
|
283
|
+
log(formatRequestLog(stats, session, provider.name, req.url, textBody.length, config.tokenCost, sessionBucket))
|
|
224
284
|
config.onCompress?.(stats, session.getTotals(), { provider: provider.name, url: req.url, bodySize: textBody.length })
|
|
225
285
|
} catch (err) {
|
|
226
286
|
log(`[tamp] passthrough (parse error): ${err.message}`)
|
package/lib/bm25.js
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// Query-aware line ranker for huge tool_result bodies (Phase 6).
|
|
2
|
+
//
|
|
3
|
+
// Pure-JS BM25 scorer, zero deps. Given a query (last user message) and a
|
|
4
|
+
// large body, score each line against the query using Okapi BM25 and keep
|
|
5
|
+
// the highest-scoring lines until a token budget is satisfied. First and
|
|
6
|
+
// last lines are always preserved as context anchors, and collapsed ranges
|
|
7
|
+
// are replaced by a "[... N lines omitted, N chars ...]" marker so the
|
|
8
|
+
// model can reason about the elision.
|
|
9
|
+
//
|
|
10
|
+
// This primitive is reusable: any future stage that wants "keep the most
|
|
11
|
+
// relevant N lines" can import tokenize/bm25Scores directly.
|
|
12
|
+
|
|
13
|
+
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
14
|
+
|
|
15
|
+
// --- Tokenizer ---
|
|
16
|
+
// Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
|
|
17
|
+
// `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
|
|
18
|
+
// separators to underscore before splitting on non-word characters. Single
|
|
19
|
+
// character tokens are dropped (too noisy for BM25 on short queries).
|
|
20
|
+
const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
|
|
21
|
+
const SPLIT_RE = /[^A-Za-z0-9_]+/
|
|
22
|
+
|
|
23
|
+
export function tokenize(text) {
|
|
24
|
+
if (typeof text !== 'string' || text.length === 0) return []
|
|
25
|
+
let normalized = text.toLowerCase()
|
|
26
|
+
// Collapse intra-word separators. Run twice so chains like `a.b.c` fully
|
|
27
|
+
// collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
|
|
28
|
+
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
29
|
+
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
30
|
+
const raw = normalized.split(SPLIT_RE)
|
|
31
|
+
const out = []
|
|
32
|
+
for (const tok of raw) {
|
|
33
|
+
if (tok.length < 2) continue
|
|
34
|
+
out.push(tok)
|
|
35
|
+
}
|
|
36
|
+
return out
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// --- BM25 ---
|
|
40
|
+
// Standard Okapi BM25. Each document is a line; query is the user's last
|
|
41
|
+
// message. Returns Float64Array of scores aligned with the documents array.
|
|
42
|
+
export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
43
|
+
const N = documents.length
|
|
44
|
+
const scores = new Float64Array(N)
|
|
45
|
+
if (N === 0) return scores
|
|
46
|
+
|
|
47
|
+
const qTokens = tokenize(query)
|
|
48
|
+
if (qTokens.length === 0) return scores
|
|
49
|
+
|
|
50
|
+
// Tokenize docs + build doc stats.
|
|
51
|
+
const docTokens = new Array(N)
|
|
52
|
+
const docLens = new Float64Array(N)
|
|
53
|
+
let totalLen = 0
|
|
54
|
+
for (let i = 0; i < N; i++) {
|
|
55
|
+
const toks = tokenize(documents[i])
|
|
56
|
+
docTokens[i] = toks
|
|
57
|
+
docLens[i] = toks.length
|
|
58
|
+
totalLen += toks.length
|
|
59
|
+
}
|
|
60
|
+
const avgdl = N > 0 ? totalLen / N : 0
|
|
61
|
+
if (avgdl === 0) return scores
|
|
62
|
+
|
|
63
|
+
// Document-frequency per unique query term.
|
|
64
|
+
const uniqQ = Array.from(new Set(qTokens))
|
|
65
|
+
const df = new Map()
|
|
66
|
+
for (const term of uniqQ) {
|
|
67
|
+
let count = 0
|
|
68
|
+
for (let i = 0; i < N; i++) {
|
|
69
|
+
if (docTokens[i].includes(term)) count += 1
|
|
70
|
+
}
|
|
71
|
+
df.set(term, count)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
|
|
75
|
+
const idf = new Map()
|
|
76
|
+
for (const term of uniqQ) {
|
|
77
|
+
const n = df.get(term) || 0
|
|
78
|
+
const v = Math.log(1 + (N - n + 0.5) / (n + 0.5))
|
|
79
|
+
idf.set(term, v)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for (let i = 0; i < N; i++) {
|
|
83
|
+
const dl = docLens[i]
|
|
84
|
+
if (dl === 0) continue
|
|
85
|
+
// Term-frequency table for this doc (only for query terms we care about).
|
|
86
|
+
const tf = new Map()
|
|
87
|
+
const toks = docTokens[i]
|
|
88
|
+
for (const t of toks) {
|
|
89
|
+
if (!idf.has(t)) continue
|
|
90
|
+
tf.set(t, (tf.get(t) || 0) + 1)
|
|
91
|
+
}
|
|
92
|
+
let s = 0
|
|
93
|
+
for (const term of uniqQ) {
|
|
94
|
+
const f = tf.get(term) || 0
|
|
95
|
+
if (f === 0) continue
|
|
96
|
+
const num = f * (k1 + 1)
|
|
97
|
+
const den = f + k1 * (1 - b + b * (dl / avgdl))
|
|
98
|
+
s += idf.get(term) * (num / den)
|
|
99
|
+
}
|
|
100
|
+
scores[i] = s
|
|
101
|
+
}
|
|
102
|
+
return scores
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// --- Line trimmer ---
|
|
106
|
+
// Keep top-K highest-scoring lines until `targetTokens` is consumed. First
|
|
107
|
+
// and last line are always preserved. Returns null when text is already
|
|
108
|
+
// under budget, has fewer than `minLines` lines, or trimming didn't help.
|
|
109
|
+
//
|
|
110
|
+
// Output shape:
|
|
111
|
+
// { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
|
|
112
|
+
export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLines = 10 } = {}) {
|
|
113
|
+
if (typeof text !== 'string' || text.length === 0) return null
|
|
114
|
+
|
|
115
|
+
const originalTokens = countTokens(text)
|
|
116
|
+
if (originalTokens <= targetTokens) return null
|
|
117
|
+
|
|
118
|
+
const lines = text.split('\n')
|
|
119
|
+
const N = lines.length
|
|
120
|
+
if (N < minLines) return null
|
|
121
|
+
|
|
122
|
+
// Per-line token costs (includes the trailing newline for all but the last
|
|
123
|
+
// line; we approximate by counting each line once and adding N-1 tokens).
|
|
124
|
+
const lineTokens = new Array(N)
|
|
125
|
+
for (let i = 0; i < N; i++) {
|
|
126
|
+
lineTokens[i] = countTokens(lines[i])
|
|
127
|
+
}
|
|
128
|
+
const newlineCost = Math.max(0, N - 1)
|
|
129
|
+
|
|
130
|
+
const scores = bm25Scores(lines, query)
|
|
131
|
+
|
|
132
|
+
// Always-keep set: first + last as anchors.
|
|
133
|
+
const keep = new Uint8Array(N)
|
|
134
|
+
keep[0] = 1
|
|
135
|
+
keep[N - 1] = 1
|
|
136
|
+
let budgetUsed = lineTokens[0] + (N > 1 ? lineTokens[N - 1] : 0)
|
|
137
|
+
// Rough marker cost reserved per collapsed gap; we'll reconcile below.
|
|
138
|
+
const MARKER_RESERVE = 16
|
|
139
|
+
|
|
140
|
+
// Sort remaining lines by score desc, then by position asc (stable earlier
|
|
141
|
+
// context preferred on ties). We pick greedily until budget is hit.
|
|
142
|
+
const candidates = []
|
|
143
|
+
for (let i = 1; i < N - 1; i++) {
|
|
144
|
+
candidates.push({ i, score: scores[i], len: lineTokens[i] })
|
|
145
|
+
}
|
|
146
|
+
candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
|
|
147
|
+
|
|
148
|
+
for (const c of candidates) {
|
|
149
|
+
if (budgetUsed + c.len > targetTokens) continue
|
|
150
|
+
keep[c.i] = 1
|
|
151
|
+
budgetUsed += c.len
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// If even the anchors + marker exceed budget, still proceed — we need to
|
|
155
|
+
// produce SOME trim. But if nothing was dropped at all, bail.
|
|
156
|
+
let keptLines = 0
|
|
157
|
+
for (let i = 0; i < N; i++) keptLines += keep[i]
|
|
158
|
+
const droppedLines = N - keptLines
|
|
159
|
+
if (droppedLines === 0) return null
|
|
160
|
+
|
|
161
|
+
// Assemble output, collapsing contiguous dropped runs into drop markers.
|
|
162
|
+
const out = []
|
|
163
|
+
let i = 0
|
|
164
|
+
while (i < N) {
|
|
165
|
+
if (keep[i]) {
|
|
166
|
+
out.push(lines[i])
|
|
167
|
+
i += 1
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
let j = i
|
|
171
|
+
let chars = 0
|
|
172
|
+
while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
|
|
173
|
+
const runLen = j - i
|
|
174
|
+
out.push(`[... ${runLen} lines omitted, ${chars} chars ...]`)
|
|
175
|
+
i = j
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const trimmedText = out.join('\n')
|
|
179
|
+
const trimmedTokens = countTokens(trimmedText)
|
|
180
|
+
// Defensive: if trimming somehow grew the text, don't return a trim.
|
|
181
|
+
if (trimmedTokens >= originalTokens) return null
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
text: trimmedText,
|
|
185
|
+
keptLines,
|
|
186
|
+
droppedLines,
|
|
187
|
+
originalTokens,
|
|
188
|
+
trimmedTokens,
|
|
189
|
+
originalLines: N,
|
|
190
|
+
budgetUsed,
|
|
191
|
+
newlineCost,
|
|
192
|
+
markerReserve: MARKER_RESERVE,
|
|
193
|
+
}
|
|
194
|
+
}
|