@sliday/tamp 0.8.19 → 0.8.21

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -54,7 +54,9 @@ Auto-detects API format, compresses tool output, forwards upstream. Error result
54
54
  | `diff` | Replace similar re-reads with diffs |
55
55
  | `prune` | Remove low-value metadata |
56
56
 
57
- Opt-in stages: `strip-comments`, `textpress` (LLM semantic compression), `graph` (session-scoped dedup — works on any coding agent: Codex, Claude Code, Aider — anywhere the same file is read twice, up to -99% per repeat block)
57
+ Opt-in stages: `strip-comments`, `textpress` (LLM semantic compression), `graph` (session-scoped dedup — works on any coding agent: Codex, Claude Code, Aider — anywhere the same file is read twice, up to -99% per repeat block), `px-render` (renders dense tool_results as PNG image blocks via [pxpipe](https://github.com/teamchong/pxpipe) — image tokens are priced by pixel area, so 4-64KB grep/code blocks drop ~80% on vision-legible models like Fable 5; see `docs/notes-px-render.md`), `stale-inputs` (prunes payloads from `tool_use.input` in older turns, keeping the call header — measured 18% on realistic requests), `stale-images` (evicts base64 images from older turns, keeping the newest 2 — ~4%)
58
+
59
+ `stale-inputs` and `stale-images` rewrite older turns, which invalidates the prompt cache. Both stay opt-in for that reason; weigh the token saving against cache-read pricing. See `reports/magic-compact-review.md`.
58
60
 
59
61
  ### LLMLingua sidecar
60
62
 
package/compress.js CHANGED
@@ -9,9 +9,12 @@ import { redactText } from './lib/redact.js'
9
9
  import { anthropic } from './providers.js'
10
10
  import { graphDeduplicateTargets } from './session-graph.js'
11
11
  import { detectTaskType, generateOutputRules, isDangerousTask } from './lib/rules-generator.js'
12
- import { extractTargetPaths } from './lib/path-extract.js'
12
+ import { extractTargetPaths, extractTargetToolNames } from './lib/path-extract.js'
13
+ import { evictStaleImages } from './lib/stale-images.js'
14
+ import { pruneStaleInputs } from './lib/stale-inputs.js'
13
15
  import { summarize, rehydrateReferences } from './lib/disclosure.js'
14
16
  import { trimLinesByRelevance } from './lib/bm25.js'
17
+ import { pxRenderTargets } from './lib/px-render.js'
15
18
 
16
19
  const cache = new Map()
17
20
  const MAX_CACHE = 500
@@ -98,6 +101,25 @@ function quickSimilarity(a, b) {
98
101
  // can be confidently attributed (via the preceding tool_use), compare against
99
102
  // the last-seen body for the same (session, path) pair. If the patch is small
100
103
  // enough, substitute it for the body. Always cache the fresh text afterwards.
104
+ // Tool results that must reach the model verbatim. Unlike a Read or a Bash
105
+ // run, re-invoking these cannot reproduce the content — it would re-prompt the
106
+ // human. Tool attribution is Anthropic-only (see lib/path-extract.js).
107
+ const PROTECTED_TOOLS = new Set(['AskUserQuestion'])
108
+
109
+ function exemptProtectedTools(targets, body, provider) {
110
+ if (provider?.name !== 'anthropic' || !targets.length) return 0
111
+ const names = extractTargetToolNames(body, targets)
112
+ let exempted = 0
113
+ for (let i = 0; i < targets.length; i++) {
114
+ const target = targets[i]
115
+ if (!target || target.skip) continue
116
+ if (!PROTECTED_TOOLS.has(names[i])) continue
117
+ target.skip = 'user-decision'
118
+ exempted += 1
119
+ }
120
+ return exempted
121
+ }
122
+
101
123
  function readDiffTargets(targets, body, sessionKey, readCache) {
102
124
  if (!readCache || !sessionKey) return
103
125
  const paths = extractTargetPaths(body, targets)
@@ -681,6 +703,13 @@ export async function compressRequest(body, config, provider) {
681
703
 
682
704
  const targets = provider.extract(body, { ...config, cacheSafe: config.cacheSafe !== false })
683
705
 
706
+ // Exempt recorded user decisions before any stage sees them. Re-running
707
+ // AskUserQuestion would re-prompt the human, so its result is the one body
708
+ // in a transcript that cannot be regenerated — the same reasoning that
709
+ // already exempts errored results at extract time. These bodies are tiny,
710
+ // so the forgone compression costs nothing.
711
+ exemptProtectedTools(targets, body, provider)
712
+
684
713
  // Redaction runs FIRST (security, not compression): mask secrets in every
685
714
  // target's text before any stage — dedup/diff/cache/llmlingua/textpress —
686
715
  // sees it, so secrets never enter a disk cache or get shipped to a third
@@ -725,6 +754,18 @@ export async function compressRequest(body, config, provider) {
725
754
  bm25TrimTargets(targets, body, provider)
726
755
  }
727
756
 
757
+ // px-render: image large dense tool_results for legible models (opt-in, lossy).
758
+ // Runs after bm25-trim so query-aware trimming keeps first claim on huge
759
+ // blocks; px-render picks up dense blocks bm25 declined or can't see.
760
+ if (config.stages.includes('px-render')) {
761
+ await pxRenderTargets(targets, body, provider, {
762
+ pxModels: config.pxModels,
763
+ pxMinChars: config.pxMinChars,
764
+ pxMaxImages: config.pxMaxImages,
765
+ isDangerousTask,
766
+ })
767
+ }
768
+
728
769
  const stats = []
729
770
  for (const target of targets) {
730
771
  if (target.skip) { stats.push({ index: target.index, skipped: target.skip }); continue }
@@ -754,6 +795,12 @@ export async function compressRequest(body, config, provider) {
754
795
  stats.push({ index: target.index, method: 'bm25-trim', originalLen: target.text.length, compressedLen: target.compressed.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.compressed) })
755
796
  continue
756
797
  }
798
+ if (target.pxRendered) {
799
+ // compressed is a block array (images + factsheet); token counts were
800
+ // computed at render time from pixel area, not text length.
801
+ stats.push({ index: target.index, method: 'px-render', originalLen: target.text.length, compressedLen: JSON.stringify(target.compressed).length, originalTokens: target.pxOriginalTokens, compressedTokens: target.pxCompressedTokens, images: target.pxImageCount })
802
+ continue
803
+ }
757
804
 
758
805
  const result = await compressBlock(target.text, config)
759
806
  if (result) {
@@ -769,7 +816,23 @@ export async function compressRequest(body, config, provider) {
769
816
  }
770
817
  }
771
818
  provider.apply(body, targets)
772
- return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount }
819
+
820
+ // stale-images: drop base64 images from older turns (opt-in, lossy). Runs
821
+ // after apply so the extractor never sees the notices it leaves behind.
822
+ let staleImages = null
823
+ if (config.stages.includes('stale-images') && provider.name === 'anthropic') {
824
+ const r = evictStaleImages(body, { keepRecent: config.staleImageKeepRecent })
825
+ if (r.evicted > 0) staleImages = r
826
+ }
827
+
828
+ // stale-inputs: drop payloads from older tool_use blocks (opt-in, lossy).
829
+ let staleInputs = null
830
+ if (config.stages.includes('stale-inputs') && provider.name === 'anthropic') {
831
+ const r = pruneStaleInputs(body, { keepRecent: config.staleInputKeepRecent })
832
+ if (r.pruned > 0) staleInputs = r
833
+ }
834
+
835
+ return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount, staleImages, staleInputs }
773
836
  }
774
837
 
775
838
  export async function compressMessages(body, config) {
package/config.js CHANGED
@@ -255,6 +255,13 @@ export function loadConfig(env = process.env, options = {}) {
255
255
  foundationModelsPath: get('TAMP_FOUNDATION_MODELS_PATH') || 'apfel',
256
256
  foundationModelsTimeout: parseInt(get('TAMP_FOUNDATION_MODELS_TIMEOUT'), 10) || 10000,
257
257
  foundationModelsSystemPrompt: get('TAMP_FOUNDATION_MODELS_SYSTEM_PROMPT') || 'Compress this text to 50% length while preserving all key information and meaning. Return only the compressed text without explanation.',
258
+ // px-render (image compression via pxpipe): allowlist defaults to the
259
+ // models MEASURED legible at reading dense rendered text (pxpipe FINDINGS
260
+ // 2026-06-16), not the expensive tier per se — opus-class models read
261
+ // dense identifiers off images poorly and stay opt-in.
262
+ pxModels: get('TAMP_PX_MODELS') || 'claude-fable-5',
263
+ pxMinChars: parseInt(get('TAMP_PX_MIN_CHARS'), 10) || 4096,
264
+ pxMaxImages: parseInt(get('TAMP_PX_MAX_IMAGES'), 10) || 6,
258
265
  tokenCost: parseFloat(get('TAMP_TOKEN_COST')) || 3,
259
266
  })
260
267
  }
@@ -278,7 +285,9 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
278
285
  # Compression level 1..9 (or: conservative, balanced, aggressive, max)
279
286
  # level=5
280
287
 
281
- # Compression preset (conservative | balanced | aggressive)
288
+ # Compression preset (conservative | balanced | aggressive | recommended)
289
+ # "recommended" is the measured best trade-off: aggressive minus llmlingua.
290
+ # Pair it with TAMP_MIN_SIZE=1024 (see whitepaper, Recommended Defaults).
282
291
  # TAMP_COMPRESSION_PRESET=balanced
283
292
 
284
293
  # Explicit stages (overrides preset if set)
@@ -298,6 +307,12 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
298
307
  # (e.g. codex, cursor, cline, aider, claude-code)
299
308
  # TAMP_AGENT=
300
309
 
310
+ # px-render: image large dense tool_results (needs pxpipe-proxy installed).
311
+ # Opt in via TAMP_STAGES=...,px-render. Allowlist = vision-legible models.
312
+ # TAMP_PX_MODELS=claude-fable-5
313
+ # TAMP_PX_MIN_CHARS=4096
314
+ # TAMP_PX_MAX_IMAGES=6
315
+
301
316
  # TAMP_MIN_SIZE=200
302
317
  # TAMP_LOG=true
303
318
  # TAMP_LOG_FILE=
@@ -47,3 +47,34 @@ export function extractTargetPaths(body, targets) {
47
47
  return index.get(toolUseId) || null
48
48
  })
49
49
  }
50
+
51
+ // Map<tool_use_id, toolName>. Same walk as buildToolUseIndex, but keyed on the
52
+ // tool NAME rather than its file path — per-tool rules need to know that a
53
+ // body came from Read (reloadable) vs AskUserQuestion (a recorded decision).
54
+ function buildToolNameIndex(messages) {
55
+ const index = new Map()
56
+ if (!Array.isArray(messages)) return index
57
+
58
+ for (const msg of messages) {
59
+ if (msg?.role !== 'assistant' || !Array.isArray(msg.content)) continue
60
+ for (const block of msg.content) {
61
+ if (block?.type !== 'tool_use' || !block.id) continue
62
+ if (typeof block.name === 'string' && block.name.length > 0) {
63
+ index.set(block.id, block.name)
64
+ }
65
+ }
66
+ }
67
+ return index
68
+ }
69
+
70
+ // Parallel to `targets`: the tool name that produced each target, or null when
71
+ // it can't be attributed. Never reorders — callers index by position.
72
+ export function extractTargetToolNames(body, targets) {
73
+ const index = buildToolNameIndex(body?.messages)
74
+ return targets.map(target => {
75
+ if (!target || target.skip) return null
76
+ const toolUseId = findToolUseIdForTarget(body, target)
77
+ if (!toolUseId) return null
78
+ return index.get(toolUseId) || null
79
+ })
80
+ }
@@ -0,0 +1,170 @@
1
+ // px-render stage: render large DENSE tool_results as PNG image blocks.
2
+ //
3
+ // Borrowed from teamchong/pxpipe (MIT): image input tokens are priced by pixel
4
+ // area (~pixels/750 on Anthropic), not text volume, so content that tokenizes
5
+ // badly as text (paths, hashes, hex, grep output at ~1-2 chars/token) packs
6
+ // ~3x more chars per token as a rendered image. Prose tokenizes fine (~3.5+
7
+ // chars/token) and LOSES money imaged — the density gate below excludes it.
8
+ //
9
+ // Model-gated: vision legibility varies wildly per model (pxpipe FINDINGS:
10
+ // fable-5 reads dense hex 13/15, opus-tier 2/15), so the allowlist defaults to
11
+ // the models measured legible, not the expensive ones per se. Reading images
12
+ // is lossy-in-fidelity: a factsheet of precision-critical tokens rides beside
13
+ // the images as plain text so exact identifiers never depend on glyph reading.
14
+
15
+ import { countTokens } from '@anthropic-ai/tokenizer'
16
+
17
+ // Anthropic vision pricing: tokens ≈ pixels / 750.
18
+ const PIXELS_PER_TOKEN = 750
19
+ // Post-render profitability margin: images + factsheet must beat text by 20%.
20
+ const PROFIT_MARGIN = 0.8
21
+ // Density pre-gate: chars-per-text-token at or above this reads as clean
22
+ // prose — the content class with the worst measured image-legibility risk and
23
+ // the least token upside. This gate is a cheap CPU skip only; the exact
24
+ // profitability decision is the post-render check below (pixel area is known
25
+ // then). Grep/code/log content sits at ~2.5-3.5 chars/token and must pass.
26
+ const MAX_DENSITY = 4.0
27
+
28
+ let pxpipeModule // memoized dynamic import; null = tried and missing
29
+ async function loadPxpipe() {
30
+ if (pxpipeModule !== undefined) return pxpipeModule
31
+ try {
32
+ pxpipeModule = await import('pxpipe-proxy')
33
+ } catch {
34
+ pxpipeModule = null
35
+ }
36
+ return pxpipeModule
37
+ }
38
+
39
+ // Bracketed variant tags ([1m] etc.) stripped so base and variant gate identically.
40
+ export function pxModelAllowed(model, allowCsv) {
41
+ if (typeof model !== 'string' || !allowCsv) return false
42
+ const base = model.replace(/\[[^\]]*\]/g, '')
43
+ return allowCsv.split(',').map(s => s.trim()).filter(Boolean)
44
+ .some(b => base === b || base.startsWith(`${b}-`))
45
+ }
46
+
47
+ // --- Lean factsheet (adapted from pxpipe src/core/factsheet.ts, MIT) ---
48
+ // Extracts precision-critical tokens (paths, SHAs, versions, flags, numbers,
49
+ // CONST_IDS) so they ride next to the images as verbatim text. Deterministic
50
+ // by construction (fixed pattern order, total sort, no Date/random) so the
51
+ // emitted line is byte-stable and never busts the prompt cache.
52
+ const FS_PATTERNS = [
53
+ /\bhttps?:\/\/[^\s)"'<>]+/g,
54
+ /\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g,
55
+ /(?:[\w@~+-]+)?(?:\/[\w.@+-]+)+\.[A-Za-z]\w{0,8}\b/g,
56
+ /\b(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b/g,
57
+ /\bv?\d+\.\d+(?:\.\d+)?(?:[-+][\w.]+)?\b/g,
58
+ /\b\d[\d,_]{3,}\b/g,
59
+ /\b[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+\b/g,
60
+ ]
61
+ const FS_MAX_TOKENS = 64
62
+ const FS_MAX_URLS = 8
63
+ const FS_MAX_SCAN = 262_144
64
+ const FS_URL = /^https?:\/\//
65
+ // Short opaque identifiers (SHAs, UUIDs, numbers, CONST_IDS) outrank long URLs:
66
+ // length is anti-correlated with OCR-risk × consequence.
67
+ const FS_TIER0 = /^(?:(?=[0-9a-f]*\d)[0-9a-f]{7,40}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+|\d[\d,_]*|\d+\.\d+)$/
68
+
69
+ function fsTier(tok) {
70
+ if (FS_TIER0.test(tok)) return 0
71
+ if (FS_URL.test(tok)) return 2
72
+ return 1
73
+ }
74
+
75
+ export function extractFactsheetTokens(text) {
76
+ const scan = text.length > FS_MAX_SCAN ? text.slice(0, FS_MAX_SCAN) : text
77
+ const seen = new Set()
78
+ for (const chunk of scan.split(/\s+/)) {
79
+ if (chunk.length < 3 || chunk.length > 512) continue
80
+ for (const re of FS_PATTERNS) {
81
+ for (const m of chunk.matchAll(re)) {
82
+ const tok = m[0].replace(/[.,;:!?]+$/, '')
83
+ if (tok.length >= 3 && tok.length <= 120) seen.add(tok)
84
+ }
85
+ }
86
+ if (seen.size >= 2048) break
87
+ }
88
+ // Substring collapse (length-desc, then lexical — total order) so partial
89
+ // forms fold into the most specific token.
90
+ const ordered = [...seen].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0))
91
+ const specific = []
92
+ for (const t of ordered) {
93
+ if (!specific.some(k => k.includes(t))) specific.push(t)
94
+ }
95
+ const ranked = specific
96
+ .map(t => ({ t, tier: fsTier(t) }))
97
+ .sort((a, b) => a.tier - b.tier || b.t.length - a.t.length || (a.t < b.t ? -1 : a.t > b.t ? 1 : 0))
98
+ const kept = []
99
+ let urls = 0
100
+ for (const { t, tier } of ranked) {
101
+ if (kept.length >= FS_MAX_TOKENS) break
102
+ if (tier === 2 && urls++ >= FS_MAX_URLS) continue
103
+ kept.push(t)
104
+ }
105
+ return kept
106
+ }
107
+
108
+ export function factsheetText(text) {
109
+ const tokens = extractFactsheetTokens(text)
110
+ if (!tokens.length) return ''
111
+ return '[Exact identifiers from the rendered image above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image: '
112
+ + tokens.join(' · ') + ']'
113
+ }
114
+
115
+ // --- The stage ---
116
+ // Claims eligible targets by setting target.compressed to a tool_result
117
+ // content ARRAY (image blocks + factsheet text). applyTargets writes it
118
+ // verbatim, and the Anthropic API accepts block arrays at tool_result.content.
119
+ // Only string-content tool_result targets are eligible (path depth 5 ending in
120
+ // 'content'): sub-text targets point at a `text` field where an array would be
121
+ // structurally invalid.
122
+ export async function pxRenderTargets(targets, body, provider, config) {
123
+ if (!provider || provider.name !== 'anthropic') return 0
124
+ if (!pxModelAllowed(body?.model, config.pxModels)) return 0
125
+ const userText = typeof provider.getLastUserText === 'function' ? (provider.getLastUserText(body) || '') : ''
126
+ if (userText && config.isDangerousTask?.(userText)) return 0
127
+
128
+ const px = await loadPxpipe()
129
+ if (!px?.renderTextToImages) return 0
130
+
131
+ let rendered = 0
132
+ for (const target of targets) {
133
+ if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.bm25Trimmed || target.compressed) continue
134
+ if (typeof target.text !== 'string') continue
135
+ if (!Array.isArray(target.path) || target.path.length !== 5 || target.path[target.path.length - 1] !== 'content') continue
136
+ if (target.text.length < config.pxMinChars) continue
137
+
138
+ const textTokens = countTokens(target.text)
139
+ if (textTokens === 0 || target.text.length / textTokens >= MAX_DENSITY) continue
140
+
141
+ let result
142
+ try {
143
+ result = await px.renderTextToImages(target.text, { reflow: true })
144
+ } catch {
145
+ continue
146
+ }
147
+ if (!result?.pages?.length || result.pages.length > config.pxMaxImages) continue
148
+ // Atlas gaps render as blank cells — silent loss, don't ship it.
149
+ if (result.droppedChars > 0) continue
150
+
151
+ const sheet = factsheetText(target.text)
152
+ const imageTokens = Math.ceil(result.pixels / PIXELS_PER_TOKEN)
153
+ const sheetTokens = sheet ? countTokens(sheet) : 0
154
+ if (imageTokens + sheetTokens >= textTokens * PROFIT_MARGIN) continue
155
+
156
+ const blocks = result.pages.map(p => ({
157
+ type: 'image',
158
+ source: { type: 'base64', media_type: 'image/png', data: Buffer.from(p.png).toString('base64') },
159
+ }))
160
+ if (sheet) blocks.push({ type: 'text', text: sheet })
161
+
162
+ target.compressed = blocks
163
+ target.pxRendered = true
164
+ target.pxOriginalTokens = textTokens
165
+ target.pxCompressedTokens = imageTokens + sheetTokens
166
+ target.pxImageCount = result.pages.length
167
+ rendered += 1
168
+ }
169
+ return rendered
170
+ }
@@ -0,0 +1,81 @@
1
+ // Stale image eviction (opt-in, lossy).
2
+ //
3
+ // Tamp's target extractor only collects `type: 'text'` sub-blocks, so base64
4
+ // image blocks inside tool_results pass through untouched. Measured across 403
5
+ // code-only sessions they carry 79% of all tokens: reading PNGs in graphics and
6
+ // game projects, not just browser screenshots.
7
+ //
8
+ // Images are also the safest thing to drop, because they are reloadable — the
9
+ // agent can re-Read the file or re-take the screenshot. What is not safe is
10
+ // dropping the image the agent just asked to see, so eviction is scoped by age:
11
+ // the newest `keepRecent` image-bearing turns keep their pixels.
12
+ //
13
+ // Cost: rewriting old turns invalidates the prompt cache, which is exactly what
14
+ // tamp's cacheSafe mode exists to avoid. That is why this stage is opt-in.
15
+ // Anthropic block shape only.
16
+
17
+ const DEFAULT_KEEP_RECENT = 2
18
+
19
+ function isImageBlock(block) {
20
+ return block?.type === 'image'
21
+ }
22
+
23
+ function hasImage(block) {
24
+ return block?.type === 'tool_result'
25
+ && Array.isArray(block.content)
26
+ && block.content.some(isImageBlock)
27
+ }
28
+
29
+ // Map<tool_use_id, file_path> so the notice can name what was dropped.
30
+ function buildPathIndex(messages) {
31
+ const index = new Map()
32
+ for (const msg of messages) {
33
+ if (msg?.role !== 'assistant' || !Array.isArray(msg.content)) continue
34
+ for (const block of msg.content) {
35
+ if (block?.type !== 'tool_use' || !block.id) continue
36
+ const path = block.input?.file_path || block.input?.path || block.input?.filePath
37
+ if (typeof path === 'string' && path.length > 0) index.set(block.id, path)
38
+ }
39
+ }
40
+ return index
41
+ }
42
+
43
+ // Named source beats a bare "an image": it tells the agent what to re-read.
44
+ function notice(path, chars) {
45
+ const subject = path ? `Image of ${path}` : 'Image'
46
+ return `<tamp-stale-image>\n${subject} dropped — it is older than the most recent turns and images are reloadable.\nIf you still need to see it, re-read the file (or re-take the screenshot).\nDropped: ${chars} base64 characters\n</tamp-stale-image>`
47
+ }
48
+
49
+ export function evictStaleImages(body, { keepRecent = DEFAULT_KEEP_RECENT } = {}) {
50
+ const messages = body?.messages
51
+ const none = { evicted: 0, chars: 0 }
52
+ if (!Array.isArray(messages)) return none
53
+
54
+ const imageTurns = []
55
+ for (let mi = 0; mi < messages.length; mi++) {
56
+ const msg = messages[mi]
57
+ if (msg?.role !== 'user' || !Array.isArray(msg.content)) continue
58
+ if (msg.content.some(hasImage)) imageTurns.push(mi)
59
+ }
60
+ if (imageTurns.length <= keepRecent) return none
61
+
62
+ const paths = buildPathIndex(messages)
63
+ const stale = imageTurns.slice(0, imageTurns.length - keepRecent)
64
+ let evicted = 0
65
+ let chars = 0
66
+
67
+ for (const mi of stale) {
68
+ for (const block of messages[mi].content) {
69
+ if (!hasImage(block)) continue
70
+ block.content = block.content.map(sub => {
71
+ if (!isImageBlock(sub)) return sub
72
+ const size = typeof sub.source?.data === 'string' ? sub.source.data.length : 0
73
+ evicted += 1
74
+ chars += size
75
+ return { type: 'text', text: notice(paths.get(block.tool_use_id), size) }
76
+ })
77
+ }
78
+ }
79
+
80
+ return { evicted, chars }
81
+ }
@@ -0,0 +1,96 @@
1
+ // Stale tool_use.input pruning (opt-in, lossy).
2
+ //
3
+ // Rule table adapted from magic-compact (https://github.com/aerovato/magic-compact,
4
+ // BSD-3-Clause, src/prune.ts). Thresholds and per-tool choices follow that work.
5
+ //
6
+ // tool_use.input carries 26% of billed tokens across 60 large sessions — the
7
+ // largest mass tamp has never touched. An old tool call still needs to show its
8
+ // SHAPE (which tool, against which file) because that is the record of what the
9
+ // agent did. Its payload does not: the written file is on disk, the edit already
10
+ // landed, the agent prompt already ran. So headers stay, payloads go.
11
+ //
12
+ // Age-scoped for the same reason as stale-images: pruning the call the agent is
13
+ // still working with would break it. Rewriting old turns invalidates the prompt
14
+ // cache, which is why this is opt-in. Anthropic block shape only.
15
+
16
+ const DEFAULT_KEEP_RECENT = 2
17
+ const LIMIT = { words: 128, chars: 1024 }
18
+ const BASH_LIMIT = 1024
19
+ const BASH_HEAD = 512
20
+
21
+ function tooSmall(text) {
22
+ if (typeof text !== 'string') return true
23
+ const trimmed = text.trim()
24
+ const words = trimmed ? trimmed.split(/\s+/).length : 0
25
+ return words <= LIMIT.words && text.length <= LIMIT.chars
26
+ }
27
+
28
+ function notice(what, chars) {
29
+ return `[tamp: ${what} omitted from this older turn (${chars} chars). Re-read the file if you need the current contents.]`
30
+ }
31
+
32
+ // field -> label. Each is a payload the agent can recover another way.
33
+ const OMIT_FIELDS = {
34
+ Write: [['content', 'file contents']],
35
+ NotebookEdit: [['new_source', 'notebook source']],
36
+ Agent: [['prompt', 'agent prompt']],
37
+ Workflow: [['script', 'workflow script']],
38
+ SendMessage: [['message', 'inter-agent message']],
39
+ }
40
+
41
+ function pruneInput(name, input) {
42
+ if (!input || typeof input !== 'object') return false
43
+ if (name === 'AskUserQuestion') return false
44
+
45
+ if (name === 'Bash') {
46
+ const command = input.command
47
+ if (typeof command !== 'string' || command.length <= BASH_LIMIT) return false
48
+ input.command = `${command.slice(0, BASH_HEAD)}\n[tamp: command truncated, ${command.length} chars total]`
49
+ return true
50
+ }
51
+
52
+ if (name === 'Edit') {
53
+ const oldStr = input.old_string
54
+ const newStr = input.new_string
55
+ if (typeof oldStr !== 'string' || typeof newStr !== 'string') return false
56
+ if (tooSmall(`${oldStr}\n${newStr}`)) return false
57
+ input.old_string = notice('edit old_string', oldStr.length)
58
+ input.new_string = notice('edit new_string', newStr.length)
59
+ return true
60
+ }
61
+
62
+ const fields = OMIT_FIELDS[name]
63
+ if (!fields) return false
64
+ let changed = false
65
+ for (const [field, label] of fields) {
66
+ const value = input[field]
67
+ if (typeof value !== 'string' || tooSmall(value)) continue
68
+ input[field] = notice(label, value.length)
69
+ changed = true
70
+ }
71
+ return changed
72
+ }
73
+
74
+ export function pruneStaleInputs(body, { keepRecent = DEFAULT_KEEP_RECENT } = {}) {
75
+ const messages = body?.messages
76
+ const none = { pruned: 0 }
77
+ if (!Array.isArray(messages)) return none
78
+
79
+ const callTurns = []
80
+ for (let mi = 0; mi < messages.length; mi++) {
81
+ const msg = messages[mi]
82
+ if (msg?.role !== 'assistant' || !Array.isArray(msg.content)) continue
83
+ if (msg.content.some(b => b?.type === 'tool_use')) callTurns.push(mi)
84
+ }
85
+ if (callTurns.length <= keepRecent) return none
86
+
87
+ let pruned = 0
88
+ for (const mi of callTurns.slice(0, callTurns.length - keepRecent)) {
89
+ for (const block of messages[mi].content) {
90
+ if (block?.type !== 'tool_use') continue
91
+ if (pruneInput(block.name, block.input)) pruned += 1
92
+ }
93
+ }
94
+
95
+ return { pruned }
96
+ }
package/metadata.js CHANGED
@@ -25,6 +25,9 @@ export const EXTRA_STAGES = Object.freeze([
25
25
  'br-cache',
26
26
  'disclosure',
27
27
  'bm25-trim',
28
+ 'px-render',
29
+ 'stale-images',
30
+ 'stale-inputs',
28
31
  ])
29
32
 
30
33
  export const ALL_STAGES = Object.freeze([
@@ -39,6 +42,8 @@ export const ALL_STAGES = Object.freeze([
39
42
  // `disclosure` IS lossy: it drops body content from the outgoing turn and
40
43
  // relies on the model quoting the marker to trigger rehydration next turn.
41
44
  // `bm25-trim` drops low-relevance lines from huge tool_results (lossy).
45
+ // `stale-images` drops base64 images from older turns (lossy — recoverable
46
+ // only by re-reading the file, not from any tamp cache).
42
47
  export const LOSSY_STAGES = Object.freeze(new Set([
43
48
  'llmlingua',
44
49
  'foundation-models',
@@ -46,6 +51,9 @@ export const LOSSY_STAGES = Object.freeze(new Set([
46
51
  'strip-comments',
47
52
  'disclosure',
48
53
  'bm25-trim',
54
+ 'px-render',
55
+ 'stale-images',
56
+ 'stale-inputs',
49
57
  ]))
50
58
 
51
59
  export function isLossy(stage) {
@@ -92,6 +100,18 @@ export const STAGE_HINTS = Object.freeze({
92
100
  summary: 'query-aware line ranking, preserves first+last, bypasses dangerous tasks',
93
101
  setup: 'TAMP_STAGES=...,bm25-trim',
94
102
  },
103
+ 'px-render': {
104
+ summary: 'image-render dense tool_results, ~3x chars/token on vision-legible models (lossy)',
105
+ setup: 'npm i pxpipe-proxy && TAMP_STAGES=...,px-render',
106
+ },
107
+ 'stale-images': {
108
+ summary: 'drop base64 images from older turns, keep the newest 2 (lossy, invalidates prompt cache)',
109
+ setup: 'TAMP_STAGES=...,stale-images',
110
+ },
111
+ 'stale-inputs': {
112
+ summary: 'drop payloads from older tool_use inputs, keep headers and the newest 2 turns (lossy)',
113
+ setup: 'TAMP_STAGES=...,stale-inputs',
114
+ },
95
115
  })
96
116
 
97
117
  export const STAGE_DESCRIPTIONS = Object.freeze({
@@ -112,6 +132,9 @@ export const STAGE_DESCRIPTIONS = Object.freeze({
112
132
  'br-cache': 'Disk-backed Brotli store for large tool_results (lossless, opt-in)',
113
133
  disclosure: '3-tier summary for huge tool_results with on-demand rehydration (lossy, aggressive-only)',
114
134
  'bm25-trim': 'Drop low-relevance lines from huge tool_results via BM25 (lossy, aggressive-only)',
135
+ 'px-render': 'Render dense tool_results as PNG image blocks via pxpipe (lossy, model-gated, opt-in)',
136
+ 'stale-images': 'Evict base64 images from older turns, keeping the newest 2 (lossy, opt-in)',
137
+ 'stale-inputs': 'Prune payloads from older tool_use inputs, keeping call headers (lossy, opt-in)',
115
138
  })
116
139
 
117
140
  export const COMPRESSION_PRESETS = Object.freeze({
@@ -139,6 +162,19 @@ export const COMPRESSION_PRESETS = Object.freeze({
139
162
  risk: 'Medium (may lose comments, verbose text)',
140
163
  level: 8,
141
164
  },
165
+ // Autoresearch study (v0.8.19, deepseek-v4-flash judge): aggressive minus
166
+ // llmlingua scored 79.95% input savings at 20/20 fact retention, vs 68.51%
167
+ // for full aggressive — llmlingua shadowed textpress on large blocks and
168
+ // corrupted numerals on small ones. Pair with TAMP_MIN_SIZE=1024.
169
+ // See bench/recommend-eval.js and the whitepaper's Recommended Defaults section.
170
+ recommended: {
171
+ name: 'Recommended',
172
+ description: 'Best measured savings/quality trade-off (aggressive minus llmlingua; use TAMP_MIN_SIZE=1024)',
173
+ stages: ['cmd-strip', 'minify', 'toon', 'strip-lines', 'whitespace', 'dedup', 'diff', 'read-diff', 'prune', 'strip-comments', 'textpress', 'br-cache', 'disclosure', 'bm25-trim'],
174
+ expectedSavings: '63-80%',
175
+ risk: 'Medium (may lose comments, paraphrase verbose text)',
176
+ level: null,
177
+ },
142
178
  })
143
179
 
144
180
  // Flat 1..9 compression ladder, parallel to the named presets. Each level
package/package.json CHANGED
@@ -14,12 +14,12 @@
14
14
  "sidecar/server.py",
15
15
  "sidecar/requirements.txt"
16
16
  ],
17
- "version": "0.8.19",
17
+ "version": "0.8.21",
18
18
  "description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
19
19
  "type": "module",
20
20
  "main": "index.js",
21
21
  "bin": {
22
- "tamp": "./bin/tamp.js"
22
+ "tamp": "bin/tamp.js"
23
23
  },
24
24
  "scripts": {
25
25
  "start": "node bin/tamp.js",
@@ -43,7 +43,7 @@
43
43
  "license": "MIT",
44
44
  "repository": {
45
45
  "type": "git",
46
- "url": "https://github.com/sliday/tamp"
46
+ "url": "git+https://github.com/sliday/tamp.git"
47
47
  },
48
48
  "homepage": "https://github.com/sliday/tamp",
49
49
  "dependencies": {
@@ -54,5 +54,8 @@
54
54
  "fzstd": "^0.1.1",
55
55
  "ink": "^5.2.0",
56
56
  "react": "^18.2.0"
57
+ },
58
+ "optionalDependencies": {
59
+ "pxpipe-proxy": "^0.8.0"
57
60
  }
58
61
  }