@sliday/tamp 0.8.7 → 0.8.15

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/README.md CHANGED
@@ -247,14 +247,19 @@ Supported on all providers: Anthropic, OpenAI Chat, OpenAI Responses (Codex), Ge
247
247
  | Variable | Default | Description |
248
248
  |----------|---------|-------------|
249
249
  | `TAMP_PORT` | `7778` | Listen port |
250
+ | `TAMP_HOST` | `127.0.0.1` | Bind address. Loopback by default; set `0.0.0.0` to expose beyond the local machine (e.g. in a container) |
250
251
  | `TAMP_UPSTREAM` | `https://api.anthropic.com` | Anthropic (default) upstream |
251
252
  | `TAMP_UPSTREAM_OPENAI` | `https://api.openai.com` | Upstream for OpenAI-compatible routes (`/v1/chat/completions`, `/v1/responses`) |
252
253
  | `TAMP_UPSTREAM_GEMINI` | `https://generativelanguage.googleapis.com` | Upstream for Gemini (`generateContent`) routes |
253
254
  | `TAMP_UPSTREAM_KIMI` | `https://api.kimi.com` | Upstream for Kimi Code routes |
254
255
  | `TAMP_UPSTREAM_MOONSHOT` | `https://api.moonshot.cn` | Upstream for Moonshot OpenAI-compatible routes |
255
256
  | `TAMP_MIN_SIZE` | `200` | Min content size (chars) |
257
+ | `TAMP_MAX_BODY` | `10485760` | Max request body (bytes, 10 MB). Larger bodies pass through uncompressed |
256
258
  | `TAMP_LOG` | `true` | Enable logging |
259
+ | `TAMP_LOG_FILE` | *(none)* | Append logs to this file path, in addition to stderr |
257
260
  | `TAMP_CACHE_SAFE` | `true` | Compress newest only (prompt-cache safe) |
261
+ | `TAMP_REDACT` | `true` | Mask secrets in tool output before anything leaves the machine |
262
+ | `TAMP_REDACT_MODE` | `mask` | `mask` (replace with marker) or `remove` (delete value) |
258
263
  | `TAMP_LLMLINGUA_URL` | *(none)* | LLMLingua sidecar URL. Set to `http://localhost:8788` to skip the auto-probe (avoids startup race if sidecar is still loading the model) |
259
264
 
260
265
  **Recommended setups:**
@@ -361,7 +366,7 @@ The real win in v0.8 is **the level knob itself** — a zip-like 1–9 dial that
361
366
 
362
367
  Tamp runs on `localhost` and forwards to the upstream you configure. Most stages are local-only: `llmlingua` talks to a sidecar on `127.0.0.1`, `foundation-models` stays on-device. The one stage that sends content off-box is `textpress` (opt-in, level 7+), which posts to OpenRouter when `OPENROUTER_API_KEY` is set — leave it off if your tool output is sensitive.
363
368
 
364
- **Secret redaction** is on the roadmap ([#6](https://github.com/sliday/tamp/issues/6)): a `redact` stage that masks API keys, tokens, and `.env` values before anything leaves the machine, running ahead of every other stage. Until it lands, Tamp forwards file contents verbatim — treat your upstream's data policy as the boundary.
369
+ **Secret redaction** is on by default ([#6](https://github.com/sliday/tamp/issues/6)). Before any stage runs — including ones that ship text off-box — Tamp masks high-confidence secrets in tool output: provider-shaped keys (AWS, GitHub, Anthropic/OpenAI, Google, Slack, Stripe), JWTs, PEM private-key blocks, and `UPPER_SNAKE` secret assignments from `.env`/shell (`API_TOKEN=…`, `DATABASE_PASSWORD=…`). Masking keeps structure (`API_TOKEN=‹redacted:secret›`) so the model still reasons about it. Set `TAMP_REDACT=false` to disable, or `TAMP_REDACT_MODE=remove` to delete rather than mask. Detection favors precision over recall: it won't catch every secret, so treat your upstream's data policy as the final boundary.
365
370
 
366
371
  This repo ships an agent harness (built with [harn.app](https://harn.app)): `AGENTS.md` plus PreToolUse / Stop hooks under `scripts/harness/` that block dangerous shell commands and gate completion on `npm test`. See `.claude/settings.json`.
367
372
 
package/bin/tamp.js CHANGED
@@ -529,7 +529,7 @@ if (skipPrompt) {
529
529
  process.exit(1)
530
530
  })
531
531
 
532
- server.listen(config.port, () => {
532
+ server.listen(config.port, config.host, () => {
533
533
  try { writePidFile(config.port) } catch (err) {
534
534
  console.error(`[tamp] Warning: could not write PID file: ${err.message}`)
535
535
  }
package/compress.js CHANGED
@@ -3,8 +3,9 @@ import { spawn } from 'node:child_process'
3
3
  import { encode } from '@toon-format/toon'
4
4
  import { countTokens } from '@anthropic-ai/tokenizer'
5
5
  import { createPatch } from 'diff'
6
- import { tryParseJSON, classifyContent, stripLineNumbers } from './detect.js'
6
+ import { tryParseJSON, classifyContent, stripLineNumbers, jsonHasUnsafeInteger } from './detect.js'
7
7
  import { rewriteCommandOutput } from './lib/rewriters/index.js'
8
+ import { redactText } from './lib/redact.js'
8
9
  import { anthropic } from './providers.js'
9
10
  import { graphDeduplicateTargets } from './session-graph.js'
10
11
  import { detectTaskType, generateOutputRules, isDangerousTask } from './lib/rules-generator.js'
@@ -23,9 +24,17 @@ function cacheKey(text) {
23
24
  export function clearCache() { cache.clear() }
24
25
 
25
26
  function normalizeWhitespace(text) {
26
- return text
27
- .replace(/[ \t]+$/gm, '')
28
- .replace(/\n{3,}/g, '\n\n')
27
+ // Right-trim each line by hand. The former `/[ \t]+$/gm` is quadratic: a long
28
+ // run of spaces/tabs NOT at line end makes the greedy match restart at every
29
+ // offset, so a padded/aligned tool_result (tables, ASCII art) stalled the
30
+ // proxy event loop for seconds. This scan touches each char once.
31
+ let out = ''
32
+ for (let line of text.split('\n')) {
33
+ let e = line.length
34
+ while (e > 0 && (line[e - 1] === ' ' || line[e - 1] === '\t')) e--
35
+ out += (e === line.length ? line : line.slice(0, e)) + '\n'
36
+ }
37
+ return out.slice(0, -1).replace(/\n{3,}/g, '\n\n')
29
38
  }
30
39
 
31
40
  // --- Stage: dedup ---
@@ -181,16 +190,19 @@ function bm25TrimTargets(targets, body, provider) {
181
190
 
182
191
  // --- Stage: prune ---
183
192
  // Unambiguously npm-internal keys — safe to drop regardless of value.
184
- const PRUNE_KEYS = new Set(['_id', '_from', '_resolved', '_integrity', '_nodeVersion', '_npmVersion', '_phantomChildren', '_requiredBy'])
185
- // `integrity` / `shasum` are common field names; only drop them when the value
186
- // is actually an npm hash, so a legitimately-named field (e.g. integrity:"high")
187
- // in a non-npm tool_result is preserved.
193
+ const PRUNE_KEYS = new Set(['_from', '_resolved', '_integrity', '_nodeVersion', '_npmVersion', '_phantomChildren', '_requiredBy'])
194
+ // `_id` / `integrity` / `shasum` are common field names; only drop them when the
195
+ // value is actually npm metadata, so a legitimately-named field (e.g.
196
+ // integrity:"high", or a MongoDB `_id` document key) in a non-npm tool_result is
197
+ // preserved. npm's `_id` is always `name@version` (optionally @scope/-prefixed).
188
198
  const NPM_INTEGRITY = /^sha\d{1,3}-[A-Za-z0-9+/]+={0,2}$/
189
199
  const NPM_SHASUM = /^[a-f0-9]{40}$/i
200
+ const NPM_PKG_ID = /^(@[\w.-]+\/)?[\w.-]+@\d[\w.\-+]*$/
190
201
 
191
202
  function shouldPrune(key, val) {
192
203
  if (PRUNE_KEYS.has(key)) return true
193
204
  if (typeof val !== 'string') return false
205
+ if (key === '_id' && NPM_PKG_ID.test(val)) return true
194
206
  if (key === 'resolved' && val.startsWith('https://registry.')) return true
195
207
  if (key === 'integrity' && NPM_INTEGRITY.test(val)) return true
196
208
  if (key === 'shasum' && NPM_SHASUM.test(val)) return true
@@ -234,12 +246,46 @@ function stripLineComments(line) {
234
246
  return line
235
247
  }
236
248
 
249
+ // Remove line-level `/* ... */` block comments (opener at start of line after
250
+ // optional spaces/tabs) and `<!-- ... -->` HTML comments, in ONE linear pass.
251
+ // A regex can't do this without O(n^2): `/^[ \t]*\/\*[\s\S]*?\*\//gm` rescans to
252
+ // EOF at every unclosed opener, so a file full of `/*` line-starts stalled the
253
+ // event loop (256 KB took ~40s). An unterminated comment is left unchanged, to
254
+ // match the old lazy regex (which required a closing delimiter to match).
255
+ function stripBlockAndHtmlComments(text) {
256
+ let out = ''
257
+ let i = 0
258
+ const n = text.length
259
+ let lineStart = true
260
+ while (i < n) {
261
+ if (lineStart) {
262
+ let j = i
263
+ while (j < n && (text[j] === ' ' || text[j] === '\t')) j++
264
+ if (text[j] === '/' && text[j + 1] === '*') {
265
+ const close = text.indexOf('*/', j + 2)
266
+ if (close === -1) { out += text.slice(i); break }
267
+ i = close + 2
268
+ lineStart = false
269
+ continue
270
+ }
271
+ }
272
+ if (text[i] === '<' && text.startsWith('<!--', i)) {
273
+ const close = text.indexOf('-->', i + 4)
274
+ if (close === -1) { out += text.slice(i); break }
275
+ i = close + 3
276
+ lineStart = false
277
+ continue
278
+ }
279
+ const c = text[i]
280
+ out += c
281
+ lineStart = c === '\n'
282
+ i++
283
+ }
284
+ return out
285
+ }
286
+
237
287
  function stripComments(text) {
238
- // Block comments: only strip if they start at line-level (not inside strings)
239
- let result = text
240
- .replace(/^[ \t]*\/\*\*[\s\S]*?\*\//gm, '')
241
- .replace(/^[ \t]*\/\*[\s\S]*?\*\//gm, '')
242
- .replace(/<!--[\s\S]*?-->/g, '')
288
+ let result = stripBlockAndHtmlComments(text)
243
289
  result = result.split('\n').map(stripLineComments).join('\n')
244
290
  return result.replace(/\n\s*\n\s*\n/g, '\n\n')
245
291
  }
@@ -374,6 +420,11 @@ export function compressText(text, config) {
374
420
 
375
421
  let raw = cls === 'json-lined' ? stripLineNumbers(text) : text
376
422
 
423
+ // Integers beyond 2^53 lose precision through JSON.parse/stringify, silently
424
+ // corrupting numeric ids/amounts. We can't compress such JSON without changing
425
+ // its meaning, so forward it untouched.
426
+ if (jsonHasUnsafeInteger(raw)) return null
427
+
377
428
  // Prune low-value fields before minifying
378
429
  let parsedValue
379
430
  if (config.stages.includes('prune')) {
@@ -598,6 +649,20 @@ export async function compressRequest(body, config, provider) {
598
649
 
599
650
  const targets = provider.extract(body, { ...config, cacheSafe: config.cacheSafe !== false })
600
651
 
652
+ // Redaction runs FIRST (security, not compression): mask secrets in every
653
+ // target's text before any stage — dedup/diff/cache/llmlingua/textpress —
654
+ // sees it, so secrets never enter a disk cache or get shipped to a third
655
+ // party. We mutate target.text in place so the whole pipeline operates on
656
+ // the redacted body; the non-compressible case is closed in the loop below.
657
+ let redactedCount = 0
658
+ if (config.redact !== false) {
659
+ for (const target of targets) {
660
+ if (target.skip || typeof target.text !== 'string') continue
661
+ const r = redactText(target.text, config.redactMode)
662
+ if (r.count > 0) { target.text = r.text; target.redacted = true; redactedCount += r.count }
663
+ }
664
+ }
665
+
601
666
  // Read-diff: session-scoped unified diff for re-reads (lossless, opt-in)
602
667
  if (config.stages.includes('read-diff') && config.readCache && config.sessionKey && provider.name === 'anthropic') {
603
668
  readDiffTargets(targets, body, config.sessionKey, config.readCache)
@@ -662,12 +727,17 @@ export async function compressRequest(body, config, provider) {
662
727
  if (result) {
663
728
  target.compressed = result.text
664
729
  stats.push({ index: target.index, ...result })
730
+ } else if (target.redacted) {
731
+ // Not otherwise compressible, but it was redacted — apply() only writes
732
+ // back when .compressed is set, so push the redacted text through.
733
+ target.compressed = target.text
734
+ stats.push({ index: target.index, method: 'redact', originalLen: target.text.length, compressedLen: target.text.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.text) })
665
735
  } else {
666
736
  stats.push({ index: target.index, skipped: 'not-compressible' })
667
737
  }
668
738
  }
669
739
  provider.apply(body, targets)
670
- return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats }
740
+ return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount }
671
741
  }
672
742
 
673
743
  export async function compressMessages(body, config) {
package/config.js CHANGED
@@ -37,14 +37,17 @@ function coerceLevelInput(raw) {
37
37
  const v = resolveLevel(n) ? n : null
38
38
  return { value: v, valid: v !== null }
39
39
  }
40
- if (s in LEVEL_ALIASES) return { value: s, valid: true }
40
+ // Object.hasOwn, NOT `in`: `in` walks the prototype chain, so
41
+ // TAMP_LEVEL=constructor / toString / valueOf pass as valid and later crash
42
+ // stagesForLevel with an inherited function instead of falling back cleanly.
43
+ if (Object.hasOwn(LEVEL_ALIASES, s)) return { value: s, valid: true }
41
44
  return { value: null, valid: false }
42
45
  }
43
46
 
44
47
  // Convert a level input (number or alias) to its canonical integer.
45
48
  function levelInputToInt(input) {
46
49
  if (typeof input === 'number') return input
47
- if (typeof input === 'string' && input in LEVEL_ALIASES) return LEVEL_ALIASES[input]
50
+ if (typeof input === 'string' && Object.hasOwn(LEVEL_ALIASES, input)) return LEVEL_ALIASES[input]
48
51
  return null
49
52
  }
50
53
 
@@ -152,7 +155,10 @@ export function loadConfig(env = process.env, options = {}) {
152
155
  stages = stagesForLevel(level)
153
156
  levelSource = chosen.source
154
157
  } else if (presetExplicit) {
155
- const preset = COMPRESSION_PRESETS[presetNameRaw]
158
+ // Object.hasOwn guard: a bare `COMPRESSION_PRESETS[presetNameRaw]` lets
159
+ // __proto__/constructor/toString return truthy prototype objects that
160
+ // pass the `if (preset)` check and then crash on `...preset.stages`.
161
+ const preset = Object.hasOwn(COMPRESSION_PRESETS, presetNameRaw) ? COMPRESSION_PRESETS[presetNameRaw] : null
156
162
  if (preset) {
157
163
  stages = [...preset.stages]
158
164
  level = typeof preset.level === 'number' ? preset.level : null
@@ -200,6 +206,11 @@ export function loadConfig(env = process.env, options = {}) {
200
206
  return Object.freeze({
201
207
  version: VERSION,
202
208
  port: parseInt(get('TAMP_PORT'), 10) || 7778,
209
+ // Bind loopback by default. The proxy forwards the caller's API key to the
210
+ // upstream, so binding all interfaces (the old no-host default) turned any
211
+ // LAN peer into an authenticated relay. Set TAMP_HOST=0.0.0.0 to opt into
212
+ // broader binding (e.g. inside a container).
213
+ host: get('TAMP_HOST') || '127.0.0.1',
203
214
  upstream: get('TAMP_UPSTREAM') || 'https://api.anthropic.com',
204
215
  upstreams: Object.freeze({
205
216
  anthropic: get('TAMP_UPSTREAM') || 'https://api.anthropic.com',
@@ -225,12 +236,17 @@ export function loadConfig(env = process.env, options = {}) {
225
236
  levelSource,
226
237
  outputMode,
227
238
  outputModeDefault: defaultOutputMode && VALID_OUTPUT_MODES.has(defaultOutputMode) ? defaultOutputMode : null,
228
- agent: get('TAMP_AGENT') || null,
239
+ agent: get('TAMP_AGENT')?.toLowerCase() || null,
229
240
  autoDetectTaskType: parseBoolean(get('TAMP_AUTO_DETECT_TASK_TYPE'), true),
230
241
  log: get('TAMP_LOG') !== 'false',
231
242
  logFile: get('TAMP_LOG_FILE') || null,
232
243
  maxBody: parseInt(get('TAMP_MAX_BODY'), 10) || 10_485_760,
233
244
  cacheSafe: parseBoolean(get('TAMP_CACHE_SAFE'), true),
245
+ // Secret redaction is a security pass, not a compression stage: on by
246
+ // default at every level, masks secrets before any stage (incl. ones that
247
+ // ship text to third parties) sees the body. TAMP_REDACT=false disables it.
248
+ redact: parseBoolean(get('TAMP_REDACT'), true),
249
+ redactMode: get('TAMP_REDACT_MODE') === 'remove' ? 'remove' : 'mask',
234
250
  llmLinguaUrl: get('TAMP_LLMLINGUA_URL') || null,
235
251
  textpressOllamaUrl: get('TAMP_TEXTPRESS_OLLAMA_URL') || 'http://localhost:11434',
236
252
  textpressOllamaModel: get('TAMP_TEXTPRESS_OLLAMA_MODEL') || 'qwen3.5:0.8b',
@@ -248,6 +264,7 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
248
264
  # https://github.com/sliday/tamp
249
265
 
250
266
  # TAMP_PORT=7778
267
+ # TAMP_HOST=127.0.0.1 # bind address; set 0.0.0.0 to expose beyond loopback
251
268
  # TAMP_UPSTREAM=https://api.anthropic.com
252
269
  # TAMP_UPSTREAM_OPENAI=https://api.openai.com
253
270
  # TAMP_UPSTREAM_GEMINI=https://generativelanguage.googleapis.com
@@ -286,6 +303,8 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
286
303
  # TAMP_LOG_FILE=
287
304
  # TAMP_MAX_BODY=10485760
288
305
  # TAMP_CACHE_SAFE=true
306
+ # TAMP_REDACT=true
307
+ # TAMP_REDACT_MODE=mask
289
308
  # TAMP_TOKEN_COST=3
290
309
  # TAMP_FOUNDATION_MODELS_PATH=apfel
291
310
  # TAMP_FOUNDATION_MODELS_TIMEOUT=10000
package/detect.js CHANGED
@@ -27,6 +27,48 @@ export function stripLineNumbers(str) {
27
27
  return lines.map(l => l.replace(LINE_NUM_RE, '')).join('\n')
28
28
  }
29
29
 
30
+ // True if the JSON text contains an integer literal outside the JS safe-integer
31
+ // range (|n| > 2^53-1). Such values lose precision through JSON.parse, so the
32
+ // minify/toon stages would silently alter a numeric id/amount — a semantic
33
+ // corruption. Scans number tokens outside string literals; floats are left
34
+ // alone (they are inherently approximate). Conservative on the safe side: when
35
+ // in doubt it returns true so the caller skips compression.
36
+ export function jsonHasUnsafeInteger(str) {
37
+ if (typeof str !== 'string') return false
38
+ let inString = false
39
+ let escaped = false
40
+ for (let i = 0; i < str.length; i++) {
41
+ const ch = str[i]
42
+ if (inString) {
43
+ if (escaped) escaped = false
44
+ else if (ch === '\\') escaped = true
45
+ else if (ch === '"') inString = false
46
+ continue
47
+ }
48
+ if (ch === '"') { inString = true; continue }
49
+ if (ch !== '-' && (ch < '0' || ch > '9')) continue
50
+ // Start of a number token. Consume the integer-part digit run.
51
+ let j = ch === '-' ? i + 1 : i
52
+ const digitStart = j
53
+ while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
54
+ const intDigits = j - digitStart
55
+ // Consume any fraction + exponent so their digits aren't re-scanned as a
56
+ // separate integer. Floats are left alone (inherently approximate).
57
+ let isFloat = false
58
+ if (str[j] === '.') { isFloat = true; j++; while (j < str.length && str[j] >= '0' && str[j] <= '9') j++ }
59
+ if (str[j] === 'e' || str[j] === 'E') {
60
+ isFloat = true; j++
61
+ if (str[j] === '+' || str[j] === '-') j++
62
+ while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
63
+ }
64
+ if (!isFloat && intDigits >= 16 && !Number.isSafeInteger(Number(str.slice(i, j)))) {
65
+ return true
66
+ }
67
+ i = j - 1
68
+ }
69
+ return false
70
+ }
71
+
30
72
  export function tryParseJSON(str) {
31
73
  if (typeof str !== 'string' || str.length === 0) return { ok: false }
32
74
  try {
@@ -40,7 +82,12 @@ export function tryParseJSON(str) {
40
82
  export function isTOON(str) {
41
83
  if (typeof str !== 'string') return false
42
84
  const firstLine = str.trimStart().split('\n')[0]
43
- return /^\[TOON\]/.test(firstLine) || /\w+\[\d+\]\{/.test(firstLine) || /\w+\[\d+\]:/.test(firstLine)
85
+ // Anchored + bounded. The former unanchored `\w+\[\d+\]\{` was quadratic: on a
86
+ // long word-char run with no bracket the greedy `\w+` backtracks and, being
87
+ // unanchored, retries at every offset — a single-line blob (minified JSON, a
88
+ // long token) stalled the event loop. TOON headers begin the line, so `^`
89
+ // alone kills it; the {1,64}/{1,15} bounds are belt-and-suspenders.
90
+ return /^\[TOON\]/.test(firstLine) || /^\s*\w{1,64}\[\d{1,15}\]\{/.test(firstLine) || /^\s*\w{1,64}\[\d{1,15}\]:/.test(firstLine)
44
91
  }
45
92
 
46
93
  export function classifyContent(str) {
package/index.js CHANGED
@@ -175,6 +175,13 @@ function pipeRequest(req, res, upstreamUrl, prefixChunks) {
175
175
  const headers = { ...req.headers }
176
176
  delete headers.host
177
177
  const upstream = openUpstream(req.method, upstreamUrl, headers, res)
178
+ // Guard the inbound stream: a client RST mid-upload emits 'error' on `req`,
179
+ // and with no listener Node rethrows it and crashes the whole proxy. The
180
+ // buffered path already guards this; the piped passthrough must too.
181
+ req.on('error', (err) => {
182
+ log(`[tamp] client upload error: ${err.code || ''} ${err.message}`)
183
+ upstream.destroy()
184
+ })
178
185
  if (prefixChunks) {
179
186
  for (const chunk of prefixChunks) upstream.write(chunk)
180
187
  }
@@ -415,8 +422,8 @@ const isMain = !process.argv[1]?.includes('node_modules') && process.argv[1] ===
415
422
 
416
423
  if (isMain) {
417
424
  const { config, server } = createProxy()
418
- server.listen(config.port, () => {
419
- console.error(`[tamp] proxy listening on http://localhost:${config.port}`)
425
+ server.listen(config.port, config.host, () => {
426
+ console.error(`[tamp] proxy listening on http://${config.host}:${config.port}`)
420
427
  console.error(`[tamp] upstream: ${config.upstream}`)
421
428
  console.error(`[tamp] stages: ${config.stages.join(', ')}`)
422
429
 
package/lib/bm25.js CHANGED
@@ -133,7 +133,15 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
133
133
  const keep = new Uint8Array(N)
134
134
  keep[0] = 1
135
135
  keep[N - 1] = 1
136
- let budgetUsed = lineTokens[0] + (N > 1 ? lineTokens[N - 1] : 0)
136
+ // Anchor the last NON-EMPTY line too. Tool output almost always ends with
137
+ // '\n', so lines[N-1] is '' — pinning only that empty terminator would let
138
+ // the real final line (often a trailing error or stack trace, exactly the
139
+ // "context anchor" this claims to preserve) be dropped as a candidate.
140
+ let lastReal = N - 1
141
+ while (lastReal > 0 && lines[lastReal] === '') lastReal--
142
+ keep[lastReal] = 1
143
+ let budgetUsed = 0
144
+ for (const idx of new Set([0, N - 1, lastReal])) budgetUsed += lineTokens[idx]
137
145
  // Rough marker cost reserved per collapsed gap; we'll reconcile below.
138
146
  const MARKER_RESERVE = 16
139
147
 
@@ -141,6 +149,7 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
141
149
  // context preferred on ties). We pick greedily until budget is hit.
142
150
  const candidates = []
143
151
  for (let i = 1; i < N - 1; i++) {
152
+ if (keep[i]) continue // skip already-anchored lines (e.g. lastReal)
144
153
  candidates.push({ i, score: scores[i], len: lineTokens[i] })
145
154
  }
146
155
  candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
package/lib/redact.js ADDED
@@ -0,0 +1,78 @@
1
+ // --- Secret redaction (security, not compression) ---
2
+ //
3
+ // Masks or removes high-confidence secrets in outbound tool_result bodies
4
+ // before anything leaves the machine — and before any stage that ships text
5
+ // to a third party (llmlingua sidecar, textpress/OpenRouter) ever sees it.
6
+ //
7
+ // Design bias: precision over recall. A false positive corrupts legitimate
8
+ // content the model needs; a false negative leaks a secret. We only match
9
+ // patterns with a recognizable shape (provider-prefixed keys, PEM blocks,
10
+ // JWTs) plus assignment lines whose KEY name is unambiguously a secret. We do
11
+ // NOT do entropy guessing — too noisy for code/log payloads.
12
+ //
13
+ // Why: https://github.com/sliday/tamp/issues/6
14
+
15
+ // Each rule: a global regex and a short label used in the mask marker. Where a
16
+ // rule needs to keep surrounding structure (assignments), it captures the
17
+ // secret in group 1 and rebuilds the line via `replace`.
18
+ const TOKEN_RULES = [
19
+ { label: 'aws-access-key', re: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[0-9A-Z]{16}\b/g },
20
+ { label: 'github-token', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/g },
21
+ { label: 'github-pat', re: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/g },
22
+ { label: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
23
+ { label: 'openai-key', re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/g },
24
+ { label: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
25
+ { label: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g },
26
+ { label: 'stripe-key', re: /\b[rs]k_live_[A-Za-z0-9]{20,}\b/g },
27
+ { label: 'google-oauth', re: /\bya29\.[A-Za-z0-9_-]{20,}\b/g },
28
+ { label: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g },
29
+ { label: 'private-key', re: /-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/g },
30
+ ]
31
+
32
+ // Assignment lines from .env / shell / config where the KEY name names a
33
+ // secret. We mask only the VALUE, keeping the key so the model still sees the
34
+ // variable exists.
35
+ //
36
+ // Precision rules learned the hard way:
37
+ // - Case-SENSITIVE uppercase keys only. Real secret env vars are UPPER_SNAKE
38
+ // (`API_KEY`, `AUTH_TOKEN`); camelCase like `apiKey = getKey()` is code,
39
+ // not a secret, and must not match.
40
+ // - The secret word is anchored to the END of the key (immediately before
41
+ // the `=`/`:`), so `AUTHOR=`, `COMPASS=`, `PASSWORD_HINT=` don't trip it.
42
+ // - Value must be 6+ non-space chars to skip empty/placeholder assignments.
43
+ // - The key prefix is BOUNDED (`{0,64}`) and word-boundary anchored. An
44
+ // unbounded `[A-Z0-9_]*` before the secret-word alternation is quadratic:
45
+ // on a long uppercase run with no match the engine retries every split
46
+ // point, so a big CONSTANT_CASE/hex tool_result would stall the proxy
47
+ // event loop for seconds. Real env-var names are well under 64 chars.
48
+ const SECRET_KEY = '\\b[A-Z0-9_]{0,64}(?:SECRET|TOKEN|PASSWORD|PASSWD|API[_-]?KEY|ACCESS[_-]?KEY|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET|CREDENTIALS?|AUTH|PASSPHRASE|SESSION[_-]?KEY)'
49
+ const ASSIGN_RE = new RegExp(
50
+ `((?:export\\s+)?${SECRET_KEY})(\\s*[:=]\\s*)(["']?)([^\\s"']{6,})(["']?)`,
51
+ 'g'
52
+ )
53
+
54
+ function marker(label, mode) {
55
+ return mode === 'remove' ? '' : `‹redacted:${label}›`
56
+ }
57
+
58
+ // Returns { text, count }. `count` is the number of secrets masked/removed.
59
+ // When count is 0 the returned text is referentially the input (no allocation
60
+ // churn for the common no-secret case).
61
+ export function redactText(text, mode = 'mask') {
62
+ if (typeof text !== 'string' || text.length === 0) return { text, count: 0 }
63
+
64
+ let count = 0
65
+ let out = text
66
+
67
+ for (const { label, re } of TOKEN_RULES) {
68
+ out = out.replace(re, () => { count++; return marker(label, mode) })
69
+ }
70
+
71
+ out = out.replace(ASSIGN_RE, (m, key, sep, q1, _val, q2) => {
72
+ count++
73
+ if (mode === 'remove') return `${key}${sep}`
74
+ return `${key}${sep}${q1}${marker('secret', mode)}${q2}`
75
+ })
76
+
77
+ return count > 0 ? { text: out, count } : { text, count: 0 }
78
+ }
@@ -32,11 +32,13 @@ export default {
32
32
  if (BUILDKIT_TRANSFER.test(line)) continue;
33
33
  if (SPINNER_BRAILLE.test(line)) continue;
34
34
  if (LEGACY_PROGRESS.test(line)) continue;
35
- // BuildKit "in-progress" duplicated step lines end with a fractional second
36
- // marker and are re-emitted as DONE later; strip the non-DONE variants.
37
- // Never strip a line carrying error/warning/failure context — a RUN step's
38
- // stdout can legitimately end in a duration (e.g. "build failed in 2.3s").
39
- if (/^#\d+\s+\S.*\s\d+\.\d+s$/.test(line) && !/\sDONE\b/.test(line) && !/\b(error|warning|fail)/i.test(line)) continue;
35
+ // BuildKit "in-progress" step frames carry a [x/y] step counter and end
36
+ // with a fractional-second marker; they are re-emitted as DONE later, so
37
+ // strip the non-DONE variants. Requiring [x/y] avoids eating a RUN step's
38
+ // own stdout (prefixed "#N <timestamp> ...", no counter), which can
39
+ // legitimately end in a duration (e.g. "webpack compiled in 2.5s"). The
40
+ // error/warning/fail guard stays as belt-and-suspenders.
41
+ if (/^#\d+\s+\[\d+\/\d+\]\s.*\s\d+\.\d+s$/.test(line) && !/\sDONE\b/.test(line) && !/\b(error|warning|fail)/i.test(line)) continue;
40
42
  out.push(line);
41
43
  }
42
44
 
@@ -10,7 +10,6 @@
10
10
  // Keeps: error/warn lines, final "added N packages in Ns", vulnerability
11
11
  // counts, "up to date", package trees.
12
12
 
13
- const SPINNER_CHARS = /[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]/;
14
13
  const PROGRESS_LINE = /^\s*[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]\s+/;
15
14
  const DOTS_ONLY = /^\s*\.{3,}\s*$/;
16
15
  const NPM_SIG = /(^|\n)(npm (install|i|ci|run)\b|added \d+ packages?|up to date|changed \d+ packages?|removed \d+ packages?)/;
@@ -38,8 +37,11 @@ export default {
38
37
  for (let i = 0; i < lines.length; i++) {
39
38
  const line = lines[i];
40
39
 
41
- // Drop spinner lines and sole-dot lines.
42
- if (PROGRESS_LINE.test(line) || SPINNER_CHARS.test(line)) continue;
40
+ // Drop leading-spinner progress lines and sole-dot lines. Only anchored
41
+ // PROGRESS_LINE (spinner glyph at line start) — an unanchored glyph test
42
+ // deleted whole lines of child-tool output that merely contained a
43
+ // braille char anywhere (e.g. `⠋ 2 warnings, 1 error`), losing signal.
44
+ if (PROGRESS_LINE.test(line)) continue;
43
45
  if (DOTS_ONLY.test(line)) {
44
46
  if (lastWasDots) continue;
45
47
  lastWasDots = true;
@@ -7,7 +7,12 @@
7
7
  // the short test summary info block, and the final "=== N passed in Xs ===".
8
8
 
9
9
  const COLLECTED = /^collected\s+\d+\s+items?\b/;
10
- const DOT_PROGRESS = /^\S.*\s+[.FsxE]{2,}\s*(\[\s*\d+%\s*\])?\s*$/;
10
+ // Per-file dot-progress: "<path>.py[::node] <status run> [ NN% ]" (percent
11
+ // optional). Anchored to a .py path prefix so a traceback/assertion line that
12
+ // merely ends in "..." is NOT mistaken for progress and silently dropped.
13
+ const DOT_PROGRESS = /^\S*\.py(::\S+)?\s+[.FsxE]{2,}\s*(\[\s*\d+%\s*\])?\s*$/;
14
+ // Wrapped progress continuation: a bare status run followed by a percent.
15
+ const PCT_PROGRESS = /^\s*[.FsxE]{2,}\s*\[\s*\d+%\s*\]\s*$/;
11
16
  const SESSION_START = /^platform\s+\S+\s+--\s+Python\s+\d/;
12
17
  const ROOTDIR = /^rootdir:\s+/;
13
18
  const PLUGINS = /^plugins:\s+/;
@@ -33,9 +38,10 @@ export default {
33
38
  if (ROOTDIR.test(line)) continue;
34
39
  if (PLUGINS.test(line)) continue;
35
40
  if (CACHEDIR.test(line)) continue;
36
- // Drop dot-progress only if the line contains ONLY test status chars
37
- // (not a traceback line that happens to end with dots).
38
- if (DOT_PROGRESS.test(line) && !/error|FAILED|PASSED/.test(line)) continue;
41
+ // Drop per-file / wrapped dot-progress lines. Both patterns are anchored
42
+ // (path prefix or trailing percent), so traceback/assertion lines that end
43
+ // in "..." are preserved without needing a fragile keyword guard.
44
+ if (DOT_PROGRESS.test(line) || PCT_PROGRESS.test(line)) continue;
39
45
  out.push(line);
40
46
  }
41
47
 
package/metadata.js CHANGED
@@ -104,7 +104,7 @@ export const STAGE_DESCRIPTIONS = Object.freeze({
104
104
  'foundation-models': 'Apple Intelligence neural compression (macOS 15+, Apple Silicon, 100% local)',
105
105
  dedup: 'Deduplicate identical tool_results',
106
106
  diff: 'Replace similar re-reads with diffs',
107
- 'read-diff': 'Session-scoped unified diff for re-reads (lossless, opt-in)',
107
+ 'read-diff': 'Session-scoped unified diff for re-reads (lossless, on by default in balanced+)',
108
108
  prune: 'Strip lockfile hashes & npm metadata',
109
109
  'strip-comments': 'Remove code comments (lossy)',
110
110
  textpress: 'LLM semantic compression (Ollama/OpenRouter)',
package/package.json CHANGED
@@ -14,7 +14,7 @@
14
14
  "sidecar/server.py",
15
15
  "sidecar/requirements.txt"
16
16
  ],
17
- "version": "0.8.7",
17
+ "version": "0.8.15",
18
18
  "description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
19
19
  "type": "module",
20
20
  "main": "index.js",
package/providers.js CHANGED
@@ -130,18 +130,16 @@ function findAnthropicReferences(body) {
130
130
  } else if (typeof block.input === 'string') {
131
131
  scanStringForRefs(block.input, ['messages', mi, 'content', bi, 'input'], out)
132
132
  }
133
- } else if (block.type === 'tool_result') {
134
- if (typeof block.content === 'string') {
135
- scanStringForRefs(block.content, ['messages', mi, 'content', bi, 'content'], out)
136
- } else if (Array.isArray(block.content)) {
137
- for (let ci = 0; ci < block.content.length; ci++) {
138
- const sub = block.content[ci]
139
- if (sub?.type === 'text' && typeof sub.text === 'string') {
140
- scanStringForRefs(sub.text, ['messages', mi, 'content', bi, 'content', ci, 'text'], out)
141
- }
142
- }
143
- }
144
133
  }
134
+ // NOTE: tool_result content is deliberately NOT scanned for markers. A
135
+ // marker is only honored when the MODEL quotes it (assistant text or
136
+ // tool_use.input) — that is the disclosure contract. tool_result bodies
137
+ // are untrusted tool output (fetched pages, file reads); scanning them
138
+ // would (1) let attacker-controlled content forge a <tamp-ref:v1:...>
139
+ // marker and pull an arbitrary body out of the shared br-cache into the
140
+ // outbound request (cross-conversation cache-pull + existence oracle),
141
+ // and (2) auto-rehydrate a disclosure summary's own marker every turn,
142
+ // defeating the compression. See rehydrateReferences.
145
143
  }
146
144
  }
147
145
  return out
@@ -221,7 +219,7 @@ const anthropic = {
221
219
  return method === 'POST' && url.startsWith('/v1/messages')
222
220
  },
223
221
  extract(body, config = {}) {
224
- if (!body?.messages?.length) return []
222
+ if (!Array.isArray(body?.messages) || !body.messages.length) return []
225
223
  if (config.cacheSafe) {
226
224
  return findLatestEligibleGroup(body.messages, extractAnthropicMessageTargets)
227
225
  }
@@ -285,13 +283,13 @@ const openai = {
285
283
  return url
286
284
  },
287
285
  extract(body, config = {}) {
288
- if (!body?.messages?.length) return []
286
+ if (!Array.isArray(body?.messages) || !body.messages.length) return []
289
287
 
290
288
  if (config.cacheSafe) {
291
289
  const targets = []
292
290
  for (let i = body.messages.length - 1; i >= 0; i--) {
293
291
  const msg = body.messages[i]
294
- if (msg.role !== 'tool') break
292
+ if (!msg || msg.role !== 'tool') break
295
293
  targets.unshift(...extractOpenAIChatTargets(msg, i))
296
294
  }
297
295
  return targets
@@ -417,7 +415,7 @@ const gemini = {
417
415
  return method === 'POST' && url.includes('generateContent')
418
416
  },
419
417
  extract(body, config = {}) {
420
- if (!body?.contents?.length) return []
418
+ if (!Array.isArray(body?.contents) || !body.contents.length) return []
421
419
  if (config.cacheSafe) {
422
420
  return findLatestEligibleGroup(body.contents, extractGeminiContentTargets)
423
421
  }
package/session-graph.js CHANGED
@@ -66,7 +66,14 @@ function conversationSeed(body) {
66
66
  const first = Array.isArray(turns) && turns.length ? turns[0] : null
67
67
  if (first == null && body.system === undefined) return ''
68
68
  try {
69
- return JSON.stringify({ system: body.system ?? null, first }).slice(0, 4096)
69
+ // Hash system and the first turn INDEPENDENTLY. A single slice over the
70
+ // combined JSON let a long system prompt (Claude Code ships a multi-KB one)
71
+ // crowd the conversation-specific first turn past the cutoff, collapsing
72
+ // distinct conversations onto one key — the exact cross-conversation leak
73
+ // this seed exists to prevent. Per-part hashes keep the seed bounded + stable.
74
+ const sys = createHash('sha256').update(JSON.stringify(body.system ?? null)).digest('hex')
75
+ const frst = createHash('sha256').update(JSON.stringify(first ?? null)).digest('hex')
76
+ return `${sys}:${frst}`
70
77
  } catch {
71
78
  return ''
72
79
  }
package/stats.js CHANGED
@@ -13,7 +13,12 @@ const c = {
13
13
  // (Gemini and OpenAI-compatible gateways) carry the API key in the query
14
14
  // string (?key=, ?api_key=), which would otherwise land in stdout, log
15
15
  // files, and the systemd journal. Path-relative URLs only — no base needed.
16
- const SECRET_QUERY_PARAM = /^(key|api_key|apikey|access_token|token)$/i
16
+ // Keyword (substring) match on the param NAME, not an exact list. The old
17
+ // anchored `^(key|api_key|apikey|access_token|token)$` leaked `refresh_token`,
18
+ // `id_token`, `client_secret`, `password`, `session`, `sig`/`signature`, `auth`
19
+ // to stdout / TAMP_LOG_FILE / the systemd journal. Over-redacting a benign
20
+ // param name in a log is harmless; leaking a credential is not.
21
+ const SECRET_QUERY_PARAM = /key|token|secret|password|passwd|sig|auth|session|credential/i
17
22
  export function redactUrl(url) {
18
23
  if (typeof url !== 'string') return url
19
24
  const q = url.indexOf('?')
@@ -51,7 +56,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
51
56
  const totalComp = compressed.reduce((a, s) => a + s.compressedLen, 0)
52
57
  const totalOrigTok = compressed.reduce((a, s) => a + (s.originalTokens || 0), 0)
53
58
  const totalCompTok = compressed.reduce((a, s) => a + (s.compressedTokens || 0), 0)
54
- const pct = (((totalOrig - totalComp) / totalOrig) * 100).toFixed(1)
59
+ const pct = (totalOrig > 0 ? ((totalOrig - totalComp) / totalOrig) * 100 : 0).toFixed(1)
55
60
  const saved = totalOrig - totalComp
56
61
  const tokSaved = totalOrigTok - totalCompTok
57
62
 
@@ -60,7 +65,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
60
65
  lines.push(`[tamp] ${c.cyan}${label}${c.reset} ${path}${sizeInfo} ${c.green}— ${n} block${n !== 1 ? 's' : ''} compressed, -${pct}%${c.reset}`)
61
66
 
62
67
  for (const s of compressed) {
63
- const sPct = (((s.originalLen - s.compressedLen) / s.originalLen) * 100).toFixed(1)
68
+ const sPct = (s.originalLen > 0 ? ((s.originalLen - s.compressedLen) / s.originalLen) * 100 : 0).toFixed(1)
64
69
  const tokInfo = s.originalTokens ? ` ${c.dim}${s.originalTokens}→${s.compressedTokens} tok${c.reset}` : ''
65
70
  lines.push(`[tamp] ${c.dim}block[${s.index}]${c.reset} ${fmtSize(s.originalLen)}→${fmtSize(s.compressedLen)} ${c.green}-${sPct}%${c.reset}${tokInfo} ${c.dim}[${s.method}]${c.reset}`)
66
71
  }