@sliday/tamp 0.8.5 → 0.8.15
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +11 -2
- package/bin/tamp.js +1 -1
- package/compress.js +99 -12
- package/config.js +35 -6
- package/detect.js +63 -5
- package/index.js +11 -4
- package/lib/bm25.js +10 -1
- package/lib/br-cache.js +5 -3
- package/lib/compress-file.js +5 -1
- package/lib/redact.js +78 -0
- package/lib/rewriters/commands/docker.js +7 -5
- package/lib/rewriters/commands/npm.js +5 -3
- package/lib/rewriters/commands/pytest.js +10 -4
- package/metadata.js +1 -1
- package/package.json +1 -1
- package/providers.js +19 -21
- package/session-graph.js +33 -2
- package/stats.js +8 -3
package/README.md
CHANGED
|
@@ -247,10 +247,19 @@ Supported on all providers: Anthropic, OpenAI Chat, OpenAI Responses (Codex), Ge
|
|
|
247
247
|
| Variable | Default | Description |
|
|
248
248
|
|----------|---------|-------------|
|
|
249
249
|
| `TAMP_PORT` | `7778` | Listen port |
|
|
250
|
-
| `
|
|
250
|
+
| `TAMP_HOST` | `127.0.0.1` | Bind address. Loopback by default; set `0.0.0.0` to expose beyond the local machine (e.g. in a container) |
|
|
251
|
+
| `TAMP_UPSTREAM` | `https://api.anthropic.com` | Anthropic (default) upstream |
|
|
252
|
+
| `TAMP_UPSTREAM_OPENAI` | `https://api.openai.com` | Upstream for OpenAI-compatible routes (`/v1/chat/completions`, `/v1/responses`) |
|
|
253
|
+
| `TAMP_UPSTREAM_GEMINI` | `https://generativelanguage.googleapis.com` | Upstream for Gemini (`generateContent`) routes |
|
|
254
|
+
| `TAMP_UPSTREAM_KIMI` | `https://api.kimi.com` | Upstream for Kimi Code routes |
|
|
255
|
+
| `TAMP_UPSTREAM_MOONSHOT` | `https://api.moonshot.cn` | Upstream for Moonshot OpenAI-compatible routes |
|
|
251
256
|
| `TAMP_MIN_SIZE` | `200` | Min content size (chars) |
|
|
257
|
+
| `TAMP_MAX_BODY` | `10485760` | Max request body (bytes, 10 MB). Larger bodies pass through uncompressed |
|
|
252
258
|
| `TAMP_LOG` | `true` | Enable logging |
|
|
259
|
+
| `TAMP_LOG_FILE` | *(none)* | Append logs to this file path, in addition to stderr |
|
|
253
260
|
| `TAMP_CACHE_SAFE` | `true` | Compress newest only (prompt-cache safe) |
|
|
261
|
+
| `TAMP_REDACT` | `true` | Mask secrets in tool output before anything leaves the machine |
|
|
262
|
+
| `TAMP_REDACT_MODE` | `mask` | `mask` (replace with marker) or `remove` (delete value) |
|
|
254
263
|
| `TAMP_LLMLINGUA_URL` | *(none)* | LLMLingua sidecar URL. Set to `http://localhost:8788` to skip the auto-probe (avoids startup race if sidecar is still loading the model) |
|
|
255
264
|
|
|
256
265
|
**Recommended setups:**
|
|
@@ -357,7 +366,7 @@ The real win in v0.8 is **the level knob itself** — a zip-like 1–9 dial that
|
|
|
357
366
|
|
|
358
367
|
Tamp runs on `localhost` and forwards to the upstream you configure. Most stages are local-only: `llmlingua` talks to a sidecar on `127.0.0.1`, `foundation-models` stays on-device. The one stage that sends content off-box is `textpress` (opt-in, level 7+), which posts to OpenRouter when `OPENROUTER_API_KEY` is set — leave it off if your tool output is sensitive.
|
|
359
368
|
|
|
360
|
-
**Secret redaction** is on
|
|
369
|
+
**Secret redaction** is on by default ([#6](https://github.com/sliday/tamp/issues/6)). Before any stage runs — including ones that ship text off-box — Tamp masks high-confidence secrets in tool output: provider-shaped keys (AWS, GitHub, Anthropic/OpenAI, Google, Slack, Stripe), JWTs, PEM private-key blocks, and `UPPER_SNAKE` secret assignments from `.env`/shell (`API_TOKEN=…`, `DATABASE_PASSWORD=…`). Masking keeps structure (`API_TOKEN=‹redacted:secret›`) so the model still reasons about it. Set `TAMP_REDACT=false` to disable, or `TAMP_REDACT_MODE=remove` to delete rather than mask. Detection favors precision over recall: it won't catch every secret, so treat your upstream's data policy as the final boundary.
|
|
361
370
|
|
|
362
371
|
This repo ships an agent harness (built with [harn.app](https://harn.app)): `AGENTS.md` plus PreToolUse / Stop hooks under `scripts/harness/` that block dangerous shell commands and gate completion on `npm test`. See `.claude/settings.json`.
|
|
363
372
|
|
package/bin/tamp.js
CHANGED
|
@@ -529,7 +529,7 @@ if (skipPrompt) {
|
|
|
529
529
|
process.exit(1)
|
|
530
530
|
})
|
|
531
531
|
|
|
532
|
-
server.listen(config.port, () => {
|
|
532
|
+
server.listen(config.port, config.host, () => {
|
|
533
533
|
try { writePidFile(config.port) } catch (err) {
|
|
534
534
|
console.error(`[tamp] Warning: could not write PID file: ${err.message}`)
|
|
535
535
|
}
|
package/compress.js
CHANGED
|
@@ -3,8 +3,9 @@ import { spawn } from 'node:child_process'
|
|
|
3
3
|
import { encode } from '@toon-format/toon'
|
|
4
4
|
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
5
5
|
import { createPatch } from 'diff'
|
|
6
|
-
import { tryParseJSON, classifyContent, stripLineNumbers } from './detect.js'
|
|
6
|
+
import { tryParseJSON, classifyContent, stripLineNumbers, jsonHasUnsafeInteger } from './detect.js'
|
|
7
7
|
import { rewriteCommandOutput } from './lib/rewriters/index.js'
|
|
8
|
+
import { redactText } from './lib/redact.js'
|
|
8
9
|
import { anthropic } from './providers.js'
|
|
9
10
|
import { graphDeduplicateTargets } from './session-graph.js'
|
|
10
11
|
import { detectTaskType, generateOutputRules, isDangerousTask } from './lib/rules-generator.js'
|
|
@@ -23,9 +24,17 @@ function cacheKey(text) {
|
|
|
23
24
|
export function clearCache() { cache.clear() }
|
|
24
25
|
|
|
25
26
|
function normalizeWhitespace(text) {
|
|
26
|
-
|
|
27
|
-
|
|
28
|
-
|
|
27
|
+
// Right-trim each line by hand. The former `/[ \t]+$/gm` is quadratic: a long
|
|
28
|
+
// run of spaces/tabs NOT at line end makes the greedy match restart at every
|
|
29
|
+
// offset, so a padded/aligned tool_result (tables, ASCII art) stalled the
|
|
30
|
+
// proxy event loop for seconds. This scan touches each char once.
|
|
31
|
+
let out = ''
|
|
32
|
+
for (let line of text.split('\n')) {
|
|
33
|
+
let e = line.length
|
|
34
|
+
while (e > 0 && (line[e - 1] === ' ' || line[e - 1] === '\t')) e--
|
|
35
|
+
out += (e === line.length ? line : line.slice(0, e)) + '\n'
|
|
36
|
+
}
|
|
37
|
+
return out.slice(0, -1).replace(/\n{3,}/g, '\n\n')
|
|
29
38
|
}
|
|
30
39
|
|
|
31
40
|
// --- Stage: dedup ---
|
|
@@ -180,11 +189,23 @@ function bm25TrimTargets(targets, body, provider) {
|
|
|
180
189
|
}
|
|
181
190
|
|
|
182
191
|
// --- Stage: prune ---
|
|
183
|
-
|
|
192
|
+
// Unambiguously npm-internal keys — safe to drop regardless of value.
|
|
193
|
+
const PRUNE_KEYS = new Set(['_from', '_resolved', '_integrity', '_nodeVersion', '_npmVersion', '_phantomChildren', '_requiredBy'])
|
|
194
|
+
// `_id` / `integrity` / `shasum` are common field names; only drop them when the
|
|
195
|
+
// value is actually npm metadata, so a legitimately-named field (e.g.
|
|
196
|
+
// integrity:"high", or a MongoDB `_id` document key) in a non-npm tool_result is
|
|
197
|
+
// preserved. npm's `_id` is always `name@version` (optionally @scope/-prefixed).
|
|
198
|
+
const NPM_INTEGRITY = /^sha\d{1,3}-[A-Za-z0-9+/]+={0,2}$/
|
|
199
|
+
const NPM_SHASUM = /^[a-f0-9]{40}$/i
|
|
200
|
+
const NPM_PKG_ID = /^(@[\w.-]+\/)?[\w.-]+@\d[\w.\-+]*$/
|
|
184
201
|
|
|
185
202
|
function shouldPrune(key, val) {
|
|
186
203
|
if (PRUNE_KEYS.has(key)) return true
|
|
187
|
-
if (
|
|
204
|
+
if (typeof val !== 'string') return false
|
|
205
|
+
if (key === '_id' && NPM_PKG_ID.test(val)) return true
|
|
206
|
+
if (key === 'resolved' && val.startsWith('https://registry.')) return true
|
|
207
|
+
if (key === 'integrity' && NPM_INTEGRITY.test(val)) return true
|
|
208
|
+
if (key === 'shasum' && NPM_SHASUM.test(val)) return true
|
|
188
209
|
return false
|
|
189
210
|
}
|
|
190
211
|
|
|
@@ -225,12 +246,46 @@ function stripLineComments(line) {
|
|
|
225
246
|
return line
|
|
226
247
|
}
|
|
227
248
|
|
|
249
|
+
// Remove line-level `/* ... */` block comments (opener at start of line after
|
|
250
|
+
// optional spaces/tabs) and `<!-- ... -->` HTML comments, in ONE linear pass.
|
|
251
|
+
// A regex can't do this without O(n^2): `/^[ \t]*\/\*[\s\S]*?\*\//gm` rescans to
|
|
252
|
+
// EOF at every unclosed opener, so a file full of `/*` line-starts stalled the
|
|
253
|
+
// event loop (256 KB took ~40s). An unterminated comment is left unchanged, to
|
|
254
|
+
// match the old lazy regex (which required a closing delimiter to match).
|
|
255
|
+
function stripBlockAndHtmlComments(text) {
|
|
256
|
+
let out = ''
|
|
257
|
+
let i = 0
|
|
258
|
+
const n = text.length
|
|
259
|
+
let lineStart = true
|
|
260
|
+
while (i < n) {
|
|
261
|
+
if (lineStart) {
|
|
262
|
+
let j = i
|
|
263
|
+
while (j < n && (text[j] === ' ' || text[j] === '\t')) j++
|
|
264
|
+
if (text[j] === '/' && text[j + 1] === '*') {
|
|
265
|
+
const close = text.indexOf('*/', j + 2)
|
|
266
|
+
if (close === -1) { out += text.slice(i); break }
|
|
267
|
+
i = close + 2
|
|
268
|
+
lineStart = false
|
|
269
|
+
continue
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
if (text[i] === '<' && text.startsWith('<!--', i)) {
|
|
273
|
+
const close = text.indexOf('-->', i + 4)
|
|
274
|
+
if (close === -1) { out += text.slice(i); break }
|
|
275
|
+
i = close + 3
|
|
276
|
+
lineStart = false
|
|
277
|
+
continue
|
|
278
|
+
}
|
|
279
|
+
const c = text[i]
|
|
280
|
+
out += c
|
|
281
|
+
lineStart = c === '\n'
|
|
282
|
+
i++
|
|
283
|
+
}
|
|
284
|
+
return out
|
|
285
|
+
}
|
|
286
|
+
|
|
228
287
|
function stripComments(text) {
|
|
229
|
-
|
|
230
|
-
let result = text
|
|
231
|
-
.replace(/^[ \t]*\/\*\*[\s\S]*?\*\//gm, '')
|
|
232
|
-
.replace(/^[ \t]*\/\*[\s\S]*?\*\//gm, '')
|
|
233
|
-
.replace(/<!--[\s\S]*?-->/g, '')
|
|
288
|
+
let result = stripBlockAndHtmlComments(text)
|
|
234
289
|
result = result.split('\n').map(stripLineComments).join('\n')
|
|
235
290
|
return result.replace(/\n\s*\n\s*\n/g, '\n\n')
|
|
236
291
|
}
|
|
@@ -365,6 +420,11 @@ export function compressText(text, config) {
|
|
|
365
420
|
|
|
366
421
|
let raw = cls === 'json-lined' ? stripLineNumbers(text) : text
|
|
367
422
|
|
|
423
|
+
// Integers beyond 2^53 lose precision through JSON.parse/stringify, silently
|
|
424
|
+
// corrupting numeric ids/amounts. We can't compress such JSON without changing
|
|
425
|
+
// its meaning, so forward it untouched.
|
|
426
|
+
if (jsonHasUnsafeInteger(raw)) return null
|
|
427
|
+
|
|
368
428
|
// Prune low-value fields before minifying
|
|
369
429
|
let parsedValue
|
|
370
430
|
if (config.stages.includes('prune')) {
|
|
@@ -476,8 +536,15 @@ async function compressWithFoundationModels(text, config) {
|
|
|
476
536
|
responseData += chunk.toString()
|
|
477
537
|
})
|
|
478
538
|
|
|
539
|
+
const timeoutMs = config.foundationModelsTimeout || 10000
|
|
479
540
|
const response = await new Promise((resolve, reject) => {
|
|
541
|
+
// Bound the subprocess: a hung apfel must not hang the request forever.
|
|
542
|
+
const timer = setTimeout(() => {
|
|
543
|
+
try { proc.kill('SIGKILL') } catch { /* already gone */ }
|
|
544
|
+
reject(new Error(`apfel timed out after ${timeoutMs}ms`))
|
|
545
|
+
}, timeoutMs)
|
|
480
546
|
proc.on('close', (code) => {
|
|
547
|
+
clearTimeout(timer)
|
|
481
548
|
if (code === 5) {
|
|
482
549
|
reject(new Error('FoundationModels not available (requires macOS 15+, Apple Silicon)'))
|
|
483
550
|
return
|
|
@@ -495,6 +562,7 @@ async function compressWithFoundationModels(text, config) {
|
|
|
495
562
|
})
|
|
496
563
|
|
|
497
564
|
proc.on('error', (err) => {
|
|
565
|
+
clearTimeout(timer)
|
|
498
566
|
reject(err)
|
|
499
567
|
})
|
|
500
568
|
})
|
|
@@ -581,6 +649,20 @@ export async function compressRequest(body, config, provider) {
|
|
|
581
649
|
|
|
582
650
|
const targets = provider.extract(body, { ...config, cacheSafe: config.cacheSafe !== false })
|
|
583
651
|
|
|
652
|
+
// Redaction runs FIRST (security, not compression): mask secrets in every
|
|
653
|
+
// target's text before any stage — dedup/diff/cache/llmlingua/textpress —
|
|
654
|
+
// sees it, so secrets never enter a disk cache or get shipped to a third
|
|
655
|
+
// party. We mutate target.text in place so the whole pipeline operates on
|
|
656
|
+
// the redacted body; the non-compressible case is closed in the loop below.
|
|
657
|
+
let redactedCount = 0
|
|
658
|
+
if (config.redact !== false) {
|
|
659
|
+
for (const target of targets) {
|
|
660
|
+
if (target.skip || typeof target.text !== 'string') continue
|
|
661
|
+
const r = redactText(target.text, config.redactMode)
|
|
662
|
+
if (r.count > 0) { target.text = r.text; target.redacted = true; redactedCount += r.count }
|
|
663
|
+
}
|
|
664
|
+
}
|
|
665
|
+
|
|
584
666
|
// Read-diff: session-scoped unified diff for re-reads (lossless, opt-in)
|
|
585
667
|
if (config.stages.includes('read-diff') && config.readCache && config.sessionKey && provider.name === 'anthropic') {
|
|
586
668
|
readDiffTargets(targets, body, config.sessionKey, config.readCache)
|
|
@@ -645,12 +727,17 @@ export async function compressRequest(body, config, provider) {
|
|
|
645
727
|
if (result) {
|
|
646
728
|
target.compressed = result.text
|
|
647
729
|
stats.push({ index: target.index, ...result })
|
|
730
|
+
} else if (target.redacted) {
|
|
731
|
+
// Not otherwise compressible, but it was redacted — apply() only writes
|
|
732
|
+
// back when .compressed is set, so push the redacted text through.
|
|
733
|
+
target.compressed = target.text
|
|
734
|
+
stats.push({ index: target.index, method: 'redact', originalLen: target.text.length, compressedLen: target.text.length, originalTokens: countTokens(target.text), compressedTokens: countTokens(target.text) })
|
|
648
735
|
} else {
|
|
649
736
|
stats.push({ index: target.index, skipped: 'not-compressible' })
|
|
650
737
|
}
|
|
651
738
|
}
|
|
652
739
|
provider.apply(body, targets)
|
|
653
|
-
return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats }
|
|
740
|
+
return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount }
|
|
654
741
|
}
|
|
655
742
|
|
|
656
743
|
export async function compressMessages(body, config) {
|
package/config.js
CHANGED
|
@@ -3,6 +3,7 @@ import { homedir } from 'node:os'
|
|
|
3
3
|
import { join } from 'node:path'
|
|
4
4
|
import {
|
|
5
5
|
DEFAULT_STAGES,
|
|
6
|
+
ALL_STAGES,
|
|
6
7
|
VERSION,
|
|
7
8
|
COMPRESSION_PRESETS,
|
|
8
9
|
COMPRESSION_LEVELS,
|
|
@@ -36,14 +37,17 @@ function coerceLevelInput(raw) {
|
|
|
36
37
|
const v = resolveLevel(n) ? n : null
|
|
37
38
|
return { value: v, valid: v !== null }
|
|
38
39
|
}
|
|
39
|
-
|
|
40
|
+
// Object.hasOwn, NOT `in`: `in` walks the prototype chain, so
|
|
41
|
+
// TAMP_LEVEL=constructor / toString / valueOf pass as valid and later crash
|
|
42
|
+
// stagesForLevel with an inherited function instead of falling back cleanly.
|
|
43
|
+
if (Object.hasOwn(LEVEL_ALIASES, s)) return { value: s, valid: true }
|
|
40
44
|
return { value: null, valid: false }
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
// Convert a level input (number or alias) to its canonical integer.
|
|
44
48
|
function levelInputToInt(input) {
|
|
45
49
|
if (typeof input === 'number') return input
|
|
46
|
-
if (typeof input === 'string' && input
|
|
50
|
+
if (typeof input === 'string' && Object.hasOwn(LEVEL_ALIASES, input)) return LEVEL_ALIASES[input]
|
|
47
51
|
return null
|
|
48
52
|
}
|
|
49
53
|
|
|
@@ -100,7 +104,13 @@ export function loadConfig(env = process.env, options = {}) {
|
|
|
100
104
|
|
|
101
105
|
const explicitStagesStr = get('TAMP_STAGES')
|
|
102
106
|
const explicitStages = explicitStagesStr
|
|
103
|
-
? explicitStagesStr.split(',').map(s => s.trim()).filter(Boolean)
|
|
107
|
+
? explicitStagesStr.split(',').map(s => s.trim()).filter(Boolean).filter(s => {
|
|
108
|
+
if (ALL_STAGES.includes(s)) return true
|
|
109
|
+
process.stderr.write(
|
|
110
|
+
`[tamp] unknown stage in TAMP_STAGES: ${JSON.stringify(s)} — ignoring. Valid stages: ${ALL_STAGES.join(', ')}\n`
|
|
111
|
+
)
|
|
112
|
+
return false
|
|
113
|
+
})
|
|
104
114
|
: []
|
|
105
115
|
|
|
106
116
|
const presetNameRaw = get('TAMP_COMPRESSION_PRESET')
|
|
@@ -145,13 +155,19 @@ export function loadConfig(env = process.env, options = {}) {
|
|
|
145
155
|
stages = stagesForLevel(level)
|
|
146
156
|
levelSource = chosen.source
|
|
147
157
|
} else if (presetExplicit) {
|
|
148
|
-
|
|
158
|
+
// Object.hasOwn guard: a bare `COMPRESSION_PRESETS[presetNameRaw]` lets
|
|
159
|
+
// __proto__/constructor/toString return truthy prototype objects that
|
|
160
|
+
// pass the `if (preset)` check and then crash on `...preset.stages`.
|
|
161
|
+
const preset = Object.hasOwn(COMPRESSION_PRESETS, presetNameRaw) ? COMPRESSION_PRESETS[presetNameRaw] : null
|
|
149
162
|
if (preset) {
|
|
150
163
|
stages = [...preset.stages]
|
|
151
164
|
level = typeof preset.level === 'number' ? preset.level : null
|
|
152
165
|
levelSource = 'preset-env'
|
|
153
166
|
} else {
|
|
154
|
-
// Unknown preset name — fall through to default balanced
|
|
167
|
+
// Unknown preset name — warn and fall through to default balanced
|
|
168
|
+
process.stderr.write(
|
|
169
|
+
`[tamp] unknown TAMP_COMPRESSION_PRESET=${JSON.stringify(presetNameRaw)} — expected one of: ${Object.keys(COMPRESSION_PRESETS).join(', ')}. Using balanced.\n`
|
|
170
|
+
)
|
|
155
171
|
stages = [...COMPRESSION_PRESETS.balanced.stages]
|
|
156
172
|
level = COMPRESSION_PRESETS.balanced.level
|
|
157
173
|
levelSource = 'default'
|
|
@@ -190,6 +206,11 @@ export function loadConfig(env = process.env, options = {}) {
|
|
|
190
206
|
return Object.freeze({
|
|
191
207
|
version: VERSION,
|
|
192
208
|
port: parseInt(get('TAMP_PORT'), 10) || 7778,
|
|
209
|
+
// Bind loopback by default. The proxy forwards the caller's API key to the
|
|
210
|
+
// upstream, so binding all interfaces (the old no-host default) turned any
|
|
211
|
+
// LAN peer into an authenticated relay. Set TAMP_HOST=0.0.0.0 to opt into
|
|
212
|
+
// broader binding (e.g. inside a container).
|
|
213
|
+
host: get('TAMP_HOST') || '127.0.0.1',
|
|
193
214
|
upstream: get('TAMP_UPSTREAM') || 'https://api.anthropic.com',
|
|
194
215
|
upstreams: Object.freeze({
|
|
195
216
|
anthropic: get('TAMP_UPSTREAM') || 'https://api.anthropic.com',
|
|
@@ -215,12 +236,17 @@ export function loadConfig(env = process.env, options = {}) {
|
|
|
215
236
|
levelSource,
|
|
216
237
|
outputMode,
|
|
217
238
|
outputModeDefault: defaultOutputMode && VALID_OUTPUT_MODES.has(defaultOutputMode) ? defaultOutputMode : null,
|
|
218
|
-
agent: get('TAMP_AGENT') || null,
|
|
239
|
+
agent: get('TAMP_AGENT')?.toLowerCase() || null,
|
|
219
240
|
autoDetectTaskType: parseBoolean(get('TAMP_AUTO_DETECT_TASK_TYPE'), true),
|
|
220
241
|
log: get('TAMP_LOG') !== 'false',
|
|
221
242
|
logFile: get('TAMP_LOG_FILE') || null,
|
|
222
243
|
maxBody: parseInt(get('TAMP_MAX_BODY'), 10) || 10_485_760,
|
|
223
244
|
cacheSafe: parseBoolean(get('TAMP_CACHE_SAFE'), true),
|
|
245
|
+
// Secret redaction is a security pass, not a compression stage: on by
|
|
246
|
+
// default at every level, masks secrets before any stage (incl. ones that
|
|
247
|
+
// ship text to third parties) sees the body. TAMP_REDACT=false disables it.
|
|
248
|
+
redact: parseBoolean(get('TAMP_REDACT'), true),
|
|
249
|
+
redactMode: get('TAMP_REDACT_MODE') === 'remove' ? 'remove' : 'mask',
|
|
224
250
|
llmLinguaUrl: get('TAMP_LLMLINGUA_URL') || null,
|
|
225
251
|
textpressOllamaUrl: get('TAMP_TEXTPRESS_OLLAMA_URL') || 'http://localhost:11434',
|
|
226
252
|
textpressOllamaModel: get('TAMP_TEXTPRESS_OLLAMA_MODEL') || 'qwen3.5:0.8b',
|
|
@@ -238,6 +264,7 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
|
|
|
238
264
|
# https://github.com/sliday/tamp
|
|
239
265
|
|
|
240
266
|
# TAMP_PORT=7778
|
|
267
|
+
# TAMP_HOST=127.0.0.1 # bind address; set 0.0.0.0 to expose beyond loopback
|
|
241
268
|
# TAMP_UPSTREAM=https://api.anthropic.com
|
|
242
269
|
# TAMP_UPSTREAM_OPENAI=https://api.openai.com
|
|
243
270
|
# TAMP_UPSTREAM_GEMINI=https://generativelanguage.googleapis.com
|
|
@@ -276,6 +303,8 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
|
|
|
276
303
|
# TAMP_LOG_FILE=
|
|
277
304
|
# TAMP_MAX_BODY=10485760
|
|
278
305
|
# TAMP_CACHE_SAFE=true
|
|
306
|
+
# TAMP_REDACT=true
|
|
307
|
+
# TAMP_REDACT_MODE=mask
|
|
279
308
|
# TAMP_TOKEN_COST=3
|
|
280
309
|
# TAMP_FOUNDATION_MODELS_PATH=apfel
|
|
281
310
|
# TAMP_FOUNDATION_MODELS_TIMEOUT=10000
|
package/detect.js
CHANGED
|
@@ -1,21 +1,74 @@
|
|
|
1
1
|
import { rewriteCommandOutput } from './lib/rewriters/index.js'
|
|
2
2
|
|
|
3
3
|
const LINE_NUM_RE = /^ *\d+[\t→]/
|
|
4
|
+
// Genuine line-numbered output (cat -n, the Read tool) is right-aligned, so the
|
|
5
|
+
// number carries leading padding. Requiring padding distinguishes it from
|
|
6
|
+
// unpadded numeric first columns in TSV/CSV tool output.
|
|
7
|
+
const PADDED_LINE_NUM_RE = /^ +\d+[\t→]/
|
|
4
8
|
|
|
5
9
|
export function stripLineNumbers(str) {
|
|
6
10
|
if (typeof str !== 'string') return str
|
|
7
11
|
const lines = str.split('\n')
|
|
8
12
|
if (lines.length < 2) return str
|
|
9
|
-
//
|
|
10
|
-
|
|
13
|
+
// Collect the leading numbers from the first few non-empty lines. Only treat
|
|
14
|
+
// them as line numbers when they are padded AND strictly consecutive — the
|
|
15
|
+
// invariants of real line numbering. This avoids silently dropping the first
|
|
16
|
+
// column of numeric tabular data (e.g. "1\tAlice\t30").
|
|
17
|
+
const nums = []
|
|
11
18
|
for (const line of lines.slice(0, 5)) {
|
|
12
19
|
if (line.length === 0) continue
|
|
13
|
-
if (
|
|
20
|
+
if (!PADDED_LINE_NUM_RE.test(line)) continue
|
|
21
|
+
nums.push(parseInt(line, 10))
|
|
22
|
+
}
|
|
23
|
+
if (nums.length < 2) return str
|
|
24
|
+
for (let i = 1; i < nums.length; i++) {
|
|
25
|
+
if (nums[i] !== nums[i - 1] + 1) return str
|
|
14
26
|
}
|
|
15
|
-
if (matches < 2) return str
|
|
16
27
|
return lines.map(l => l.replace(LINE_NUM_RE, '')).join('\n')
|
|
17
28
|
}
|
|
18
29
|
|
|
30
|
+
// True if the JSON text contains an integer literal outside the JS safe-integer
|
|
31
|
+
// range (|n| > 2^53-1). Such values lose precision through JSON.parse, so the
|
|
32
|
+
// minify/toon stages would silently alter a numeric id/amount — a semantic
|
|
33
|
+
// corruption. Scans number tokens outside string literals; floats are left
|
|
34
|
+
// alone (they are inherently approximate). Conservative on the safe side: when
|
|
35
|
+
// in doubt it returns true so the caller skips compression.
|
|
36
|
+
export function jsonHasUnsafeInteger(str) {
|
|
37
|
+
if (typeof str !== 'string') return false
|
|
38
|
+
let inString = false
|
|
39
|
+
let escaped = false
|
|
40
|
+
for (let i = 0; i < str.length; i++) {
|
|
41
|
+
const ch = str[i]
|
|
42
|
+
if (inString) {
|
|
43
|
+
if (escaped) escaped = false
|
|
44
|
+
else if (ch === '\\') escaped = true
|
|
45
|
+
else if (ch === '"') inString = false
|
|
46
|
+
continue
|
|
47
|
+
}
|
|
48
|
+
if (ch === '"') { inString = true; continue }
|
|
49
|
+
if (ch !== '-' && (ch < '0' || ch > '9')) continue
|
|
50
|
+
// Start of a number token. Consume the integer-part digit run.
|
|
51
|
+
let j = ch === '-' ? i + 1 : i
|
|
52
|
+
const digitStart = j
|
|
53
|
+
while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
|
|
54
|
+
const intDigits = j - digitStart
|
|
55
|
+
// Consume any fraction + exponent so their digits aren't re-scanned as a
|
|
56
|
+
// separate integer. Floats are left alone (inherently approximate).
|
|
57
|
+
let isFloat = false
|
|
58
|
+
if (str[j] === '.') { isFloat = true; j++; while (j < str.length && str[j] >= '0' && str[j] <= '9') j++ }
|
|
59
|
+
if (str[j] === 'e' || str[j] === 'E') {
|
|
60
|
+
isFloat = true; j++
|
|
61
|
+
if (str[j] === '+' || str[j] === '-') j++
|
|
62
|
+
while (j < str.length && str[j] >= '0' && str[j] <= '9') j++
|
|
63
|
+
}
|
|
64
|
+
if (!isFloat && intDigits >= 16 && !Number.isSafeInteger(Number(str.slice(i, j)))) {
|
|
65
|
+
return true
|
|
66
|
+
}
|
|
67
|
+
i = j - 1
|
|
68
|
+
}
|
|
69
|
+
return false
|
|
70
|
+
}
|
|
71
|
+
|
|
19
72
|
export function tryParseJSON(str) {
|
|
20
73
|
if (typeof str !== 'string' || str.length === 0) return { ok: false }
|
|
21
74
|
try {
|
|
@@ -29,7 +82,12 @@ export function tryParseJSON(str) {
|
|
|
29
82
|
export function isTOON(str) {
|
|
30
83
|
if (typeof str !== 'string') return false
|
|
31
84
|
const firstLine = str.trimStart().split('\n')[0]
|
|
32
|
-
|
|
85
|
+
// Anchored + bounded. The former unanchored `\w+\[\d+\]\{` was quadratic: on a
|
|
86
|
+
// long word-char run with no bracket the greedy `\w+` backtracks and, being
|
|
87
|
+
// unanchored, retries at every offset — a single-line blob (minified JSON, a
|
|
88
|
+
// long token) stalled the event loop. TOON headers begin the line, so `^`
|
|
89
|
+
// alone kills it; the {1,64}/{1,15} bounds are belt-and-suspenders.
|
|
90
|
+
return /^\[TOON\]/.test(firstLine) || /^\s*\w{1,64}\[\d{1,15}\]\{/.test(firstLine) || /^\s*\w{1,64}\[\d{1,15}\]:/.test(firstLine)
|
|
33
91
|
}
|
|
34
92
|
|
|
35
93
|
export function classifyContent(str) {
|
package/index.js
CHANGED
|
@@ -175,6 +175,13 @@ function pipeRequest(req, res, upstreamUrl, prefixChunks) {
|
|
|
175
175
|
const headers = { ...req.headers }
|
|
176
176
|
delete headers.host
|
|
177
177
|
const upstream = openUpstream(req.method, upstreamUrl, headers, res)
|
|
178
|
+
// Guard the inbound stream: a client RST mid-upload emits 'error' on `req`,
|
|
179
|
+
// and with no listener Node rethrows it and crashes the whole proxy. The
|
|
180
|
+
// buffered path already guards this; the piped passthrough must too.
|
|
181
|
+
req.on('error', (err) => {
|
|
182
|
+
log(`[tamp] client upload error: ${err.code || ''} ${err.message}`)
|
|
183
|
+
upstream.destroy()
|
|
184
|
+
})
|
|
178
185
|
if (prefixChunks) {
|
|
179
186
|
for (const chunk of prefixChunks) upstream.write(chunk)
|
|
180
187
|
}
|
|
@@ -336,7 +343,7 @@ return http.createServer(async (req, res) => {
|
|
|
336
343
|
const parsed = JSON.parse(textBody.toString('utf-8'))
|
|
337
344
|
|
|
338
345
|
const sessionKey = (config.stages?.includes('graph') || config.stages?.includes('read-diff'))
|
|
339
|
-
? deriveSessionKey(req.headers)
|
|
346
|
+
? deriveSessionKey(req.headers, parsed)
|
|
340
347
|
: null
|
|
341
348
|
const sessionBucket = config.stages?.includes('graph') && sessionKey
|
|
342
349
|
? sessionStore.getBucket(sessionKey)
|
|
@@ -406,7 +413,7 @@ function probeSidecar(config) {
|
|
|
406
413
|
req.on('error', () => {
|
|
407
414
|
sidecarAvailable = false
|
|
408
415
|
console.error('[tamp] \u26a0 llmlingua stage enabled but sidecar not running \u2014 text blocks won\u2019t compress')
|
|
409
|
-
console.error('[tamp] start with: uv run --with fastapi --with uvicorn --with llmlingua --with mlx uvicorn server:app --host
|
|
416
|
+
console.error('[tamp] start with: uv run --with fastapi --with uvicorn --with llmlingua --with mlx uvicorn server:app --host 127.0.0.1 --port 8788 --app-dir sidecar')
|
|
410
417
|
})
|
|
411
418
|
req.on('timeout', () => { req.destroy() })
|
|
412
419
|
}
|
|
@@ -415,8 +422,8 @@ const isMain = !process.argv[1]?.includes('node_modules') && process.argv[1] ===
|
|
|
415
422
|
|
|
416
423
|
if (isMain) {
|
|
417
424
|
const { config, server } = createProxy()
|
|
418
|
-
server.listen(config.port, () => {
|
|
419
|
-
console.error(`[tamp] proxy listening on http
|
|
425
|
+
server.listen(config.port, config.host, () => {
|
|
426
|
+
console.error(`[tamp] proxy listening on http://${config.host}:${config.port}`)
|
|
420
427
|
console.error(`[tamp] upstream: ${config.upstream}`)
|
|
421
428
|
console.error(`[tamp] stages: ${config.stages.join(', ')}`)
|
|
422
429
|
|
package/lib/bm25.js
CHANGED
|
@@ -133,7 +133,15 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
133
133
|
const keep = new Uint8Array(N)
|
|
134
134
|
keep[0] = 1
|
|
135
135
|
keep[N - 1] = 1
|
|
136
|
-
|
|
136
|
+
// Anchor the last NON-EMPTY line too. Tool output almost always ends with
|
|
137
|
+
// '\n', so lines[N-1] is '' — pinning only that empty terminator would let
|
|
138
|
+
// the real final line (often a trailing error or stack trace, exactly the
|
|
139
|
+
// "context anchor" this claims to preserve) be dropped as a candidate.
|
|
140
|
+
let lastReal = N - 1
|
|
141
|
+
while (lastReal > 0 && lines[lastReal] === '') lastReal--
|
|
142
|
+
keep[lastReal] = 1
|
|
143
|
+
let budgetUsed = 0
|
|
144
|
+
for (const idx of new Set([0, N - 1, lastReal])) budgetUsed += lineTokens[idx]
|
|
137
145
|
// Rough marker cost reserved per collapsed gap; we'll reconcile below.
|
|
138
146
|
const MARKER_RESERVE = 16
|
|
139
147
|
|
|
@@ -141,6 +149,7 @@ export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLine
|
|
|
141
149
|
// context preferred on ties). We pick greedily until budget is hit.
|
|
142
150
|
const candidates = []
|
|
143
151
|
for (let i = 1; i < N - 1; i++) {
|
|
152
|
+
if (keep[i]) continue // skip already-anchored lines (e.g. lastReal)
|
|
144
153
|
candidates.push({ i, score: scores[i], len: lineTokens[i] })
|
|
145
154
|
}
|
|
146
155
|
candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
|
package/lib/br-cache.js
CHANGED
|
@@ -39,7 +39,9 @@ function filePathFor(cacheDir, hash) {
|
|
|
39
39
|
}
|
|
40
40
|
|
|
41
41
|
export function createBrCache({ cacheDir = defaultCacheDir(), maxBytes = DEFAULT_MAX_BYTES, minSize = DEFAULT_MIN_SIZE } = {}) {
|
|
42
|
-
|
|
42
|
+
// 0o700: cached bodies are full tool_results (code, file contents, command
|
|
43
|
+
// output) that may contain secrets — keep them owner-only on shared machines.
|
|
44
|
+
try { mkdirSync(cacheDir, { recursive: true, mode: 0o700 }) } catch { /* ignore */ }
|
|
43
45
|
|
|
44
46
|
let hits = 0
|
|
45
47
|
let misses = 0
|
|
@@ -101,7 +103,7 @@ export function createBrCache({ cacheDir = defaultCacheDir(), maxBytes = DEFAULT
|
|
|
101
103
|
} catch { /* fall through and rewrite */ }
|
|
102
104
|
}
|
|
103
105
|
|
|
104
|
-
try { mkdirSync(shardDir, { recursive: true }) } catch { /* ignore */ }
|
|
106
|
+
try { mkdirSync(shardDir, { recursive: true, mode: 0o700 }) } catch { /* ignore */ }
|
|
105
107
|
|
|
106
108
|
let compressed
|
|
107
109
|
try {
|
|
@@ -114,7 +116,7 @@ export function createBrCache({ cacheDir = defaultCacheDir(), maxBytes = DEFAULT
|
|
|
114
116
|
|
|
115
117
|
const tmpPath = `${finalPath}.tmp`
|
|
116
118
|
try {
|
|
117
|
-
writeFileSync(tmpPath, compressed)
|
|
119
|
+
writeFileSync(tmpPath, compressed, { mode: 0o600 })
|
|
118
120
|
renameSync(tmpPath, finalPath)
|
|
119
121
|
brotliBytes = compressed.length
|
|
120
122
|
} catch {
|
package/lib/compress-file.js
CHANGED
|
@@ -23,7 +23,11 @@ const STRATEGIES = Object.freeze({
|
|
|
23
23
|
minSize: 200,
|
|
24
24
|
},
|
|
25
25
|
json: {
|
|
26
|
-
|
|
26
|
+
// No 'toon' here: this strategy rewrites a file in place that keeps its
|
|
27
|
+
// .json extension, so the output must stay valid JSON. TOON is not JSON.
|
|
28
|
+
// minify (lossless) and prune (documented npm-metadata strip) keep it
|
|
29
|
+
// parseable.
|
|
30
|
+
stages: ['minify', 'prune'],
|
|
27
31
|
preserveStructure: false,
|
|
28
32
|
minSize: 100,
|
|
29
33
|
},
|
package/lib/redact.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// --- Secret redaction (security, not compression) ---
|
|
2
|
+
//
|
|
3
|
+
// Masks or removes high-confidence secrets in outbound tool_result bodies
|
|
4
|
+
// before anything leaves the machine — and before any stage that ships text
|
|
5
|
+
// to a third party (llmlingua sidecar, textpress/OpenRouter) ever sees it.
|
|
6
|
+
//
|
|
7
|
+
// Design bias: precision over recall. A false positive corrupts legitimate
|
|
8
|
+
// content the model needs; a false negative leaks a secret. We only match
|
|
9
|
+
// patterns with a recognizable shape (provider-prefixed keys, PEM blocks,
|
|
10
|
+
// JWTs) plus assignment lines whose KEY name is unambiguously a secret. We do
|
|
11
|
+
// NOT do entropy guessing — too noisy for code/log payloads.
|
|
12
|
+
//
|
|
13
|
+
// Why: https://github.com/sliday/tamp/issues/6
|
|
14
|
+
|
|
15
|
+
// Each rule: a global regex and a short label used in the mask marker. Where a
|
|
16
|
+
// rule needs to keep surrounding structure (assignments), it captures the
|
|
17
|
+
// secret in group 1 and rebuilds the line via `replace`.
|
|
18
|
+
const TOKEN_RULES = [
|
|
19
|
+
{ label: 'aws-access-key', re: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[0-9A-Z]{16}\b/g },
|
|
20
|
+
{ label: 'github-token', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/g },
|
|
21
|
+
{ label: 'github-pat', re: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/g },
|
|
22
|
+
{ label: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
|
|
23
|
+
{ label: 'openai-key', re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/g },
|
|
24
|
+
{ label: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
|
|
25
|
+
{ label: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g },
|
|
26
|
+
{ label: 'stripe-key', re: /\b[rs]k_live_[A-Za-z0-9]{20,}\b/g },
|
|
27
|
+
{ label: 'google-oauth', re: /\bya29\.[A-Za-z0-9_-]{20,}\b/g },
|
|
28
|
+
{ label: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g },
|
|
29
|
+
{ label: 'private-key', re: /-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/g },
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
// Assignment lines from .env / shell / config where the KEY name names a
|
|
33
|
+
// secret. We mask only the VALUE, keeping the key so the model still sees the
|
|
34
|
+
// variable exists.
|
|
35
|
+
//
|
|
36
|
+
// Precision rules learned the hard way:
|
|
37
|
+
// - Case-SENSITIVE uppercase keys only. Real secret env vars are UPPER_SNAKE
|
|
38
|
+
// (`API_KEY`, `AUTH_TOKEN`); camelCase like `apiKey = getKey()` is code,
|
|
39
|
+
// not a secret, and must not match.
|
|
40
|
+
// - The secret word is anchored to the END of the key (immediately before
|
|
41
|
+
// the `=`/`:`), so `AUTHOR=`, `COMPASS=`, `PASSWORD_HINT=` don't trip it.
|
|
42
|
+
// - Value must be 6+ non-space chars to skip empty/placeholder assignments.
|
|
43
|
+
// - The key prefix is BOUNDED (`{0,64}`) and word-boundary anchored. An
|
|
44
|
+
// unbounded `[A-Z0-9_]*` before the secret-word alternation is quadratic:
|
|
45
|
+
// on a long uppercase run with no match the engine retries every split
|
|
46
|
+
// point, so a big CONSTANT_CASE/hex tool_result would stall the proxy
|
|
47
|
+
// event loop for seconds. Real env-var names are well under 64 chars.
|
|
48
|
+
const SECRET_KEY = '\\b[A-Z0-9_]{0,64}(?:SECRET|TOKEN|PASSWORD|PASSWD|API[_-]?KEY|ACCESS[_-]?KEY|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET|CREDENTIALS?|AUTH|PASSPHRASE|SESSION[_-]?KEY)'
|
|
49
|
+
const ASSIGN_RE = new RegExp(
|
|
50
|
+
`((?:export\\s+)?${SECRET_KEY})(\\s*[:=]\\s*)(["']?)([^\\s"']{6,})(["']?)`,
|
|
51
|
+
'g'
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
function marker(label, mode) {
|
|
55
|
+
return mode === 'remove' ? '' : `‹redacted:${label}›`
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Returns { text, count }. `count` is the number of secrets masked/removed.
|
|
59
|
+
// When count is 0 the returned text is referentially the input (no allocation
|
|
60
|
+
// churn for the common no-secret case).
|
|
61
|
+
export function redactText(text, mode = 'mask') {
|
|
62
|
+
if (typeof text !== 'string' || text.length === 0) return { text, count: 0 }
|
|
63
|
+
|
|
64
|
+
let count = 0
|
|
65
|
+
let out = text
|
|
66
|
+
|
|
67
|
+
for (const { label, re } of TOKEN_RULES) {
|
|
68
|
+
out = out.replace(re, () => { count++; return marker(label, mode) })
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
out = out.replace(ASSIGN_RE, (m, key, sep, q1, _val, q2) => {
|
|
72
|
+
count++
|
|
73
|
+
if (mode === 'remove') return `${key}${sep}`
|
|
74
|
+
return `${key}${sep}${q1}${marker('secret', mode)}${q2}`
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
return count > 0 ? { text: out, count } : { text, count: 0 }
|
|
78
|
+
}
|
|
@@ -32,11 +32,13 @@ export default {
|
|
|
32
32
|
if (BUILDKIT_TRANSFER.test(line)) continue;
|
|
33
33
|
if (SPINNER_BRAILLE.test(line)) continue;
|
|
34
34
|
if (LEGACY_PROGRESS.test(line)) continue;
|
|
35
|
-
// BuildKit "in-progress"
|
|
36
|
-
// marker
|
|
37
|
-
//
|
|
38
|
-
// stdout
|
|
39
|
-
|
|
35
|
+
// BuildKit "in-progress" step frames carry a [x/y] step counter and end
|
|
36
|
+
// with a fractional-second marker; they are re-emitted as DONE later, so
|
|
37
|
+
// strip the non-DONE variants. Requiring [x/y] avoids eating a RUN step's
|
|
38
|
+
// own stdout (prefixed "#N <timestamp> ...", no counter), which can
|
|
39
|
+
// legitimately end in a duration (e.g. "webpack compiled in 2.5s"). The
|
|
40
|
+
// error/warning/fail guard stays as belt-and-suspenders.
|
|
41
|
+
if (/^#\d+\s+\[\d+\/\d+\]\s.*\s\d+\.\d+s$/.test(line) && !/\sDONE\b/.test(line) && !/\b(error|warning|fail)/i.test(line)) continue;
|
|
40
42
|
out.push(line);
|
|
41
43
|
}
|
|
42
44
|
|
|
@@ -10,7 +10,6 @@
|
|
|
10
10
|
// Keeps: error/warn lines, final "added N packages in Ns", vulnerability
|
|
11
11
|
// counts, "up to date", package trees.
|
|
12
12
|
|
|
13
|
-
const SPINNER_CHARS = /[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]/;
|
|
14
13
|
const PROGRESS_LINE = /^\s*[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]\s+/;
|
|
15
14
|
const DOTS_ONLY = /^\s*\.{3,}\s*$/;
|
|
16
15
|
const NPM_SIG = /(^|\n)(npm (install|i|ci|run)\b|added \d+ packages?|up to date|changed \d+ packages?|removed \d+ packages?)/;
|
|
@@ -38,8 +37,11 @@ export default {
|
|
|
38
37
|
for (let i = 0; i < lines.length; i++) {
|
|
39
38
|
const line = lines[i];
|
|
40
39
|
|
|
41
|
-
// Drop spinner lines and sole-dot lines.
|
|
42
|
-
|
|
40
|
+
// Drop leading-spinner progress lines and sole-dot lines. Only anchored
|
|
41
|
+
// PROGRESS_LINE (spinner glyph at line start) — an unanchored glyph test
|
|
42
|
+
// deleted whole lines of child-tool output that merely contained a
|
|
43
|
+
// braille char anywhere (e.g. `⠋ 2 warnings, 1 error`), losing signal.
|
|
44
|
+
if (PROGRESS_LINE.test(line)) continue;
|
|
43
45
|
if (DOTS_ONLY.test(line)) {
|
|
44
46
|
if (lastWasDots) continue;
|
|
45
47
|
lastWasDots = true;
|
|
@@ -7,7 +7,12 @@
|
|
|
7
7
|
// the short test summary info block, and the final "=== N passed in Xs ===".
|
|
8
8
|
|
|
9
9
|
const COLLECTED = /^collected\s+\d+\s+items?\b/;
|
|
10
|
-
|
|
10
|
+
// Per-file dot-progress: "<path>.py[::node] <status run> [ NN% ]" (percent
|
|
11
|
+
// optional). Anchored to a .py path prefix so a traceback/assertion line that
|
|
12
|
+
// merely ends in "..." is NOT mistaken for progress and silently dropped.
|
|
13
|
+
const DOT_PROGRESS = /^\S*\.py(::\S+)?\s+[.FsxE]{2,}\s*(\[\s*\d+%\s*\])?\s*$/;
|
|
14
|
+
// Wrapped progress continuation: a bare status run followed by a percent.
|
|
15
|
+
const PCT_PROGRESS = /^\s*[.FsxE]{2,}\s*\[\s*\d+%\s*\]\s*$/;
|
|
11
16
|
const SESSION_START = /^platform\s+\S+\s+--\s+Python\s+\d/;
|
|
12
17
|
const ROOTDIR = /^rootdir:\s+/;
|
|
13
18
|
const PLUGINS = /^plugins:\s+/;
|
|
@@ -33,9 +38,10 @@ export default {
|
|
|
33
38
|
if (ROOTDIR.test(line)) continue;
|
|
34
39
|
if (PLUGINS.test(line)) continue;
|
|
35
40
|
if (CACHEDIR.test(line)) continue;
|
|
36
|
-
// Drop
|
|
37
|
-
// (
|
|
38
|
-
|
|
41
|
+
// Drop per-file / wrapped dot-progress lines. Both patterns are anchored
|
|
42
|
+
// (path prefix or trailing percent), so traceback/assertion lines that end
|
|
43
|
+
// in "..." are preserved without needing a fragile keyword guard.
|
|
44
|
+
if (DOT_PROGRESS.test(line) || PCT_PROGRESS.test(line)) continue;
|
|
39
45
|
out.push(line);
|
|
40
46
|
}
|
|
41
47
|
|
package/metadata.js
CHANGED
|
@@ -104,7 +104,7 @@ export const STAGE_DESCRIPTIONS = Object.freeze({
|
|
|
104
104
|
'foundation-models': 'Apple Intelligence neural compression (macOS 15+, Apple Silicon, 100% local)',
|
|
105
105
|
dedup: 'Deduplicate identical tool_results',
|
|
106
106
|
diff: 'Replace similar re-reads with diffs',
|
|
107
|
-
'read-diff': 'Session-scoped unified diff for re-reads (lossless,
|
|
107
|
+
'read-diff': 'Session-scoped unified diff for re-reads (lossless, on by default in balanced+)',
|
|
108
108
|
prune: 'Strip lockfile hashes & npm metadata',
|
|
109
109
|
'strip-comments': 'Remove code comments (lossy)',
|
|
110
110
|
textpress: 'LLM semantic compression (Ollama/OpenRouter)',
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"sidecar/server.py",
|
|
15
15
|
"sidecar/requirements.txt"
|
|
16
16
|
],
|
|
17
|
-
"version": "0.8.
|
|
17
|
+
"version": "0.8.15",
|
|
18
18
|
"description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
|
|
19
19
|
"type": "module",
|
|
20
20
|
"main": "index.js",
|
package/providers.js
CHANGED
|
@@ -21,14 +21,14 @@ function findLatestEligibleGroup(items, extractTargets) {
|
|
|
21
21
|
|
|
22
22
|
function extractAnthropicMessageTargets(msg, mi) {
|
|
23
23
|
const targets = []
|
|
24
|
-
if (msg.role !== 'user') return targets
|
|
24
|
+
if (!msg || msg.role !== 'user') return targets
|
|
25
25
|
|
|
26
26
|
if (typeof msg.content === 'string') {
|
|
27
27
|
targets.push({ path: ['messages', mi, 'content'], text: msg.content, index: mi })
|
|
28
28
|
} else if (Array.isArray(msg.content)) {
|
|
29
29
|
for (let i = 0; i < msg.content.length; i++) {
|
|
30
30
|
const block = msg.content[i]
|
|
31
|
-
if (block.type !== 'tool_result') continue
|
|
31
|
+
if (!block || block.type !== 'tool_result') continue
|
|
32
32
|
if (block.is_error) {
|
|
33
33
|
targets.push({ skip: 'error', index: i })
|
|
34
34
|
continue
|
|
@@ -39,7 +39,7 @@ function extractAnthropicMessageTargets(msg, mi) {
|
|
|
39
39
|
} else if (Array.isArray(block.content)) {
|
|
40
40
|
for (let j = 0; j < block.content.length; j++) {
|
|
41
41
|
const sub = block.content[j]
|
|
42
|
-
if (sub.type === 'text') {
|
|
42
|
+
if (sub && sub.type === 'text') {
|
|
43
43
|
targets.push({ path: ['messages', mi, 'content', i, 'content', j, 'text'], text: sub.text, index: i })
|
|
44
44
|
}
|
|
45
45
|
}
|
|
@@ -130,18 +130,16 @@ function findAnthropicReferences(body) {
|
|
|
130
130
|
} else if (typeof block.input === 'string') {
|
|
131
131
|
scanStringForRefs(block.input, ['messages', mi, 'content', bi, 'input'], out)
|
|
132
132
|
}
|
|
133
|
-
} else if (block.type === 'tool_result') {
|
|
134
|
-
if (typeof block.content === 'string') {
|
|
135
|
-
scanStringForRefs(block.content, ['messages', mi, 'content', bi, 'content'], out)
|
|
136
|
-
} else if (Array.isArray(block.content)) {
|
|
137
|
-
for (let ci = 0; ci < block.content.length; ci++) {
|
|
138
|
-
const sub = block.content[ci]
|
|
139
|
-
if (sub?.type === 'text' && typeof sub.text === 'string') {
|
|
140
|
-
scanStringForRefs(sub.text, ['messages', mi, 'content', bi, 'content', ci, 'text'], out)
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
133
|
}
|
|
134
|
+
// NOTE: tool_result content is deliberately NOT scanned for markers. A
|
|
135
|
+
// marker is only honored when the MODEL quotes it (assistant text or
|
|
136
|
+
// tool_use.input) — that is the disclosure contract. tool_result bodies
|
|
137
|
+
// are untrusted tool output (fetched pages, file reads); scanning them
|
|
138
|
+
// would (1) let attacker-controlled content forge a <tamp-ref:v1:...>
|
|
139
|
+
// marker and pull an arbitrary body out of the shared br-cache into the
|
|
140
|
+
// outbound request (cross-conversation cache-pull + existence oracle),
|
|
141
|
+
// and (2) auto-rehydrate a disclosure summary's own marker every turn,
|
|
142
|
+
// defeating the compression. See rehydrateReferences.
|
|
145
143
|
}
|
|
146
144
|
}
|
|
147
145
|
return out
|
|
@@ -221,7 +219,7 @@ const anthropic = {
|
|
|
221
219
|
return method === 'POST' && url.startsWith('/v1/messages')
|
|
222
220
|
},
|
|
223
221
|
extract(body, config = {}) {
|
|
224
|
-
if (!body?.messages
|
|
222
|
+
if (!Array.isArray(body?.messages) || !body.messages.length) return []
|
|
225
223
|
if (config.cacheSafe) {
|
|
226
224
|
return findLatestEligibleGroup(body.messages, extractAnthropicMessageTargets)
|
|
227
225
|
}
|
|
@@ -251,7 +249,7 @@ const anthropic = {
|
|
|
251
249
|
const OPENAI_COMPAT_SKIP_BLOCK_TYPES = new Set(['thinking', 'partial'])
|
|
252
250
|
|
|
253
251
|
function extractOpenAIChatTargets(msg, i) {
|
|
254
|
-
if (msg.role !== 'tool') return []
|
|
252
|
+
if (!msg || msg.role !== 'tool') return []
|
|
255
253
|
// Some OpenAI-compat providers (Kimi, etc.) emit non-string tool content
|
|
256
254
|
// with typed blocks like { type: 'thinking' }. Skip those.
|
|
257
255
|
if (typeof msg.content !== 'string') {
|
|
@@ -285,13 +283,13 @@ const openai = {
|
|
|
285
283
|
return url
|
|
286
284
|
},
|
|
287
285
|
extract(body, config = {}) {
|
|
288
|
-
if (!body?.messages
|
|
286
|
+
if (!Array.isArray(body?.messages) || !body.messages.length) return []
|
|
289
287
|
|
|
290
288
|
if (config.cacheSafe) {
|
|
291
289
|
const targets = []
|
|
292
290
|
for (let i = body.messages.length - 1; i >= 0; i--) {
|
|
293
291
|
const msg = body.messages[i]
|
|
294
|
-
if (msg.role !== 'tool') break
|
|
292
|
+
if (!msg || msg.role !== 'tool') break
|
|
295
293
|
targets.unshift(...extractOpenAIChatTargets(msg, i))
|
|
296
294
|
}
|
|
297
295
|
return targets
|
|
@@ -393,11 +391,11 @@ const moonshot = {
|
|
|
393
391
|
|
|
394
392
|
function extractGeminiContentTargets(content, ci) {
|
|
395
393
|
const targets = []
|
|
396
|
-
if (!content.parts?.length) return targets
|
|
394
|
+
if (!content || !content.parts?.length) return targets
|
|
397
395
|
|
|
398
396
|
for (let pi = 0; pi < content.parts.length; pi++) {
|
|
399
397
|
const part = content.parts[pi]
|
|
400
|
-
if (!part.functionResponse?.response) continue
|
|
398
|
+
if (!part || !part.functionResponse?.response) continue
|
|
401
399
|
const resp = part.functionResponse.response
|
|
402
400
|
const text = typeof resp === 'string' ? resp : JSON.stringify(resp, null, 2)
|
|
403
401
|
targets.push({
|
|
@@ -417,7 +415,7 @@ const gemini = {
|
|
|
417
415
|
return method === 'POST' && url.includes('generateContent')
|
|
418
416
|
},
|
|
419
417
|
extract(body, config = {}) {
|
|
420
|
-
if (!body?.contents
|
|
418
|
+
if (!Array.isArray(body?.contents) || !body.contents.length) return []
|
|
421
419
|
if (config.cacheSafe) {
|
|
422
420
|
return findLatestEligibleGroup(body.contents, extractGeminiContentTargets)
|
|
423
421
|
}
|
package/session-graph.js
CHANGED
|
@@ -55,11 +55,42 @@ export function createSessionBucket({ brCache = null } = {}) {
|
|
|
55
55
|
}
|
|
56
56
|
}
|
|
57
57
|
|
|
58
|
-
|
|
58
|
+
// A conversation-stable fingerprint: the system prompt plus the first turn.
|
|
59
|
+
// Claude Code (and peers) resend an immutable opening every turn, so this is
|
|
60
|
+
// constant within a conversation but differs across conversations — even when
|
|
61
|
+
// they share one API token. Empty when no body is available (callers that omit
|
|
62
|
+
// it get the legacy token-only key).
|
|
63
|
+
function conversationSeed(body) {
|
|
64
|
+
if (!body || typeof body !== 'object') return ''
|
|
65
|
+
const turns = body.messages || body.contents || body.input
|
|
66
|
+
const first = Array.isArray(turns) && turns.length ? turns[0] : null
|
|
67
|
+
if (first == null && body.system === undefined) return ''
|
|
68
|
+
try {
|
|
69
|
+
// Hash system and the first turn INDEPENDENTLY. A single slice over the
|
|
70
|
+
// combined JSON let a long system prompt (Claude Code ships a multi-KB one)
|
|
71
|
+
// crowd the conversation-specific first turn past the cutoff, collapsing
|
|
72
|
+
// distinct conversations onto one key — the exact cross-conversation leak
|
|
73
|
+
// this seed exists to prevent. Per-part hashes keep the seed bounded + stable.
|
|
74
|
+
const sys = createHash('sha256').update(JSON.stringify(body.system ?? null)).digest('hex')
|
|
75
|
+
const frst = createHash('sha256').update(JSON.stringify(first ?? null)).digest('hex')
|
|
76
|
+
return `${sys}:${frst}`
|
|
77
|
+
} catch {
|
|
78
|
+
return ''
|
|
79
|
+
}
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
// Session key scopes the read-diff / graph caches. It MUST be conversation-
|
|
83
|
+
// specific: keying on the token alone let content from one conversation be
|
|
84
|
+
// referenced (read-diff, graph) in another that shares the token, emitting a
|
|
85
|
+
// diff/marker against a base the second conversation's model never saw. A
|
|
86
|
+
// too-unique seed merely disables cross-request dedup (full content is
|
|
87
|
+
// forwarded) — the safe failure direction.
|
|
88
|
+
export function deriveSessionKey(headers, body) {
|
|
59
89
|
if (!headers) return null
|
|
60
90
|
const auth = headers.authorization || headers['x-api-key'] || ''
|
|
61
91
|
if (!auth) return null
|
|
62
|
-
|
|
92
|
+
const seed = conversationSeed(body)
|
|
93
|
+
return createHash('sha256').update(`${auth} ${seed}`).digest('hex').slice(0, 16)
|
|
63
94
|
}
|
|
64
95
|
|
|
65
96
|
export function graphDeduplicateTargets(targets, bucket, { minBytes = MIN_BYTES } = {}) {
|
package/stats.js
CHANGED
|
@@ -13,7 +13,12 @@ const c = {
|
|
|
13
13
|
// (Gemini and OpenAI-compatible gateways) carry the API key in the query
|
|
14
14
|
// string (?key=, ?api_key=), which would otherwise land in stdout, log
|
|
15
15
|
// files, and the systemd journal. Path-relative URLs only — no base needed.
|
|
16
|
-
|
|
16
|
+
// Keyword (substring) match on the param NAME, not an exact list. The old
|
|
17
|
+
// anchored `^(key|api_key|apikey|access_token|token)$` leaked `refresh_token`,
|
|
18
|
+
// `id_token`, `client_secret`, `password`, `session`, `sig`/`signature`, `auth`
|
|
19
|
+
// to stdout / TAMP_LOG_FILE / the systemd journal. Over-redacting a benign
|
|
20
|
+
// param name in a log is harmless; leaking a credential is not.
|
|
21
|
+
const SECRET_QUERY_PARAM = /key|token|secret|password|passwd|sig|auth|session|credential/i
|
|
17
22
|
export function redactUrl(url) {
|
|
18
23
|
if (typeof url !== 'string') return url
|
|
19
24
|
const q = url.indexOf('?')
|
|
@@ -51,7 +56,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
|
|
|
51
56
|
const totalComp = compressed.reduce((a, s) => a + s.compressedLen, 0)
|
|
52
57
|
const totalOrigTok = compressed.reduce((a, s) => a + (s.originalTokens || 0), 0)
|
|
53
58
|
const totalCompTok = compressed.reduce((a, s) => a + (s.compressedTokens || 0), 0)
|
|
54
|
-
const pct = (((totalOrig - totalComp) / totalOrig) * 100).toFixed(1)
|
|
59
|
+
const pct = (totalOrig > 0 ? ((totalOrig - totalComp) / totalOrig) * 100 : 0).toFixed(1)
|
|
55
60
|
const saved = totalOrig - totalComp
|
|
56
61
|
const tokSaved = totalOrigTok - totalCompTok
|
|
57
62
|
|
|
@@ -60,7 +65,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
|
|
|
60
65
|
lines.push(`[tamp] ${c.cyan}${label}${c.reset} ${path}${sizeInfo} ${c.green}— ${n} block${n !== 1 ? 's' : ''} compressed, -${pct}%${c.reset}`)
|
|
61
66
|
|
|
62
67
|
for (const s of compressed) {
|
|
63
|
-
const sPct = (((s.originalLen - s.compressedLen) / s.originalLen) * 100).toFixed(1)
|
|
68
|
+
const sPct = (s.originalLen > 0 ? ((s.originalLen - s.compressedLen) / s.originalLen) * 100 : 0).toFixed(1)
|
|
64
69
|
const tokInfo = s.originalTokens ? ` ${c.dim}${s.originalTokens}→${s.compressedTokens} tok${c.reset}` : ''
|
|
65
70
|
lines.push(`[tamp] ${c.dim}block[${s.index}]${c.reset} ${fmtSize(s.originalLen)}→${fmtSize(s.compressedLen)} ${c.green}-${sPct}%${c.reset}${tokInfo} ${c.dim}[${s.method}]${c.reset}`)
|
|
66
71
|
}
|