@sliday/tamp 0.8.7 → 0.8.16
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +6 -1
- package/bin/tamp.js +1 -1
- package/compress.js +92 -16
- package/config.js +23 -4
- package/detect.js +48 -1
- package/index.js +9 -2
- package/lib/bm25.js +344 -45
- package/lib/code-chunks.js +269 -0
- package/lib/redact.js +78 -0
- package/lib/rewriters/commands/docker.js +7 -5
- package/lib/rewriters/commands/npm.js +5 -3
- package/lib/rewriters/commands/pytest.js +10 -4
- package/metadata.js +1 -1
- package/package.json +3 -1
- package/providers.js +13 -15
- package/session-graph.js +8 -1
- package/stats.js +8 -3
|
@@ -0,0 +1,269 @@
|
|
|
1
|
+
// Block structure detection for code bodies (Phase 7).
|
|
2
|
+
//
|
|
3
|
+
// `bm25-trim` drops individual lines. On code that produces syntactically
|
|
4
|
+
// broken output: a kept line from inside a function body, with its signature
|
|
5
|
+
// and closing brace gone, tells the model almost nothing. This module finds
|
|
6
|
+
// block ranges so the trimmer can enforce a simple invariant — keep a line,
|
|
7
|
+
// keep the headers and closers of every block enclosing it.
|
|
8
|
+
//
|
|
9
|
+
// Deliberately not tree-sitter. That is a native dependency, and Tamp ships a
|
|
10
|
+
// pure-JS package with a tight `files:` list. Two cheap detectors cover the
|
|
11
|
+
// languages coding agents actually read, and both refuse to guess: when the
|
|
12
|
+
// structure does not check out, `detectBlocks` returns null and the caller
|
|
13
|
+
// falls back to plain line trimming.
|
|
14
|
+
|
|
15
|
+
// --- Language selection ---
|
|
16
|
+
|
|
17
|
+
const BRACE_EXT = new Set([
|
|
18
|
+
'js', 'jsx', 'mjs', 'cjs', 'ts', 'tsx', 'mts', 'cts',
|
|
19
|
+
'java', 'c', 'h', 'cc', 'cpp', 'hpp', 'cs', 'go', 'rs',
|
|
20
|
+
'swift', 'kt', 'kts', 'scala', 'php', 'dart', 'zig',
|
|
21
|
+
])
|
|
22
|
+
const INDENT_EXT = new Set(['py', 'pyi', 'yaml', 'yml'])
|
|
23
|
+
|
|
24
|
+
export function languageForPath(path) {
|
|
25
|
+
if (typeof path !== 'string' || path.length === 0) return null
|
|
26
|
+
const dot = path.lastIndexOf('.')
|
|
27
|
+
if (dot < 0) return null
|
|
28
|
+
const ext = path.slice(dot + 1).toLowerCase()
|
|
29
|
+
if (BRACE_EXT.has(ext)) return 'brace'
|
|
30
|
+
if (INDENT_EXT.has(ext)) return 'indent'
|
|
31
|
+
return null
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
// Path-based detection only works on Anthropic requests (see
|
|
35
|
+
// lib/path-extract.js). Everywhere else we sniff the body itself.
|
|
36
|
+
const PY_BLOCK_RE = /^\s*(?:async\s+)?(?:def|class|if|for|while|with|try|elif|else|except|finally)\b.*:\s*$/
|
|
37
|
+
|
|
38
|
+
export function sniffLanguage(lines) {
|
|
39
|
+
if (!Array.isArray(lines) || lines.length < 10) return null
|
|
40
|
+
let braceOpens = 0
|
|
41
|
+
let pyBlocks = 0
|
|
42
|
+
const sample = Math.min(lines.length, 400)
|
|
43
|
+
for (let i = 0; i < sample; i++) {
|
|
44
|
+
const line = lines[i]
|
|
45
|
+
if (typeof line !== 'string') continue
|
|
46
|
+
if (/\{\s*$/.test(line)) braceOpens += 1
|
|
47
|
+
if (PY_BLOCK_RE.test(line)) pyBlocks += 1
|
|
48
|
+
}
|
|
49
|
+
// A low bar is safe: the brace detector independently returns null unless
|
|
50
|
+
// the whole body balances, so a wrong guess on a log file self-corrects.
|
|
51
|
+
if (braceOpens >= 3 && braceOpens > pyBlocks) return 'brace'
|
|
52
|
+
if (pyBlocks >= 3) return 'indent'
|
|
53
|
+
return null
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
// --- Shared helpers ---
|
|
57
|
+
|
|
58
|
+
export function indentWidth(line) {
|
|
59
|
+
if (typeof line !== 'string') return 0
|
|
60
|
+
let n = 0
|
|
61
|
+
while (n < line.length && (line[n] === ' ' || line[n] === '\t')) n += 1
|
|
62
|
+
return n
|
|
63
|
+
}
|
|
64
|
+
|
|
65
|
+
// --- Brace detector ---
|
|
66
|
+
//
|
|
67
|
+
// One left-to-right character scan carrying string AND comment state (F2).
|
|
68
|
+
//
|
|
69
|
+
// The previous version detected `/*` on the RAW line before stripping strings,
|
|
70
|
+
// so any `/*` substring — a URL path, a glob, prose about comment syntax —
|
|
71
|
+
// flipped the scanner into block-comment state and swallowed every following
|
|
72
|
+
// line's braces until an unrelated `*/`. It fired on this repo's own
|
|
73
|
+
// compress.js:258 and config.js:220, silently disabling block-aware trimming
|
|
74
|
+
// for 2 of 6 source files. Injecting one line into a 214KB body flipped 465
|
|
75
|
+
// detected blocks to 0 and produced 72 unclosed braces in the output.
|
|
76
|
+
//
|
|
77
|
+
// It also missed a complete `/* } */` on one line, because it declined to
|
|
78
|
+
// enter comment state when `*/` followed on the same line, and the regex-based
|
|
79
|
+
// stripper never handled `/* */` at all.
|
|
80
|
+
//
|
|
81
|
+
// Scanning once, left to right, makes both cases fall out for free: a `/*`
|
|
82
|
+
// inside a string is never seen as a comment opener because we are in string
|
|
83
|
+
// state when we reach it.
|
|
84
|
+
//
|
|
85
|
+
// Returns null when braces do not balance, or when comment state is still open
|
|
86
|
+
// at end of input — the honest signal that the scan mis-parsed something
|
|
87
|
+
// (regex literals and JSX are still not modelled).
|
|
88
|
+
|
|
89
|
+
function detectBraceBlocks(lines) {
|
|
90
|
+
const blocks = []
|
|
91
|
+
const open = []
|
|
92
|
+
let inBlockComment = false
|
|
93
|
+
|
|
94
|
+
for (let i = 0; i < lines.length; i++) {
|
|
95
|
+
const line = lines[i]
|
|
96
|
+
if (typeof line !== 'string') return null
|
|
97
|
+
|
|
98
|
+
let quote = null // "'", '"' or '`' while inside a string literal
|
|
99
|
+
let escaped = false
|
|
100
|
+
|
|
101
|
+
for (let c = 0; c < line.length; c++) {
|
|
102
|
+
const ch = line[c]
|
|
103
|
+
|
|
104
|
+
if (inBlockComment) {
|
|
105
|
+
if (ch === '*' && line[c + 1] === '/') { inBlockComment = false; c += 1 }
|
|
106
|
+
continue
|
|
107
|
+
}
|
|
108
|
+
if (quote !== null) {
|
|
109
|
+
if (escaped) { escaped = false; continue }
|
|
110
|
+
if (ch === '\\') { escaped = true; continue }
|
|
111
|
+
if (ch === quote) quote = null
|
|
112
|
+
continue
|
|
113
|
+
}
|
|
114
|
+
if (ch === '"' || ch === "'" || ch === '`') { quote = ch; continue }
|
|
115
|
+
if (ch === '/' && line[c + 1] === '*') { inBlockComment = true; c += 1; continue }
|
|
116
|
+
if (ch === '/' && line[c + 1] === '/') break // line comment: rest of line is noise
|
|
117
|
+
if (ch === '#') break // ditto for shell/python-style comments
|
|
118
|
+
|
|
119
|
+
if (ch === '{') {
|
|
120
|
+
// The header is the line the brace opens on. A brace that opens and
|
|
121
|
+
// closes on the same line yields a zero-length body and is dropped
|
|
122
|
+
// below, so object literals inline in an argument list cost nothing.
|
|
123
|
+
open.push(i)
|
|
124
|
+
} else if (ch === '}') {
|
|
125
|
+
const headerLine = open.pop()
|
|
126
|
+
if (headerLine === undefined) return null // unbalanced — give up
|
|
127
|
+
if (i > headerLine + 1) {
|
|
128
|
+
blocks.push({ headerLine, startLine: headerLine + 1, endLine: i - 1, closeLine: i })
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
// Single-quoted and double-quoted strings do not span lines in any language
|
|
133
|
+
// this detector claims; reset so one stray quote cannot swallow the file.
|
|
134
|
+
// Backtick state deliberately carries over (template literals are multiline).
|
|
135
|
+
if (quote === '"' || quote === "'") quote = null
|
|
136
|
+
}
|
|
137
|
+
if (open.length > 0 || inBlockComment) return null
|
|
138
|
+
return blocks
|
|
139
|
+
}
|
|
140
|
+
|
|
141
|
+
// --- Indent detector ---
|
|
142
|
+
//
|
|
143
|
+
// A block opens on a line ending in `:` and runs while indentation exceeds the
|
|
144
|
+
// header's. Blank lines belong to the block; they do not close it.
|
|
145
|
+
//
|
|
146
|
+
// Built in ONE pass with an indent stack (C3). The previous version scanned
|
|
147
|
+
// forward from every header to find its end, and because blank lines were
|
|
148
|
+
// skipped with `continue` rather than terminating the scan, every enclosing
|
|
149
|
+
// header re-walked the same blank field: O(headers x blank_lines), unbounded by
|
|
150
|
+
// body bytes since a blank line costs one byte but is visited once per open
|
|
151
|
+
// block. Measured 2000 headers over 200k blank lines (2.2MB) at 10783ms.
|
|
152
|
+
|
|
153
|
+
function detectIndentBlocks(lines) {
|
|
154
|
+
const blocks = []
|
|
155
|
+
const stack = [] // { headerLine, indent }, outermost first
|
|
156
|
+
let lastContentLine = -1
|
|
157
|
+
|
|
158
|
+
for (let i = 0; i < lines.length; i++) {
|
|
159
|
+
const line = lines[i]
|
|
160
|
+
if (typeof line !== 'string') return null
|
|
161
|
+
if (isBlank(line)) continue // blanks belong to whatever block is open
|
|
162
|
+
|
|
163
|
+
const indent = indentWidth(line)
|
|
164
|
+
while (stack.length > 0 && indent <= stack[stack.length - 1].indent) {
|
|
165
|
+
const done = stack.pop()
|
|
166
|
+
if (lastContentLine > done.headerLine) {
|
|
167
|
+
blocks.push({
|
|
168
|
+
headerLine: done.headerLine,
|
|
169
|
+
startLine: done.headerLine + 1,
|
|
170
|
+
endLine: lastContentLine,
|
|
171
|
+
closeLine: null,
|
|
172
|
+
})
|
|
173
|
+
}
|
|
174
|
+
}
|
|
175
|
+
if (PY_BLOCK_RE.test(line)) stack.push({ headerLine: i, indent })
|
|
176
|
+
lastContentLine = i
|
|
177
|
+
}
|
|
178
|
+
while (stack.length > 0) {
|
|
179
|
+
const done = stack.pop()
|
|
180
|
+
if (lastContentLine > done.headerLine) {
|
|
181
|
+
blocks.push({
|
|
182
|
+
headerLine: done.headerLine,
|
|
183
|
+
startLine: done.headerLine + 1,
|
|
184
|
+
endLine: lastContentLine,
|
|
185
|
+
closeLine: null,
|
|
186
|
+
})
|
|
187
|
+
}
|
|
188
|
+
}
|
|
189
|
+
return blocks
|
|
190
|
+
}
|
|
191
|
+
|
|
192
|
+
// Non-allocating blank check. `line.trim() === ''` allocated a string per line,
|
|
193
|
+
// which the old per-header rescan paid once per open block per blank line.
|
|
194
|
+
function isBlank(line) {
|
|
195
|
+
for (let i = 0; i < line.length; i++) {
|
|
196
|
+
const ch = line[i]
|
|
197
|
+
if (ch !== ' ' && ch !== '\t' && ch !== '\r') return false
|
|
198
|
+
}
|
|
199
|
+
return true
|
|
200
|
+
}
|
|
201
|
+
|
|
202
|
+
// --- Public entry ---
|
|
203
|
+
//
|
|
204
|
+
// `detectBlocks(lines, { path })` -> array of blocks, or null when no detector
|
|
205
|
+
// applies or the structure did not check out. Blocks are unordered with
|
|
206
|
+
// respect to nesting; use `buildStructure` to get the lookup maps.
|
|
207
|
+
|
|
208
|
+
export function detectBlocks(lines, { path = null } = {}) {
|
|
209
|
+
if (!Array.isArray(lines) || lines.length === 0) return null
|
|
210
|
+
const language = languageForPath(path) || sniffLanguage(lines)
|
|
211
|
+
if (language === 'brace') return detectBraceBlocks(lines)
|
|
212
|
+
if (language === 'indent') return detectIndentBlocks(lines)
|
|
213
|
+
return null
|
|
214
|
+
}
|
|
215
|
+
|
|
216
|
+
// Flatten blocks into per-line lookups the trimmer needs:
|
|
217
|
+
// parent[i] innermost block header enclosing line i, or -1
|
|
218
|
+
// closer[h] ARRAY of closing lines for blocks whose header is line h
|
|
219
|
+
// headerOf[c] ARRAY of header lines for blocks closing on line c
|
|
220
|
+
//
|
|
221
|
+
// `headerOf` is the inverse of `closer`: a bare `}` sits outside its own
|
|
222
|
+
// block's body, so without it a closing line looks parentless and can be kept
|
|
223
|
+
// on its own, stranding a `}` with no signature.
|
|
224
|
+
//
|
|
225
|
+
// Both are LISTS, not scalars (C2). One physical line can open or close more
|
|
226
|
+
// than one block — `function f() { if (x) {` opens two, `} }` closes two.
|
|
227
|
+
// Scalar maps keyed by line number silently kept only the LAST write, so the
|
|
228
|
+
// outer closer was lost and the trimmer emitted a signature with no matching
|
|
229
|
+
// brace. Verified before the fix: closer[0] === 2 (the inner `}`) not 3.
|
|
230
|
+
//
|
|
231
|
+
// `parent` is filled by one stack sweep rather than by painting each block's
|
|
232
|
+
// span (C1). Painting costs O(sum of block spans) = O(N x nesting depth),
|
|
233
|
+
// quadratic on deeply nested input: a body of N/2 `{` then N/2 `}` measured
|
|
234
|
+
// 200ms at 80KB, 1188ms at 200KB, 4741ms at 400KB. The sweep is O(N + blocks).
|
|
235
|
+
export function buildStructure(lines, blocks) {
|
|
236
|
+
const n = lines.length
|
|
237
|
+
const parent = new Int32Array(n).fill(-1)
|
|
238
|
+
const closer = new Array(n)
|
|
239
|
+
const headerOf = new Array(n)
|
|
240
|
+
if (!Array.isArray(blocks) || blocks.length === 0) return { parent, closer, headerOf }
|
|
241
|
+
|
|
242
|
+
for (const block of blocks) {
|
|
243
|
+
const h = block.headerLine
|
|
244
|
+
const c = block.closeLine
|
|
245
|
+
if (h < 0 || h >= n || c === null || c < 0 || c >= n) continue
|
|
246
|
+
;(closer[h] || (closer[h] = [])).push(c)
|
|
247
|
+
;(headerOf[c] || (headerOf[c] = [])).push(h)
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// Outer-first among blocks starting on the same line, so the narrower one
|
|
251
|
+
// lands on top of the stack and wins `parent` for the lines both cover.
|
|
252
|
+
const byStart = blocks
|
|
253
|
+
.slice()
|
|
254
|
+
.sort((a, b) => (a.startLine - b.startLine) || (b.endLine - a.endLine))
|
|
255
|
+
const stack = []
|
|
256
|
+
let bi = 0
|
|
257
|
+
for (let i = 0; i < n; i++) {
|
|
258
|
+
while (stack.length > 0 && stack[stack.length - 1].endLine < i) stack.pop()
|
|
259
|
+
while (bi < byStart.length && byStart[bi].startLine <= i) {
|
|
260
|
+
const b = byStart[bi]
|
|
261
|
+
bi += 1
|
|
262
|
+
if (b.endLine >= i) stack.push(b)
|
|
263
|
+
}
|
|
264
|
+
// A block never covers its own header (startLine is headerLine + 1), so
|
|
265
|
+
// parent[headerLine] keeps pointing at the enclosing block.
|
|
266
|
+
parent[i] = stack.length > 0 ? stack[stack.length - 1].headerLine : -1
|
|
267
|
+
}
|
|
268
|
+
return { parent, closer, headerOf }
|
|
269
|
+
}
|
package/lib/redact.js
ADDED
|
@@ -0,0 +1,78 @@
|
|
|
1
|
+
// --- Secret redaction (security, not compression) ---
|
|
2
|
+
//
|
|
3
|
+
// Masks or removes high-confidence secrets in outbound tool_result bodies
|
|
4
|
+
// before anything leaves the machine — and before any stage that ships text
|
|
5
|
+
// to a third party (llmlingua sidecar, textpress/OpenRouter) ever sees it.
|
|
6
|
+
//
|
|
7
|
+
// Design bias: precision over recall. A false positive corrupts legitimate
|
|
8
|
+
// content the model needs; a false negative leaks a secret. We only match
|
|
9
|
+
// patterns with a recognizable shape (provider-prefixed keys, PEM blocks,
|
|
10
|
+
// JWTs) plus assignment lines whose KEY name is unambiguously a secret. We do
|
|
11
|
+
// NOT do entropy guessing — too noisy for code/log payloads.
|
|
12
|
+
//
|
|
13
|
+
// Why: https://github.com/sliday/tamp/issues/6
|
|
14
|
+
|
|
15
|
+
// Each rule: a global regex and a short label used in the mask marker. Where a
|
|
16
|
+
// rule needs to keep surrounding structure (assignments), it captures the
|
|
17
|
+
// secret in group 1 and rebuilds the line via `replace`.
|
|
18
|
+
const TOKEN_RULES = [
|
|
19
|
+
{ label: 'aws-access-key', re: /\b(?:AKIA|ASIA|AGPA|AIDA|AROA|AIPA|ANPA|ANVA)[0-9A-Z]{16}\b/g },
|
|
20
|
+
{ label: 'github-token', re: /\bgh[pousr]_[A-Za-z0-9]{36,}\b/g },
|
|
21
|
+
{ label: 'github-pat', re: /\bgithub_pat_[A-Za-z0-9_]{60,}\b/g },
|
|
22
|
+
{ label: 'anthropic-key', re: /\bsk-ant-[A-Za-z0-9_-]{20,}\b/g },
|
|
23
|
+
{ label: 'openai-key', re: /\bsk-(?:proj-)?[A-Za-z0-9_-]{20,}\b/g },
|
|
24
|
+
{ label: 'google-api-key', re: /\bAIza[0-9A-Za-z_-]{35}\b/g },
|
|
25
|
+
{ label: 'slack-token', re: /\bxox[baprs]-[A-Za-z0-9-]{10,}\b/g },
|
|
26
|
+
{ label: 'stripe-key', re: /\b[rs]k_live_[A-Za-z0-9]{20,}\b/g },
|
|
27
|
+
{ label: 'google-oauth', re: /\bya29\.[A-Za-z0-9_-]{20,}\b/g },
|
|
28
|
+
{ label: 'jwt', re: /\beyJ[A-Za-z0-9_-]{10,}\.eyJ[A-Za-z0-9_-]{10,}\.[A-Za-z0-9_-]{10,}\b/g },
|
|
29
|
+
{ label: 'private-key', re: /-----BEGIN (?:[A-Z0-9 ]+ )?PRIVATE KEY-----[\s\S]*?-----END (?:[A-Z0-9 ]+ )?PRIVATE KEY-----/g },
|
|
30
|
+
]
|
|
31
|
+
|
|
32
|
+
// Assignment lines from .env / shell / config where the KEY name names a
|
|
33
|
+
// secret. We mask only the VALUE, keeping the key so the model still sees the
|
|
34
|
+
// variable exists.
|
|
35
|
+
//
|
|
36
|
+
// Precision rules learned the hard way:
|
|
37
|
+
// - Case-SENSITIVE uppercase keys only. Real secret env vars are UPPER_SNAKE
|
|
38
|
+
// (`API_KEY`, `AUTH_TOKEN`); camelCase like `apiKey = getKey()` is code,
|
|
39
|
+
// not a secret, and must not match.
|
|
40
|
+
// - The secret word is anchored to the END of the key (immediately before
|
|
41
|
+
// the `=`/`:`), so `AUTHOR=`, `COMPASS=`, `PASSWORD_HINT=` don't trip it.
|
|
42
|
+
// - Value must be 6+ non-space chars to skip empty/placeholder assignments.
|
|
43
|
+
// - The key prefix is BOUNDED (`{0,64}`) and word-boundary anchored. An
|
|
44
|
+
// unbounded `[A-Z0-9_]*` before the secret-word alternation is quadratic:
|
|
45
|
+
// on a long uppercase run with no match the engine retries every split
|
|
46
|
+
// point, so a big CONSTANT_CASE/hex tool_result would stall the proxy
|
|
47
|
+
// event loop for seconds. Real env-var names are well under 64 chars.
|
|
48
|
+
const SECRET_KEY = '\\b[A-Z0-9_]{0,64}(?:SECRET|TOKEN|PASSWORD|PASSWD|API[_-]?KEY|ACCESS[_-]?KEY|PRIVATE[_-]?KEY|CLIENT[_-]?SECRET|CREDENTIALS?|AUTH|PASSPHRASE|SESSION[_-]?KEY)'
|
|
49
|
+
const ASSIGN_RE = new RegExp(
|
|
50
|
+
`((?:export\\s+)?${SECRET_KEY})(\\s*[:=]\\s*)(["']?)([^\\s"']{6,})(["']?)`,
|
|
51
|
+
'g'
|
|
52
|
+
)
|
|
53
|
+
|
|
54
|
+
function marker(label, mode) {
|
|
55
|
+
return mode === 'remove' ? '' : `‹redacted:${label}›`
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
// Returns { text, count }. `count` is the number of secrets masked/removed.
|
|
59
|
+
// When count is 0 the returned text is referentially the input (no allocation
|
|
60
|
+
// churn for the common no-secret case).
|
|
61
|
+
export function redactText(text, mode = 'mask') {
|
|
62
|
+
if (typeof text !== 'string' || text.length === 0) return { text, count: 0 }
|
|
63
|
+
|
|
64
|
+
let count = 0
|
|
65
|
+
let out = text
|
|
66
|
+
|
|
67
|
+
for (const { label, re } of TOKEN_RULES) {
|
|
68
|
+
out = out.replace(re, () => { count++; return marker(label, mode) })
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
out = out.replace(ASSIGN_RE, (m, key, sep, q1, _val, q2) => {
|
|
72
|
+
count++
|
|
73
|
+
if (mode === 'remove') return `${key}${sep}`
|
|
74
|
+
return `${key}${sep}${q1}${marker('secret', mode)}${q2}`
|
|
75
|
+
})
|
|
76
|
+
|
|
77
|
+
return count > 0 ? { text: out, count } : { text, count: 0 }
|
|
78
|
+
}
|
|
@@ -32,11 +32,13 @@ export default {
|
|
|
32
32
|
if (BUILDKIT_TRANSFER.test(line)) continue;
|
|
33
33
|
if (SPINNER_BRAILLE.test(line)) continue;
|
|
34
34
|
if (LEGACY_PROGRESS.test(line)) continue;
|
|
35
|
-
// BuildKit "in-progress"
|
|
36
|
-
// marker
|
|
37
|
-
//
|
|
38
|
-
// stdout
|
|
39
|
-
|
|
35
|
+
// BuildKit "in-progress" step frames carry a [x/y] step counter and end
|
|
36
|
+
// with a fractional-second marker; they are re-emitted as DONE later, so
|
|
37
|
+
// strip the non-DONE variants. Requiring [x/y] avoids eating a RUN step's
|
|
38
|
+
// own stdout (prefixed "#N <timestamp> ...", no counter), which can
|
|
39
|
+
// legitimately end in a duration (e.g. "webpack compiled in 2.5s"). The
|
|
40
|
+
// error/warning/fail guard stays as belt-and-suspenders.
|
|
41
|
+
if (/^#\d+\s+\[\d+\/\d+\]\s.*\s\d+\.\d+s$/.test(line) && !/\sDONE\b/.test(line) && !/\b(error|warning|fail)/i.test(line)) continue;
|
|
40
42
|
out.push(line);
|
|
41
43
|
}
|
|
42
44
|
|
|
@@ -10,7 +10,6 @@
|
|
|
10
10
|
// Keeps: error/warn lines, final "added N packages in Ns", vulnerability
|
|
11
11
|
// counts, "up to date", package trees.
|
|
12
12
|
|
|
13
|
-
const SPINNER_CHARS = /[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]/;
|
|
14
13
|
const PROGRESS_LINE = /^\s*[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]\s+/;
|
|
15
14
|
const DOTS_ONLY = /^\s*\.{3,}\s*$/;
|
|
16
15
|
const NPM_SIG = /(^|\n)(npm (install|i|ci|run)\b|added \d+ packages?|up to date|changed \d+ packages?|removed \d+ packages?)/;
|
|
@@ -38,8 +37,11 @@ export default {
|
|
|
38
37
|
for (let i = 0; i < lines.length; i++) {
|
|
39
38
|
const line = lines[i];
|
|
40
39
|
|
|
41
|
-
// Drop spinner lines and sole-dot lines.
|
|
42
|
-
|
|
40
|
+
// Drop leading-spinner progress lines and sole-dot lines. Only anchored
|
|
41
|
+
// PROGRESS_LINE (spinner glyph at line start) — an unanchored glyph test
|
|
42
|
+
// deleted whole lines of child-tool output that merely contained a
|
|
43
|
+
// braille char anywhere (e.g. `⠋ 2 warnings, 1 error`), losing signal.
|
|
44
|
+
if (PROGRESS_LINE.test(line)) continue;
|
|
43
45
|
if (DOTS_ONLY.test(line)) {
|
|
44
46
|
if (lastWasDots) continue;
|
|
45
47
|
lastWasDots = true;
|
|
@@ -7,7 +7,12 @@
|
|
|
7
7
|
// the short test summary info block, and the final "=== N passed in Xs ===".
|
|
8
8
|
|
|
9
9
|
const COLLECTED = /^collected\s+\d+\s+items?\b/;
|
|
10
|
-
|
|
10
|
+
// Per-file dot-progress: "<path>.py[::node] <status run> [ NN% ]" (percent
|
|
11
|
+
// optional). Anchored to a .py path prefix so a traceback/assertion line that
|
|
12
|
+
// merely ends in "..." is NOT mistaken for progress and silently dropped.
|
|
13
|
+
const DOT_PROGRESS = /^\S*\.py(::\S+)?\s+[.FsxE]{2,}\s*(\[\s*\d+%\s*\])?\s*$/;
|
|
14
|
+
// Wrapped progress continuation: a bare status run followed by a percent.
|
|
15
|
+
const PCT_PROGRESS = /^\s*[.FsxE]{2,}\s*\[\s*\d+%\s*\]\s*$/;
|
|
11
16
|
const SESSION_START = /^platform\s+\S+\s+--\s+Python\s+\d/;
|
|
12
17
|
const ROOTDIR = /^rootdir:\s+/;
|
|
13
18
|
const PLUGINS = /^plugins:\s+/;
|
|
@@ -33,9 +38,10 @@ export default {
|
|
|
33
38
|
if (ROOTDIR.test(line)) continue;
|
|
34
39
|
if (PLUGINS.test(line)) continue;
|
|
35
40
|
if (CACHEDIR.test(line)) continue;
|
|
36
|
-
// Drop
|
|
37
|
-
// (
|
|
38
|
-
|
|
41
|
+
// Drop per-file / wrapped dot-progress lines. Both patterns are anchored
|
|
42
|
+
// (path prefix or trailing percent), so traceback/assertion lines that end
|
|
43
|
+
// in "..." are preserved without needing a fragile keyword guard.
|
|
44
|
+
if (DOT_PROGRESS.test(line) || PCT_PROGRESS.test(line)) continue;
|
|
39
45
|
out.push(line);
|
|
40
46
|
}
|
|
41
47
|
|
package/metadata.js
CHANGED
|
@@ -104,7 +104,7 @@ export const STAGE_DESCRIPTIONS = Object.freeze({
|
|
|
104
104
|
'foundation-models': 'Apple Intelligence neural compression (macOS 15+, Apple Silicon, 100% local)',
|
|
105
105
|
dedup: 'Deduplicate identical tool_results',
|
|
106
106
|
diff: 'Replace similar re-reads with diffs',
|
|
107
|
-
'read-diff': 'Session-scoped unified diff for re-reads (lossless,
|
|
107
|
+
'read-diff': 'Session-scoped unified diff for re-reads (lossless, on by default in balanced+)',
|
|
108
108
|
prune: 'Strip lockfile hashes & npm metadata',
|
|
109
109
|
'strip-comments': 'Remove code comments (lossy)',
|
|
110
110
|
textpress: 'LLM semantic compression (Ollama/OpenRouter)',
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"sidecar/server.py",
|
|
15
15
|
"sidecar/requirements.txt"
|
|
16
16
|
],
|
|
17
|
-
"version": "0.8.
|
|
17
|
+
"version": "0.8.16",
|
|
18
18
|
"description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
|
|
19
19
|
"type": "module",
|
|
20
20
|
"main": "index.js",
|
|
@@ -23,9 +23,11 @@
|
|
|
23
23
|
},
|
|
24
24
|
"scripts": {
|
|
25
25
|
"start": "node bin/tamp.js",
|
|
26
|
+
"prepublishOnly": "node scripts/publish-guard.js",
|
|
26
27
|
"test": "node --test test/*.test.js",
|
|
27
28
|
"test:sidecar": "node test/capture-golden.js --verify",
|
|
28
29
|
"bench:semantic": "node bench/semantic-eval.js",
|
|
30
|
+
"bench:retrieval": "node bench/retrieval-eval.js",
|
|
29
31
|
"test:quality": "node bench/quality-eval.js",
|
|
30
32
|
"compat:probe": "node test/compat/probe-proxy.js",
|
|
31
33
|
"compat:run": "node test/compat/run.mjs"
|
package/providers.js
CHANGED
|
@@ -130,18 +130,16 @@ function findAnthropicReferences(body) {
|
|
|
130
130
|
} else if (typeof block.input === 'string') {
|
|
131
131
|
scanStringForRefs(block.input, ['messages', mi, 'content', bi, 'input'], out)
|
|
132
132
|
}
|
|
133
|
-
} else if (block.type === 'tool_result') {
|
|
134
|
-
if (typeof block.content === 'string') {
|
|
135
|
-
scanStringForRefs(block.content, ['messages', mi, 'content', bi, 'content'], out)
|
|
136
|
-
} else if (Array.isArray(block.content)) {
|
|
137
|
-
for (let ci = 0; ci < block.content.length; ci++) {
|
|
138
|
-
const sub = block.content[ci]
|
|
139
|
-
if (sub?.type === 'text' && typeof sub.text === 'string') {
|
|
140
|
-
scanStringForRefs(sub.text, ['messages', mi, 'content', bi, 'content', ci, 'text'], out)
|
|
141
|
-
}
|
|
142
|
-
}
|
|
143
|
-
}
|
|
144
133
|
}
|
|
134
|
+
// NOTE: tool_result content is deliberately NOT scanned for markers. A
|
|
135
|
+
// marker is only honored when the MODEL quotes it (assistant text or
|
|
136
|
+
// tool_use.input) — that is the disclosure contract. tool_result bodies
|
|
137
|
+
// are untrusted tool output (fetched pages, file reads); scanning them
|
|
138
|
+
// would (1) let attacker-controlled content forge a <tamp-ref:v1:...>
|
|
139
|
+
// marker and pull an arbitrary body out of the shared br-cache into the
|
|
140
|
+
// outbound request (cross-conversation cache-pull + existence oracle),
|
|
141
|
+
// and (2) auto-rehydrate a disclosure summary's own marker every turn,
|
|
142
|
+
// defeating the compression. See rehydrateReferences.
|
|
145
143
|
}
|
|
146
144
|
}
|
|
147
145
|
return out
|
|
@@ -221,7 +219,7 @@ const anthropic = {
|
|
|
221
219
|
return method === 'POST' && url.startsWith('/v1/messages')
|
|
222
220
|
},
|
|
223
221
|
extract(body, config = {}) {
|
|
224
|
-
if (!body?.messages
|
|
222
|
+
if (!Array.isArray(body?.messages) || !body.messages.length) return []
|
|
225
223
|
if (config.cacheSafe) {
|
|
226
224
|
return findLatestEligibleGroup(body.messages, extractAnthropicMessageTargets)
|
|
227
225
|
}
|
|
@@ -285,13 +283,13 @@ const openai = {
|
|
|
285
283
|
return url
|
|
286
284
|
},
|
|
287
285
|
extract(body, config = {}) {
|
|
288
|
-
if (!body?.messages
|
|
286
|
+
if (!Array.isArray(body?.messages) || !body.messages.length) return []
|
|
289
287
|
|
|
290
288
|
if (config.cacheSafe) {
|
|
291
289
|
const targets = []
|
|
292
290
|
for (let i = body.messages.length - 1; i >= 0; i--) {
|
|
293
291
|
const msg = body.messages[i]
|
|
294
|
-
if (msg.role !== 'tool') break
|
|
292
|
+
if (!msg || msg.role !== 'tool') break
|
|
295
293
|
targets.unshift(...extractOpenAIChatTargets(msg, i))
|
|
296
294
|
}
|
|
297
295
|
return targets
|
|
@@ -417,7 +415,7 @@ const gemini = {
|
|
|
417
415
|
return method === 'POST' && url.includes('generateContent')
|
|
418
416
|
},
|
|
419
417
|
extract(body, config = {}) {
|
|
420
|
-
if (!body?.contents
|
|
418
|
+
if (!Array.isArray(body?.contents) || !body.contents.length) return []
|
|
421
419
|
if (config.cacheSafe) {
|
|
422
420
|
return findLatestEligibleGroup(body.contents, extractGeminiContentTargets)
|
|
423
421
|
}
|
package/session-graph.js
CHANGED
|
@@ -66,7 +66,14 @@ function conversationSeed(body) {
|
|
|
66
66
|
const first = Array.isArray(turns) && turns.length ? turns[0] : null
|
|
67
67
|
if (first == null && body.system === undefined) return ''
|
|
68
68
|
try {
|
|
69
|
-
|
|
69
|
+
// Hash system and the first turn INDEPENDENTLY. A single slice over the
|
|
70
|
+
// combined JSON let a long system prompt (Claude Code ships a multi-KB one)
|
|
71
|
+
// crowd the conversation-specific first turn past the cutoff, collapsing
|
|
72
|
+
// distinct conversations onto one key — the exact cross-conversation leak
|
|
73
|
+
// this seed exists to prevent. Per-part hashes keep the seed bounded + stable.
|
|
74
|
+
const sys = createHash('sha256').update(JSON.stringify(body.system ?? null)).digest('hex')
|
|
75
|
+
const frst = createHash('sha256').update(JSON.stringify(first ?? null)).digest('hex')
|
|
76
|
+
return `${sys}:${frst}`
|
|
70
77
|
} catch {
|
|
71
78
|
return ''
|
|
72
79
|
}
|
package/stats.js
CHANGED
|
@@ -13,7 +13,12 @@ const c = {
|
|
|
13
13
|
// (Gemini and OpenAI-compatible gateways) carry the API key in the query
|
|
14
14
|
// string (?key=, ?api_key=), which would otherwise land in stdout, log
|
|
15
15
|
// files, and the systemd journal. Path-relative URLs only — no base needed.
|
|
16
|
-
|
|
16
|
+
// Keyword (substring) match on the param NAME, not an exact list. The old
|
|
17
|
+
// anchored `^(key|api_key|apikey|access_token|token)$` leaked `refresh_token`,
|
|
18
|
+
// `id_token`, `client_secret`, `password`, `session`, `sig`/`signature`, `auth`
|
|
19
|
+
// to stdout / TAMP_LOG_FILE / the systemd journal. Over-redacting a benign
|
|
20
|
+
// param name in a log is harmless; leaking a credential is not.
|
|
21
|
+
const SECRET_QUERY_PARAM = /key|token|secret|password|passwd|sig|auth|session|credential/i
|
|
17
22
|
export function redactUrl(url) {
|
|
18
23
|
if (typeof url !== 'string') return url
|
|
19
24
|
const q = url.indexOf('?')
|
|
@@ -51,7 +56,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
|
|
|
51
56
|
const totalComp = compressed.reduce((a, s) => a + s.compressedLen, 0)
|
|
52
57
|
const totalOrigTok = compressed.reduce((a, s) => a + (s.originalTokens || 0), 0)
|
|
53
58
|
const totalCompTok = compressed.reduce((a, s) => a + (s.compressedTokens || 0), 0)
|
|
54
|
-
const pct = (((totalOrig - totalComp) / totalOrig) * 100).toFixed(1)
|
|
59
|
+
const pct = (totalOrig > 0 ? ((totalOrig - totalComp) / totalOrig) * 100 : 0).toFixed(1)
|
|
55
60
|
const saved = totalOrig - totalComp
|
|
56
61
|
const tokSaved = totalOrigTok - totalCompTok
|
|
57
62
|
|
|
@@ -60,7 +65,7 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
|
|
|
60
65
|
lines.push(`[tamp] ${c.cyan}${label}${c.reset} ${path}${sizeInfo} ${c.green}— ${n} block${n !== 1 ? 's' : ''} compressed, -${pct}%${c.reset}`)
|
|
61
66
|
|
|
62
67
|
for (const s of compressed) {
|
|
63
|
-
const sPct = (((s.originalLen - s.compressedLen) / s.originalLen) * 100).toFixed(1)
|
|
68
|
+
const sPct = (s.originalLen > 0 ? ((s.originalLen - s.compressedLen) / s.originalLen) * 100 : 0).toFixed(1)
|
|
64
69
|
const tokInfo = s.originalTokens ? ` ${c.dim}${s.originalTokens}→${s.compressedTokens} tok${c.reset}` : ''
|
|
65
70
|
lines.push(`[tamp] ${c.dim}block[${s.index}]${c.reset} ${fmtSize(s.originalLen)}→${fmtSize(s.compressedLen)} ${c.green}-${sPct}%${c.reset}${tokInfo} ${c.dim}[${s.method}]${c.reset}`)
|
|
66
71
|
}
|