@sliday/tamp 0.5.8 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +13 -7
- package/bin/lifecycle.js +16 -4
- package/bin/tamp.js +46 -0
- package/bin/ui/Banner.js +7 -1
- package/bin/ui/Dashboard.js +1 -0
- package/compress.js +184 -4
- package/config.js +23 -2
- package/detect.js +10 -0
- package/index.js +67 -7
- package/lib/bm25.js +194 -0
- package/lib/br-cache.js +157 -0
- package/lib/disclosure.js +124 -0
- package/lib/path-extract.js +49 -0
- package/lib/read-cache.js +55 -0
- package/lib/rewriters/commands/cargo.js +50 -0
- package/lib/rewriters/commands/docker.js +54 -0
- package/lib/rewriters/commands/git.js +81 -0
- package/lib/rewriters/commands/jest.js +49 -0
- package/lib/rewriters/commands/npm.js +82 -0
- package/lib/rewriters/commands/pip.js +58 -0
- package/lib/rewriters/commands/prisma.js +50 -0
- package/lib/rewriters/commands/pytest.js +55 -0
- package/lib/rewriters/commands/terraform.js +49 -0
- package/lib/rewriters/commands/wget-curl.js +67 -0
- package/lib/rewriters/index.js +77 -0
- package/lib/rules-generator.js +45 -2
- package/metadata.js +37 -5
- package/package.json +2 -2
- package/providers.js +124 -1
- package/session-graph.js +36 -3
- package/stats.js +12 -1
package/lib/bm25.js
ADDED
|
@@ -0,0 +1,194 @@
|
|
|
1
|
+
// Query-aware line ranker for huge tool_result bodies (Phase 6).
|
|
2
|
+
//
|
|
3
|
+
// Pure-JS BM25 scorer, zero deps. Given a query (last user message) and a
|
|
4
|
+
// large body, score each line against the query using Okapi BM25 and keep
|
|
5
|
+
// the highest-scoring lines until a token budget is satisfied. First and
|
|
6
|
+
// last lines are always preserved as context anchors, and collapsed ranges
|
|
7
|
+
// are replaced by a "[... N lines omitted, N chars ...]" marker so the
|
|
8
|
+
// model can reason about the elision.
|
|
9
|
+
//
|
|
10
|
+
// This primitive is reusable: any future stage that wants "keep the most
|
|
11
|
+
// relevant N lines" can import tokenize/bm25Scores directly.
|
|
12
|
+
|
|
13
|
+
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
14
|
+
|
|
15
|
+
// --- Tokenizer ---
|
|
16
|
+
// Preserve identifiers with dots/underscores/dashes (e.g. `user.email`,
|
|
17
|
+
// `max_tokens`, `read-diff`) as single tokens. We collapse intra-word
|
|
18
|
+
// separators to underscore before splitting on non-word characters. Single
|
|
19
|
+
// character tokens are dropped (too noisy for BM25 on short queries).
|
|
20
|
+
const INTRA_WORD_RE = /([A-Za-z0-9])[._\-]+([A-Za-z0-9])/g
|
|
21
|
+
const SPLIT_RE = /[^A-Za-z0-9_]+/
|
|
22
|
+
|
|
23
|
+
export function tokenize(text) {
|
|
24
|
+
if (typeof text !== 'string' || text.length === 0) return []
|
|
25
|
+
let normalized = text.toLowerCase()
|
|
26
|
+
// Collapse intra-word separators. Run twice so chains like `a.b.c` fully
|
|
27
|
+
// collapse (first pass joins a.b, second joins a_b.c -> a_b_c).
|
|
28
|
+
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
29
|
+
normalized = normalized.replace(INTRA_WORD_RE, '$1_$2')
|
|
30
|
+
const raw = normalized.split(SPLIT_RE)
|
|
31
|
+
const out = []
|
|
32
|
+
for (const tok of raw) {
|
|
33
|
+
if (tok.length < 2) continue
|
|
34
|
+
out.push(tok)
|
|
35
|
+
}
|
|
36
|
+
return out
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
// --- BM25 ---
|
|
40
|
+
// Standard Okapi BM25. Each document is a line; query is the user's last
|
|
41
|
+
// message. Returns Float64Array of scores aligned with the documents array.
|
|
42
|
+
export function bm25Scores(documents, query, { k1 = 1.5, b = 0.75 } = {}) {
|
|
43
|
+
const N = documents.length
|
|
44
|
+
const scores = new Float64Array(N)
|
|
45
|
+
if (N === 0) return scores
|
|
46
|
+
|
|
47
|
+
const qTokens = tokenize(query)
|
|
48
|
+
if (qTokens.length === 0) return scores
|
|
49
|
+
|
|
50
|
+
// Tokenize docs + build doc stats.
|
|
51
|
+
const docTokens = new Array(N)
|
|
52
|
+
const docLens = new Float64Array(N)
|
|
53
|
+
let totalLen = 0
|
|
54
|
+
for (let i = 0; i < N; i++) {
|
|
55
|
+
const toks = tokenize(documents[i])
|
|
56
|
+
docTokens[i] = toks
|
|
57
|
+
docLens[i] = toks.length
|
|
58
|
+
totalLen += toks.length
|
|
59
|
+
}
|
|
60
|
+
const avgdl = N > 0 ? totalLen / N : 0
|
|
61
|
+
if (avgdl === 0) return scores
|
|
62
|
+
|
|
63
|
+
// Document-frequency per unique query term.
|
|
64
|
+
const uniqQ = Array.from(new Set(qTokens))
|
|
65
|
+
const df = new Map()
|
|
66
|
+
for (const term of uniqQ) {
|
|
67
|
+
let count = 0
|
|
68
|
+
for (let i = 0; i < N; i++) {
|
|
69
|
+
if (docTokens[i].includes(term)) count += 1
|
|
70
|
+
}
|
|
71
|
+
df.set(term, count)
|
|
72
|
+
}
|
|
73
|
+
|
|
74
|
+
// Precompute IDF per term (Okapi BM25 variant with +1 smoothing).
|
|
75
|
+
const idf = new Map()
|
|
76
|
+
for (const term of uniqQ) {
|
|
77
|
+
const n = df.get(term) || 0
|
|
78
|
+
const v = Math.log(1 + (N - n + 0.5) / (n + 0.5))
|
|
79
|
+
idf.set(term, v)
|
|
80
|
+
}
|
|
81
|
+
|
|
82
|
+
for (let i = 0; i < N; i++) {
|
|
83
|
+
const dl = docLens[i]
|
|
84
|
+
if (dl === 0) continue
|
|
85
|
+
// Term-frequency table for this doc (only for query terms we care about).
|
|
86
|
+
const tf = new Map()
|
|
87
|
+
const toks = docTokens[i]
|
|
88
|
+
for (const t of toks) {
|
|
89
|
+
if (!idf.has(t)) continue
|
|
90
|
+
tf.set(t, (tf.get(t) || 0) + 1)
|
|
91
|
+
}
|
|
92
|
+
let s = 0
|
|
93
|
+
for (const term of uniqQ) {
|
|
94
|
+
const f = tf.get(term) || 0
|
|
95
|
+
if (f === 0) continue
|
|
96
|
+
const num = f * (k1 + 1)
|
|
97
|
+
const den = f + k1 * (1 - b + b * (dl / avgdl))
|
|
98
|
+
s += idf.get(term) * (num / den)
|
|
99
|
+
}
|
|
100
|
+
scores[i] = s
|
|
101
|
+
}
|
|
102
|
+
return scores
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
// --- Line trimmer ---
|
|
106
|
+
// Keep top-K highest-scoring lines until `targetTokens` is consumed. First
|
|
107
|
+
// and last line are always preserved. Returns null when text is already
|
|
108
|
+
// under budget, has fewer than `minLines` lines, or trimming didn't help.
|
|
109
|
+
//
|
|
110
|
+
// Output shape:
|
|
111
|
+
// { text, keptLines, droppedLines, originalTokens, trimmedTokens, originalLines }
|
|
112
|
+
export function trimLinesByRelevance(text, query, { targetTokens = 4096, minLines = 10 } = {}) {
|
|
113
|
+
if (typeof text !== 'string' || text.length === 0) return null
|
|
114
|
+
|
|
115
|
+
const originalTokens = countTokens(text)
|
|
116
|
+
if (originalTokens <= targetTokens) return null
|
|
117
|
+
|
|
118
|
+
const lines = text.split('\n')
|
|
119
|
+
const N = lines.length
|
|
120
|
+
if (N < minLines) return null
|
|
121
|
+
|
|
122
|
+
// Per-line token costs (includes the trailing newline for all but the last
|
|
123
|
+
// line; we approximate by counting each line once and adding N-1 tokens).
|
|
124
|
+
const lineTokens = new Array(N)
|
|
125
|
+
for (let i = 0; i < N; i++) {
|
|
126
|
+
lineTokens[i] = countTokens(lines[i])
|
|
127
|
+
}
|
|
128
|
+
const newlineCost = Math.max(0, N - 1)
|
|
129
|
+
|
|
130
|
+
const scores = bm25Scores(lines, query)
|
|
131
|
+
|
|
132
|
+
// Always-keep set: first + last as anchors.
|
|
133
|
+
const keep = new Uint8Array(N)
|
|
134
|
+
keep[0] = 1
|
|
135
|
+
keep[N - 1] = 1
|
|
136
|
+
let budgetUsed = lineTokens[0] + (N > 1 ? lineTokens[N - 1] : 0)
|
|
137
|
+
// Rough marker cost reserved per collapsed gap; we'll reconcile below.
|
|
138
|
+
const MARKER_RESERVE = 16
|
|
139
|
+
|
|
140
|
+
// Sort remaining lines by score desc, then by position asc (stable earlier
|
|
141
|
+
// context preferred on ties). We pick greedily until budget is hit.
|
|
142
|
+
const candidates = []
|
|
143
|
+
for (let i = 1; i < N - 1; i++) {
|
|
144
|
+
candidates.push({ i, score: scores[i], len: lineTokens[i] })
|
|
145
|
+
}
|
|
146
|
+
candidates.sort((a, b) => (b.score - a.score) || (a.i - b.i))
|
|
147
|
+
|
|
148
|
+
for (const c of candidates) {
|
|
149
|
+
if (budgetUsed + c.len > targetTokens) continue
|
|
150
|
+
keep[c.i] = 1
|
|
151
|
+
budgetUsed += c.len
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
// If even the anchors + marker exceed budget, still proceed — we need to
|
|
155
|
+
// produce SOME trim. But if nothing was dropped at all, bail.
|
|
156
|
+
let keptLines = 0
|
|
157
|
+
for (let i = 0; i < N; i++) keptLines += keep[i]
|
|
158
|
+
const droppedLines = N - keptLines
|
|
159
|
+
if (droppedLines === 0) return null
|
|
160
|
+
|
|
161
|
+
// Assemble output, collapsing contiguous dropped runs into drop markers.
|
|
162
|
+
const out = []
|
|
163
|
+
let i = 0
|
|
164
|
+
while (i < N) {
|
|
165
|
+
if (keep[i]) {
|
|
166
|
+
out.push(lines[i])
|
|
167
|
+
i += 1
|
|
168
|
+
continue
|
|
169
|
+
}
|
|
170
|
+
let j = i
|
|
171
|
+
let chars = 0
|
|
172
|
+
while (j < N && !keep[j]) { chars += lines[j].length + 1; j += 1 }
|
|
173
|
+
const runLen = j - i
|
|
174
|
+
out.push(`[... ${runLen} lines omitted, ${chars} chars ...]`)
|
|
175
|
+
i = j
|
|
176
|
+
}
|
|
177
|
+
|
|
178
|
+
const trimmedText = out.join('\n')
|
|
179
|
+
const trimmedTokens = countTokens(trimmedText)
|
|
180
|
+
// Defensive: if trimming somehow grew the text, don't return a trim.
|
|
181
|
+
if (trimmedTokens >= originalTokens) return null
|
|
182
|
+
|
|
183
|
+
return {
|
|
184
|
+
text: trimmedText,
|
|
185
|
+
keptLines,
|
|
186
|
+
droppedLines,
|
|
187
|
+
originalTokens,
|
|
188
|
+
trimmedTokens,
|
|
189
|
+
originalLines: N,
|
|
190
|
+
budgetUsed,
|
|
191
|
+
newlineCost,
|
|
192
|
+
markerReserve: MARKER_RESERVE,
|
|
193
|
+
}
|
|
194
|
+
}
|
package/lib/br-cache.js
ADDED
|
@@ -0,0 +1,157 @@
|
|
|
1
|
+
// Disk-backed, content-addressable Brotli store for large bodies.
|
|
2
|
+
// Hash = SHA-256(text).digest('hex'); file path = <cacheDir>/<hash[0:2]>/<hash>.br
|
|
3
|
+
//
|
|
4
|
+
// Intended as the storage substrate for Phase 5 progressive disclosure. In
|
|
5
|
+
// Phase 3 we just offload large referenced bodies out of RAM and surface
|
|
6
|
+
// counters in the banner — we do NOT emit new markers upstream yet.
|
|
7
|
+
|
|
8
|
+
import { createHash } from 'node:crypto'
|
|
9
|
+
import { brotliCompressSync, brotliDecompressSync, constants as zlibConstants } from 'node:zlib'
|
|
10
|
+
import {
|
|
11
|
+
mkdirSync,
|
|
12
|
+
writeFileSync,
|
|
13
|
+
readFileSync,
|
|
14
|
+
renameSync,
|
|
15
|
+
unlinkSync,
|
|
16
|
+
readdirSync,
|
|
17
|
+
statSync,
|
|
18
|
+
utimesSync,
|
|
19
|
+
existsSync,
|
|
20
|
+
} from 'node:fs'
|
|
21
|
+
import { join } from 'node:path'
|
|
22
|
+
import { homedir, tmpdir } from 'node:os'
|
|
23
|
+
|
|
24
|
+
const DEFAULT_MAX_BYTES = 512 * 1024 * 1024
|
|
25
|
+
const DEFAULT_MIN_SIZE = 8 * 1024
|
|
26
|
+
const BROTLI_QUALITY = 4
|
|
27
|
+
|
|
28
|
+
function defaultCacheDir() {
|
|
29
|
+
const home = homedir() || tmpdir()
|
|
30
|
+
return join(home, '.cache', 'tamp', 'br')
|
|
31
|
+
}
|
|
32
|
+
|
|
33
|
+
function hashText(text) {
|
|
34
|
+
return createHash('sha256').update(text).digest('hex')
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
function filePathFor(cacheDir, hash) {
|
|
38
|
+
return join(cacheDir, hash.slice(0, 2), `${hash}.br`)
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function createBrCache({ cacheDir = defaultCacheDir(), maxBytes = DEFAULT_MAX_BYTES, minSize = DEFAULT_MIN_SIZE } = {}) {
|
|
42
|
+
try { mkdirSync(cacheDir, { recursive: true }) } catch { /* ignore */ }
|
|
43
|
+
|
|
44
|
+
let hits = 0
|
|
45
|
+
let misses = 0
|
|
46
|
+
let evictions = 0
|
|
47
|
+
|
|
48
|
+
function scan() {
|
|
49
|
+
const files = []
|
|
50
|
+
let totalBytes = 0
|
|
51
|
+
let shards
|
|
52
|
+
try { shards = readdirSync(cacheDir) } catch { return { files, totalBytes } }
|
|
53
|
+
for (const shard of shards) {
|
|
54
|
+
const shardPath = join(cacheDir, shard)
|
|
55
|
+
let entries
|
|
56
|
+
try { entries = readdirSync(shardPath) } catch { continue }
|
|
57
|
+
for (const name of entries) {
|
|
58
|
+
if (!name.endsWith('.br')) continue
|
|
59
|
+
const p = join(shardPath, name)
|
|
60
|
+
try {
|
|
61
|
+
const st = statSync(p)
|
|
62
|
+
if (!st.isFile()) continue
|
|
63
|
+
files.push({ path: p, size: st.size, atimeMs: st.atimeMs })
|
|
64
|
+
totalBytes += st.size
|
|
65
|
+
} catch { /* skip unreadable */ }
|
|
66
|
+
}
|
|
67
|
+
}
|
|
68
|
+
return { files, totalBytes }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function pruneInternal() {
|
|
72
|
+
const { files, totalBytes } = scan()
|
|
73
|
+
if (totalBytes <= maxBytes) return { totalBytes, evicted: 0 }
|
|
74
|
+
files.sort((a, b) => a.atimeMs - b.atimeMs) // oldest access first
|
|
75
|
+
let current = totalBytes
|
|
76
|
+
let evicted = 0
|
|
77
|
+
for (const f of files) {
|
|
78
|
+
if (current <= maxBytes) break
|
|
79
|
+
try { unlinkSync(f.path); current -= f.size; evicted += 1; evictions += 1 } catch { /* ignore */ }
|
|
80
|
+
}
|
|
81
|
+
return { totalBytes: current, evicted }
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function put(text) {
|
|
85
|
+
if (typeof text !== 'string') return null
|
|
86
|
+
const byteLen = Buffer.byteLength(text, 'utf8')
|
|
87
|
+
if (byteLen < minSize) return null
|
|
88
|
+
|
|
89
|
+
const hash = hashText(text)
|
|
90
|
+
const finalPath = filePathFor(cacheDir, hash)
|
|
91
|
+
const shardDir = join(cacheDir, hash.slice(0, 2))
|
|
92
|
+
|
|
93
|
+
let brotliBytes
|
|
94
|
+
if (existsSync(finalPath)) {
|
|
95
|
+
try {
|
|
96
|
+
brotliBytes = statSync(finalPath).size
|
|
97
|
+
// Refresh access time for LRU.
|
|
98
|
+
const now = new Date()
|
|
99
|
+
try { utimesSync(finalPath, now, now) } catch { /* ignore */ }
|
|
100
|
+
return { hash, storedBytes: byteLen, brotliBytes }
|
|
101
|
+
} catch { /* fall through and rewrite */ }
|
|
102
|
+
}
|
|
103
|
+
|
|
104
|
+
try { mkdirSync(shardDir, { recursive: true }) } catch { /* ignore */ }
|
|
105
|
+
|
|
106
|
+
let compressed
|
|
107
|
+
try {
|
|
108
|
+
compressed = brotliCompressSync(Buffer.from(text, 'utf8'), {
|
|
109
|
+
params: { [zlibConstants.BROTLI_PARAM_QUALITY]: BROTLI_QUALITY },
|
|
110
|
+
})
|
|
111
|
+
} catch {
|
|
112
|
+
return null
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const tmpPath = `${finalPath}.tmp`
|
|
116
|
+
try {
|
|
117
|
+
writeFileSync(tmpPath, compressed)
|
|
118
|
+
renameSync(tmpPath, finalPath)
|
|
119
|
+
brotliBytes = compressed.length
|
|
120
|
+
} catch {
|
|
121
|
+
try { unlinkSync(tmpPath) } catch { /* ignore */ }
|
|
122
|
+
return null
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
pruneInternal()
|
|
126
|
+
return { hash, storedBytes: byteLen, brotliBytes }
|
|
127
|
+
}
|
|
128
|
+
|
|
129
|
+
function get(hash) {
|
|
130
|
+
if (typeof hash !== 'string' || !/^[a-f0-9]{64}$/.test(hash)) { misses += 1; return null }
|
|
131
|
+
const p = filePathFor(cacheDir, hash)
|
|
132
|
+
let buf
|
|
133
|
+
try { buf = readFileSync(p) } catch { misses += 1; return null }
|
|
134
|
+
let decoded
|
|
135
|
+
try { decoded = brotliDecompressSync(buf) } catch { misses += 1; return null }
|
|
136
|
+
const now = new Date()
|
|
137
|
+
try { utimesSync(p, now, now) } catch { /* ignore */ }
|
|
138
|
+
hits += 1
|
|
139
|
+
return decoded.toString('utf8')
|
|
140
|
+
}
|
|
141
|
+
|
|
142
|
+
function has(hash) {
|
|
143
|
+
if (typeof hash !== 'string' || !/^[a-f0-9]{64}$/.test(hash)) return false
|
|
144
|
+
return existsSync(filePathFor(cacheDir, hash))
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
function stats() {
|
|
148
|
+
const { files, totalBytes } = scan()
|
|
149
|
+
return { entries: files.length, totalBytes, hits, misses, evictions }
|
|
150
|
+
}
|
|
151
|
+
|
|
152
|
+
function prune() {
|
|
153
|
+
return pruneInternal()
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return { put, get, has, stats, prune, minSize, cacheDir }
|
|
157
|
+
}
|
|
@@ -0,0 +1,124 @@
|
|
|
1
|
+
// Progressive disclosure for huge tool_result bodies (Phase 5).
|
|
2
|
+
//
|
|
3
|
+
// Given a tool_result text body >= 32 KB, emit a 3-tier summary in the
|
|
4
|
+
// outgoing turn (signature + head excerpt + ellipsis + tail excerpt + marker),
|
|
5
|
+
// and retain the full body in the br-cache (built in Phase 3). If the model
|
|
6
|
+
// later quotes the marker in a follow-up request, `rehydrateReferences`
|
|
7
|
+
// injects the full body back in before we forward upstream.
|
|
8
|
+
//
|
|
9
|
+
// Marker format - strict, regex-extractable, non-ambiguous:
|
|
10
|
+
// <tamp-ref:v1:HEXHASH:BYTES>
|
|
11
|
+
// v1 = format version. BYTES = original body byte length.
|
|
12
|
+
|
|
13
|
+
import { createHash } from 'node:crypto'
|
|
14
|
+
|
|
15
|
+
export const REF_MARKER = /<tamp-ref:v1:([a-f0-9]{64}):(\d+)>/g
|
|
16
|
+
|
|
17
|
+
function sha256(text) {
|
|
18
|
+
return createHash('sha256').update(text).digest('hex')
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
function utf8Bytes(text) {
|
|
22
|
+
return Buffer.byteLength(text, 'utf8')
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
// Slice by UTF-8 byte count without splitting a codepoint mid-sequence.
|
|
26
|
+
function sliceHeadBytes(text, bytes) {
|
|
27
|
+
const buf = Buffer.from(text, 'utf8')
|
|
28
|
+
if (buf.length <= bytes) return text
|
|
29
|
+
let cut = bytes
|
|
30
|
+
while (cut > 0 && (buf[cut] & 0xC0) === 0x80) cut -= 1
|
|
31
|
+
return buf.slice(0, cut).toString('utf8')
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
function sliceTailBytes(text, bytes) {
|
|
35
|
+
const buf = Buffer.from(text, 'utf8')
|
|
36
|
+
if (buf.length <= bytes) return text
|
|
37
|
+
let cut = buf.length - bytes
|
|
38
|
+
while (cut < buf.length && (buf[cut] & 0xC0) === 0x80) cut += 1
|
|
39
|
+
return buf.slice(cut).toString('utf8')
|
|
40
|
+
}
|
|
41
|
+
|
|
42
|
+
function formatSize(bytes) {
|
|
43
|
+
if (bytes >= 1024 * 1024) return `${(bytes / (1024 * 1024)).toFixed(1)} MB`
|
|
44
|
+
if (bytes >= 1024) return `${(bytes / 1024).toFixed(1)} KB`
|
|
45
|
+
return `${bytes} B`
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
/**
|
|
49
|
+
* Build a 3-tier summary block for a large tool_result body.
|
|
50
|
+
* Returns null if the body is smaller than minSize.
|
|
51
|
+
* Shape: { summary, hash, originalBytes, summaryBytes }.
|
|
52
|
+
*/
|
|
53
|
+
export function summarize(text, { hash, minSize = 32 * 1024, headBytes = 2048, tailBytes = 1024 } = {}) {
|
|
54
|
+
if (typeof text !== 'string') return null
|
|
55
|
+
const originalBytes = utf8Bytes(text)
|
|
56
|
+
if (originalBytes < minSize) return null
|
|
57
|
+
|
|
58
|
+
const fullHash = hash || sha256(text)
|
|
59
|
+
const head = sliceHeadBytes(text, headBytes)
|
|
60
|
+
const tail = sliceTailBytes(text, tailBytes)
|
|
61
|
+
const omitted = originalBytes - utf8Bytes(head) - utf8Bytes(tail)
|
|
62
|
+
|
|
63
|
+
const signature = `[tamp-disclosure v1] ${formatSize(originalBytes)}, sha256:${fullHash.slice(0, 12)}... - quote <tamp-ref:v1:${fullHash}:${originalBytes}> to expand`
|
|
64
|
+
const ellipsis = `\n[... ${formatSize(Math.max(0, omitted))} omitted ...]\n`
|
|
65
|
+
const marker = `<tamp-ref:v1:${fullHash}:${originalBytes}>`
|
|
66
|
+
|
|
67
|
+
const summary = `${signature}\n${head}${ellipsis}${tail}\n${marker}`
|
|
68
|
+
return { summary, hash: fullHash, originalBytes, summaryBytes: utf8Bytes(summary) }
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
function freshMarkerRegex() {
|
|
72
|
+
return new RegExp(REF_MARKER.source, REF_MARKER.flags)
|
|
73
|
+
}
|
|
74
|
+
|
|
75
|
+
/**
|
|
76
|
+
* Scan a provider request body for <tamp-ref:v1:HASH:BYTES> markers that
|
|
77
|
+
* the model quoted back. Returns an array of { hash, bytes, path, match }.
|
|
78
|
+
* Path and shape are defined by the provider's findReferences method.
|
|
79
|
+
*/
|
|
80
|
+
export function findReferenceQuotes(body, provider) {
|
|
81
|
+
if (typeof provider?.findReferences === 'function') {
|
|
82
|
+
return provider.findReferences(body)
|
|
83
|
+
}
|
|
84
|
+
return []
|
|
85
|
+
}
|
|
86
|
+
|
|
87
|
+
/**
|
|
88
|
+
* Rehydrate the previous turn's full body into the outgoing request.
|
|
89
|
+
* For each reference quote found, look up the hash in brCache and replace
|
|
90
|
+
* the marker with an expanded block. Safe no-op on cache miss.
|
|
91
|
+
*/
|
|
92
|
+
export function rehydrateReferences(body, provider, brCache) {
|
|
93
|
+
if (!provider || !brCache || typeof brCache.get !== 'function') return { rehydrated: 0, missed: 0 }
|
|
94
|
+
const refs = findReferenceQuotes(body, provider)
|
|
95
|
+
if (!refs.length) return { rehydrated: 0, missed: 0 }
|
|
96
|
+
|
|
97
|
+
const seen = new Map()
|
|
98
|
+
let rehydrated = 0
|
|
99
|
+
let missed = 0
|
|
100
|
+
|
|
101
|
+
for (const ref of refs) {
|
|
102
|
+
let entry = seen.get(ref.hash)
|
|
103
|
+
if (!entry) {
|
|
104
|
+
const full = brCache.get(ref.hash)
|
|
105
|
+
if (full == null) {
|
|
106
|
+
entry = { full: null }
|
|
107
|
+
} else {
|
|
108
|
+
const header = `<tamp-ref:v1:${ref.hash} expanded - ${utf8Bytes(full)} bytes follow>`
|
|
109
|
+
entry = { full: `${header}\n${full}\n</tamp-ref expanded>` }
|
|
110
|
+
}
|
|
111
|
+
seen.set(ref.hash, entry)
|
|
112
|
+
}
|
|
113
|
+
if (entry.full == null) { missed += 1; ref.missed = true }
|
|
114
|
+
else { rehydrated += 1; ref.expansion = entry.full }
|
|
115
|
+
}
|
|
116
|
+
|
|
117
|
+
if (rehydrated === 0) return { rehydrated: 0, missed }
|
|
118
|
+
if (typeof provider.applyRehydration === 'function') {
|
|
119
|
+
provider.applyRehydration(body, refs)
|
|
120
|
+
}
|
|
121
|
+
return { rehydrated, missed }
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
export const _internal = { sliceHeadBytes, sliceTailBytes, utf8Bytes, freshMarkerRegex, sha256 }
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
// Given a `targets` array and the raw body, return a parallel array of
|
|
2
|
+
// `path | null` such that targets[i] belongs to file paths[i].
|
|
3
|
+
// Strategy: scan message history for tool_use blocks whose id matches the
|
|
4
|
+
// tool_use_id of the tool_result; extract input.file_path (Anthropic Read),
|
|
5
|
+
// input.path, or input.filePath. Provider-specific — Anthropic only.
|
|
6
|
+
|
|
7
|
+
function buildToolUseIndex(messages) {
|
|
8
|
+
// Map<tool_use_id, { file_path }>. Scan assistant messages for tool_use
|
|
9
|
+
// blocks and record their resolved file path once.
|
|
10
|
+
const index = new Map()
|
|
11
|
+
if (!Array.isArray(messages)) return index
|
|
12
|
+
|
|
13
|
+
for (const msg of messages) {
|
|
14
|
+
if (msg?.role !== 'assistant' || !Array.isArray(msg.content)) continue
|
|
15
|
+
for (const block of msg.content) {
|
|
16
|
+
if (block?.type !== 'tool_use' || !block.id || !block.input) continue
|
|
17
|
+
const input = block.input
|
|
18
|
+
const filePath = input.file_path || input.path || input.filePath
|
|
19
|
+
if (typeof filePath === 'string' && filePath.length > 0) {
|
|
20
|
+
index.set(block.id, filePath)
|
|
21
|
+
}
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
return index
|
|
25
|
+
}
|
|
26
|
+
|
|
27
|
+
function findToolUseIdForTarget(body, target) {
|
|
28
|
+
// Target path shape (Anthropic): ['messages', mi, 'content', i, ...]
|
|
29
|
+
// The tool_result block lives at body.messages[mi].content[i].
|
|
30
|
+
const path = target?.path
|
|
31
|
+
if (!Array.isArray(path) || path[0] !== 'messages') return null
|
|
32
|
+
const mi = path[1]
|
|
33
|
+
const ci = path[3]
|
|
34
|
+
const msg = body?.messages?.[mi]
|
|
35
|
+
if (!msg || !Array.isArray(msg.content)) return null
|
|
36
|
+
const block = msg.content[ci]
|
|
37
|
+
if (!block || block.type !== 'tool_result') return null
|
|
38
|
+
return typeof block.tool_use_id === 'string' ? block.tool_use_id : null
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
export function extractTargetPaths(body, targets) {
|
|
42
|
+
const index = buildToolUseIndex(body?.messages)
|
|
43
|
+
return targets.map(target => {
|
|
44
|
+
if (!target || target.skip) return null
|
|
45
|
+
const toolUseId = findToolUseIdForTarget(body, target)
|
|
46
|
+
if (!toolUseId) return null
|
|
47
|
+
return index.get(toolUseId) || null
|
|
48
|
+
})
|
|
49
|
+
}
|
|
@@ -0,0 +1,55 @@
|
|
|
1
|
+
// Session-scoped LRU cache of last-seen Read bodies, keyed by
|
|
2
|
+
// `sessionKey + ':' + path`. Used by the `read-diff` stage to replace a
|
|
3
|
+
// re-read of a known file with a unified diff against the prior copy.
|
|
4
|
+
// Map insertion order gives us LRU for free.
|
|
5
|
+
|
|
6
|
+
const DEFAULT_TTL_MS = 2 * 60 * 60 * 1000
|
|
7
|
+
const DEFAULT_MAX_ENTRIES = 256
|
|
8
|
+
|
|
9
|
+
function makeKey(sessionKey, path) {
|
|
10
|
+
return `${sessionKey}:${path}`
|
|
11
|
+
}
|
|
12
|
+
|
|
13
|
+
export function createReadCache({ maxEntries = DEFAULT_MAX_ENTRIES, ttlMs = DEFAULT_TTL_MS } = {}) {
|
|
14
|
+
const store = new Map()
|
|
15
|
+
let hits = 0
|
|
16
|
+
let misses = 0
|
|
17
|
+
|
|
18
|
+
function prune() {
|
|
19
|
+
const cutoff = Date.now() - ttlMs
|
|
20
|
+
for (const [k, entry] of store) {
|
|
21
|
+
if (entry.lastSeen < cutoff) store.delete(k)
|
|
22
|
+
}
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
function get(sessionKey, path) {
|
|
26
|
+
if (!sessionKey || !path) return null
|
|
27
|
+
prune()
|
|
28
|
+
const key = makeKey(sessionKey, path)
|
|
29
|
+
const entry = store.get(key)
|
|
30
|
+
if (!entry) { misses += 1; return null }
|
|
31
|
+
// Refresh LRU position.
|
|
32
|
+
store.delete(key)
|
|
33
|
+
entry.lastSeen = Date.now()
|
|
34
|
+
store.set(key, entry)
|
|
35
|
+
hits += 1
|
|
36
|
+
return entry.text
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function put(sessionKey, path, text) {
|
|
40
|
+
if (!sessionKey || !path || typeof text !== 'string') return
|
|
41
|
+
const key = makeKey(sessionKey, path)
|
|
42
|
+
if (store.has(key)) store.delete(key)
|
|
43
|
+
else if (store.size >= maxEntries) {
|
|
44
|
+
const oldest = store.keys().next().value
|
|
45
|
+
store.delete(oldest)
|
|
46
|
+
}
|
|
47
|
+
store.set(key, { text, lastSeen: Date.now() })
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
function stats() {
|
|
51
|
+
return { entries: store.size, hits, misses }
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
return { get, put, stats, prune, _store: store }
|
|
55
|
+
}
|
|
@@ -0,0 +1,50 @@
|
|
|
1
|
+
// Targets:
|
|
2
|
+
// cargo build / cargo check / cargo run / cargo test
|
|
3
|
+
//
|
|
4
|
+
// Strips: " Compiling foo v1.0.0 (path)" intermediate lines when followed by
|
|
5
|
+
// a "Finished" line; progress bar spinner lines like " Building [===> ]".
|
|
6
|
+
// Keeps: "Finished" line, warnings, errors, test results, panic traces.
|
|
7
|
+
|
|
8
|
+
const COMPILING_LINE = /^\s{2,}Compiling\s+[A-Za-z0-9_.\-]+\s+v[\d.]+/;
|
|
9
|
+
const PROGRESS_LINE = /^\s*Building\s+\[[=> ]+\]/;
|
|
10
|
+
const DOWNLOADING = /^\s{2,}(Downloading|Downloaded)\s+[A-Za-z0-9_.\-]+\s+v[\d.]+/;
|
|
11
|
+
const CARGO_SIG = /(^|\n)(\s{2,}(Compiling|Finished|Building|Downloading|Downloaded|Running|Fresh)\s|error\[E\d+\]|warning:\s)/;
|
|
12
|
+
|
|
13
|
+
export default {
|
|
14
|
+
name: 'cargo',
|
|
15
|
+
|
|
16
|
+
match(text) {
|
|
17
|
+
if (typeof text !== 'string' || text.length === 0) return false;
|
|
18
|
+
return CARGO_SIG.test(text);
|
|
19
|
+
},
|
|
20
|
+
|
|
21
|
+
rewrite(text) {
|
|
22
|
+
if (!text) return text;
|
|
23
|
+
|
|
24
|
+
// Only strip Compiling lines when a Finished line is present (build succeeded).
|
|
25
|
+
const hasFinished = /^\s{2,}Finished\s+/m.test(text);
|
|
26
|
+
|
|
27
|
+
const lines = text.split('\n');
|
|
28
|
+
const out = [];
|
|
29
|
+
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
if (PROGRESS_LINE.test(line)) continue;
|
|
32
|
+
if (DOWNLOADING.test(line)) continue;
|
|
33
|
+
if (hasFinished && COMPILING_LINE.test(line)) continue;
|
|
34
|
+
out.push(line);
|
|
35
|
+
}
|
|
36
|
+
|
|
37
|
+
const collapsed = [];
|
|
38
|
+
let blank = 0;
|
|
39
|
+
for (const l of out) {
|
|
40
|
+
if (l.trim() === '') {
|
|
41
|
+
blank++;
|
|
42
|
+
if (blank > 1) continue;
|
|
43
|
+
} else {
|
|
44
|
+
blank = 0;
|
|
45
|
+
}
|
|
46
|
+
collapsed.push(l);
|
|
47
|
+
}
|
|
48
|
+
return collapsed.join('\n');
|
|
49
|
+
},
|
|
50
|
+
};
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
// Targets:
|
|
2
|
+
// docker build ...
|
|
3
|
+
// docker buildx build ...
|
|
4
|
+
//
|
|
5
|
+
// Strips: BuildKit spinner lines ("#5 [2/7] RUN ... 0.3s"), transferred-bytes
|
|
6
|
+
// progress ("=> => transferring context: 123kB 0.1s"), cache-hit duplicates
|
|
7
|
+
// "=> CACHED [foo] ..." retained, but "=> => resolve sha256:..." noise stripped.
|
|
8
|
+
// Keeps: "Successfully built <id>", "Successfully tagged <name>", ERROR lines,
|
|
9
|
+
// final image sha, step headings.
|
|
10
|
+
|
|
11
|
+
const BUILDKIT_SUB = /^#\d+\s+=>\s+=>\s+/; // "#5 => => transferring ..." / "=> => resolve sha256:..."
|
|
12
|
+
const BUILDKIT_TRANSFER = /^=>\s+=>\s+(transferring|resolve|sha256|naming to|writing image|exporting)/;
|
|
13
|
+
const SPINNER_BRAILLE = /^[⠋⠙⠹⠸⠼⠴⠦⠧⠇⠏]/;
|
|
14
|
+
const LEGACY_PROGRESS = /^\s*[0-9a-f]{12}: (Downloading|Extracting|Pulling fs layer|Waiting|Verifying Checksum|Download complete|Pull complete)\s/;
|
|
15
|
+
const DOCKER_SIG = /(^|\n)(Successfully built |Successfully tagged |Step \d+\/\d+\s*:|Sending build context|#\d+\s+\[|ERROR: |=>\s+\[)/;
|
|
16
|
+
|
|
17
|
+
export default {
|
|
18
|
+
name: 'docker',
|
|
19
|
+
|
|
20
|
+
match(text) {
|
|
21
|
+
if (typeof text !== 'string' || text.length === 0) return false;
|
|
22
|
+
return DOCKER_SIG.test(text);
|
|
23
|
+
},
|
|
24
|
+
|
|
25
|
+
rewrite(text) {
|
|
26
|
+
if (!text) return text;
|
|
27
|
+
const lines = text.split('\n');
|
|
28
|
+
const out = [];
|
|
29
|
+
|
|
30
|
+
for (const line of lines) {
|
|
31
|
+
if (BUILDKIT_SUB.test(line)) continue;
|
|
32
|
+
if (BUILDKIT_TRANSFER.test(line)) continue;
|
|
33
|
+
if (SPINNER_BRAILLE.test(line)) continue;
|
|
34
|
+
if (LEGACY_PROGRESS.test(line)) continue;
|
|
35
|
+
// BuildKit "in-progress" duplicated step lines end with a fractional second
|
|
36
|
+
// marker and are re-emitted as DONE later; strip the non-DONE variants.
|
|
37
|
+
if (/^#\d+\s+\S.*\s\d+\.\d+s$/.test(line) && !/\sDONE\b/.test(line)) continue;
|
|
38
|
+
out.push(line);
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
const collapsed = [];
|
|
42
|
+
let blank = 0;
|
|
43
|
+
for (const l of out) {
|
|
44
|
+
if (l.trim() === '') {
|
|
45
|
+
blank++;
|
|
46
|
+
if (blank > 1) continue;
|
|
47
|
+
} else {
|
|
48
|
+
blank = 0;
|
|
49
|
+
}
|
|
50
|
+
collapsed.push(l);
|
|
51
|
+
}
|
|
52
|
+
return collapsed.join('\n');
|
|
53
|
+
},
|
|
54
|
+
};
|