@sliday/tamp 0.5.9 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/bin/lifecycle.js +16 -4
- package/compress.js +160 -6
- package/config.js +23 -2
- package/detect.js +10 -0
- package/index.js +67 -7
- package/lib/bm25.js +194 -0
- package/lib/br-cache.js +157 -0
- package/lib/disclosure.js +124 -0
- package/lib/path-extract.js +49 -0
- package/lib/read-cache.js +55 -0
- package/lib/rewriters/commands/cargo.js +50 -0
- package/lib/rewriters/commands/docker.js +54 -0
- package/lib/rewriters/commands/git.js +81 -0
- package/lib/rewriters/commands/jest.js +49 -0
- package/lib/rewriters/commands/npm.js +82 -0
- package/lib/rewriters/commands/pip.js +58 -0
- package/lib/rewriters/commands/prisma.js +50 -0
- package/lib/rewriters/commands/pytest.js +55 -0
- package/lib/rewriters/commands/terraform.js +49 -0
- package/lib/rewriters/commands/wget-curl.js +67 -0
- package/lib/rewriters/index.js +77 -0
- package/lib/rules-generator.js +45 -2
- package/metadata.js +37 -5
- package/package.json +2 -2
- package/providers.js +117 -0
- package/session-graph.js +36 -3
- package/stats.js +12 -1
package/providers.js
CHANGED
|
@@ -94,6 +94,117 @@ function appendToLastUserMessageAnthropic(messages, text) {
|
|
|
94
94
|
return false
|
|
95
95
|
}
|
|
96
96
|
|
|
97
|
+
// ---- Disclosure: find and rehydrate <tamp-ref:v1:...> quotes (Phase 5) ----
|
|
98
|
+
|
|
99
|
+
const DISCLOSURE_MARKER = /<tamp-ref:v1:([a-f0-9]{64}):(\d+)>/g
|
|
100
|
+
|
|
101
|
+
function scanStringForRefs(str, pathDesc, out) {
|
|
102
|
+
if (typeof str !== 'string' || str.indexOf('<tamp-ref:v1:') === -1) return
|
|
103
|
+
const re = new RegExp(DISCLOSURE_MARKER.source, DISCLOSURE_MARKER.flags)
|
|
104
|
+
let m
|
|
105
|
+
while ((m = re.exec(str)) !== null) {
|
|
106
|
+
out.push({ hash: m[1], bytes: Number(m[2]), match: m[0], path: pathDesc.slice() })
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
110
|
+
function findAnthropicReferences(body) {
|
|
111
|
+
const out = []
|
|
112
|
+
if (!Array.isArray(body?.messages)) return out
|
|
113
|
+
for (let mi = 0; mi < body.messages.length; mi++) {
|
|
114
|
+
const msg = body.messages[mi]
|
|
115
|
+
if (!msg) continue
|
|
116
|
+
if (typeof msg.content === 'string') {
|
|
117
|
+
scanStringForRefs(msg.content, ['messages', mi, 'content'], out)
|
|
118
|
+
continue
|
|
119
|
+
}
|
|
120
|
+
if (!Array.isArray(msg.content)) continue
|
|
121
|
+
for (let bi = 0; bi < msg.content.length; bi++) {
|
|
122
|
+
const block = msg.content[bi]
|
|
123
|
+
if (!block) continue
|
|
124
|
+
if (block.type === 'text' && typeof block.text === 'string') {
|
|
125
|
+
scanStringForRefs(block.text, ['messages', mi, 'content', bi, 'text'], out)
|
|
126
|
+
} else if (block.type === 'tool_use') {
|
|
127
|
+
if (block.input && typeof block.input === 'object') {
|
|
128
|
+
const serialized = JSON.stringify(block.input)
|
|
129
|
+
scanStringForRefs(serialized, ['messages', mi, 'content', bi, 'input'], out)
|
|
130
|
+
} else if (typeof block.input === 'string') {
|
|
131
|
+
scanStringForRefs(block.input, ['messages', mi, 'content', bi, 'input'], out)
|
|
132
|
+
}
|
|
133
|
+
} else if (block.type === 'tool_result') {
|
|
134
|
+
if (typeof block.content === 'string') {
|
|
135
|
+
scanStringForRefs(block.content, ['messages', mi, 'content', bi, 'content'], out)
|
|
136
|
+
} else if (Array.isArray(block.content)) {
|
|
137
|
+
for (let ci = 0; ci < block.content.length; ci++) {
|
|
138
|
+
const sub = block.content[ci]
|
|
139
|
+
if (sub?.type === 'text' && typeof sub.text === 'string') {
|
|
140
|
+
scanStringForRefs(sub.text, ['messages', mi, 'content', bi, 'content', ci, 'text'], out)
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
}
|
|
147
|
+
return out
|
|
148
|
+
}
|
|
149
|
+
|
|
150
|
+
function getAtPath(body, path) {
|
|
151
|
+
let obj = body
|
|
152
|
+
for (let i = 0; i < path.length; i++) obj = obj?.[path[i]]
|
|
153
|
+
return obj
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
function setAtPath(body, path, value) {
|
|
157
|
+
let obj = body
|
|
158
|
+
for (let i = 0; i < path.length - 1; i++) obj = obj[path[i]]
|
|
159
|
+
obj[path[path.length - 1]] = value
|
|
160
|
+
}
|
|
161
|
+
|
|
162
|
+
// Replace marker(s) at each ref's path with the expanded block. For
|
|
163
|
+
// tool_use.input objects we round-trip via JSON. Idempotent: if the marker
|
|
164
|
+
// has already been replaced by an expansion (which contains "expanded -"
|
|
165
|
+
// sentinel for the same hash), we skip further replacement.
|
|
166
|
+
function applyAnthropicRehydration(body, refs) {
|
|
167
|
+
const groups = new Map()
|
|
168
|
+
for (const ref of refs) {
|
|
169
|
+
if (!ref.expansion) continue
|
|
170
|
+
const key = ref.path.join('|')
|
|
171
|
+
if (!groups.has(key)) groups.set(key, { path: ref.path, items: [] })
|
|
172
|
+
groups.get(key).items.push(ref)
|
|
173
|
+
}
|
|
174
|
+
|
|
175
|
+
for (const { path, items } of groups.values()) {
|
|
176
|
+
const isToolUseInput = path[path.length - 1] === 'input'
|
|
177
|
+
let current
|
|
178
|
+
if (isToolUseInput) {
|
|
179
|
+
const val = getAtPath(body, path)
|
|
180
|
+
current = typeof val === 'string' ? val : JSON.stringify(val)
|
|
181
|
+
} else {
|
|
182
|
+
current = getAtPath(body, path)
|
|
183
|
+
}
|
|
184
|
+
if (typeof current !== 'string') continue
|
|
185
|
+
|
|
186
|
+
let updated = current
|
|
187
|
+
for (const ref of items) {
|
|
188
|
+
const expandedSentinel = `<tamp-ref:v1:${ref.hash} expanded -`
|
|
189
|
+
if (updated.indexOf(expandedSentinel) !== -1) continue
|
|
190
|
+
const idx = updated.indexOf(ref.match)
|
|
191
|
+
if (idx === -1) continue
|
|
192
|
+
updated = updated.slice(0, idx) + ref.expansion + updated.slice(idx + ref.match.length)
|
|
193
|
+
}
|
|
194
|
+
|
|
195
|
+
if (updated === current) continue
|
|
196
|
+
|
|
197
|
+
if (isToolUseInput) {
|
|
198
|
+
try {
|
|
199
|
+
const parsed = JSON.parse(updated)
|
|
200
|
+
setAtPath(body, path, parsed)
|
|
201
|
+
continue
|
|
202
|
+
} catch { /* fall through to string */ }
|
|
203
|
+
}
|
|
204
|
+
setAtPath(body, path, updated)
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
97
208
|
const anthropic = {
|
|
98
209
|
name: 'anthropic',
|
|
99
210
|
match(method, url) {
|
|
@@ -116,6 +227,12 @@ const anthropic = {
|
|
|
116
227
|
injectOutputHint(body, text) {
|
|
117
228
|
return appendToLastUserMessageAnthropic(body?.messages, text)
|
|
118
229
|
},
|
|
230
|
+
findReferences(body) {
|
|
231
|
+
return findAnthropicReferences(body)
|
|
232
|
+
},
|
|
233
|
+
applyRehydration(body, refs) {
|
|
234
|
+
return applyAnthropicRehydration(body, refs)
|
|
235
|
+
},
|
|
119
236
|
}
|
|
120
237
|
|
|
121
238
|
function extractOpenAIChatTargets(msg, i) {
|
package/session-graph.js
CHANGED
|
@@ -4,7 +4,7 @@ const SESSION_TTL_MS = 2 * 60 * 60 * 1000
|
|
|
4
4
|
const MAX_SESSIONS = 50
|
|
5
5
|
const MIN_BYTES = 512
|
|
6
6
|
|
|
7
|
-
export function createSessionStore({ ttlMs = SESSION_TTL_MS, maxSessions = MAX_SESSIONS } = {}) {
|
|
7
|
+
export function createSessionStore({ ttlMs = SESSION_TTL_MS, maxSessions = MAX_SESSIONS, brCache = null } = {}) {
|
|
8
8
|
const store = new Map()
|
|
9
9
|
|
|
10
10
|
function prune() {
|
|
@@ -26,7 +26,7 @@ export function createSessionStore({ ttlMs = SESSION_TTL_MS, maxSessions = MAX_S
|
|
|
26
26
|
const oldest = store.keys().next().value
|
|
27
27
|
store.delete(oldest)
|
|
28
28
|
}
|
|
29
|
-
bucket = {
|
|
29
|
+
bucket = createSessionBucket({ brCache })
|
|
30
30
|
store.set(key, bucket)
|
|
31
31
|
return bucket
|
|
32
32
|
}
|
|
@@ -34,6 +34,27 @@ export function createSessionStore({ ttlMs = SESSION_TTL_MS, maxSessions = MAX_S
|
|
|
34
34
|
return { getBucket, prune, _store: store }
|
|
35
35
|
}
|
|
36
36
|
|
|
37
|
+
// A bucket tracks per-session graph state. Optionally pipes large target
|
|
38
|
+
// bodies through a Brotli disk cache so the in-memory structure stays lean
|
|
39
|
+
// and content persists across proxy restarts for Phase 5 rehydration.
|
|
40
|
+
export function createSessionBucket({ brCache = null } = {}) {
|
|
41
|
+
const refs = new Map()
|
|
42
|
+
let brotliBytes = 0
|
|
43
|
+
let offloaded = 0
|
|
44
|
+
let hits = 0
|
|
45
|
+
|
|
46
|
+
return {
|
|
47
|
+
refs,
|
|
48
|
+
nextId: 0,
|
|
49
|
+
lastSeen: Date.now(),
|
|
50
|
+
brCache,
|
|
51
|
+
// Counters incremented by graphDeduplicateTargets when brCache is attached.
|
|
52
|
+
brStats() { return { brotliBytes, offloaded, hits } },
|
|
53
|
+
_recordOffload(bytes) { brotliBytes += bytes; offloaded += 1 },
|
|
54
|
+
_recordHit() { hits += 1 },
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
|
|
37
58
|
export function deriveSessionKey(headers) {
|
|
38
59
|
if (!headers) return null
|
|
39
60
|
const auth = headers.authorization || headers['x-api-key'] || ''
|
|
@@ -43,6 +64,7 @@ export function deriveSessionKey(headers) {
|
|
|
43
64
|
|
|
44
65
|
export function graphDeduplicateTargets(targets, bucket, { minBytes = MIN_BYTES } = {}) {
|
|
45
66
|
if (!bucket) return
|
|
67
|
+
const brCache = bucket.brCache
|
|
46
68
|
for (const target of targets) {
|
|
47
69
|
if (target.skip || target.dedup || target.diffed || target.compressed) continue
|
|
48
70
|
if (typeof target.text !== 'string' || target.text.length < minBytes) continue
|
|
@@ -52,9 +74,20 @@ export function graphDeduplicateTargets(targets, bucket, { minBytes = MIN_BYTES
|
|
|
52
74
|
if (existing) {
|
|
53
75
|
target.compressed = `<tamp-file-ref id="${existing.id}" sha="${hash}" bytes="${target.text.length}"/>`
|
|
54
76
|
target.graphed = true
|
|
77
|
+
if (brCache && existing.brHash) bucket._recordHit()
|
|
55
78
|
continue
|
|
56
79
|
}
|
|
57
80
|
bucket.nextId += 1
|
|
58
|
-
|
|
81
|
+
const entry = { id: bucket.nextId, bytes: target.text.length }
|
|
82
|
+
// Opportunistic offload to the disk store — Phase 3 collects the hash
|
|
83
|
+
// for forward-compat with Phase 5 rehydration; no markers change yet.
|
|
84
|
+
if (brCache && typeof brCache.put === 'function' && target.text.length >= brCache.minSize) {
|
|
85
|
+
const stored = brCache.put(target.text)
|
|
86
|
+
if (stored) {
|
|
87
|
+
entry.brHash = stored.hash
|
|
88
|
+
bucket._recordOffload(stored.brotliBytes)
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
bucket.refs.set(hash, entry)
|
|
59
92
|
}
|
|
60
93
|
}
|
package/stats.js
CHANGED
|
@@ -9,7 +9,7 @@ const c = {
|
|
|
9
9
|
red: '\x1b[31m',
|
|
10
10
|
}
|
|
11
11
|
|
|
12
|
-
export function formatRequestLog(stats, session, providerName, url, bodySize, tokenCost) {
|
|
12
|
+
export function formatRequestLog(stats, session, providerName, url, bodySize, tokenCost, sessionBucket) {
|
|
13
13
|
const compressed = stats.filter(s => s.method)
|
|
14
14
|
const label = providerName || 'anthropic'
|
|
15
15
|
const path = url || '/v1/messages'
|
|
@@ -58,6 +58,17 @@ export function formatRequestLog(stats, session, providerName, url, bodySize, to
|
|
|
58
58
|
lines.push(`[tamp] ${c.magenta}session${c.reset} ${fmtSize(t.totalSaved)} chars, ${t.totalTokensSaved} tokens saved across ${t.compressionCount} blocks ${c.dim}(${sessionPct}% avg)${c.reset}${moneyInfo}`)
|
|
59
59
|
}
|
|
60
60
|
|
|
61
|
+
// br-cache counters — only render when the bucket reports activity, so
|
|
62
|
+
// turning the stage off produces no extra rows.
|
|
63
|
+
if (sessionBucket && typeof sessionBucket.brStats === 'function') {
|
|
64
|
+
const br = sessionBucket.brStats()
|
|
65
|
+
if (br && (br.offloaded > 0 || br.hits > 0)) {
|
|
66
|
+
const totalLookups = br.offloaded + br.hits
|
|
67
|
+
const ratio = totalLookups > 0 ? ((br.hits / totalLookups) * 100).toFixed(1) : '0.0'
|
|
68
|
+
lines.push(`[tamp] ${c.cyan}br-cache${c.reset} ${fmtSize(br.brotliBytes)} stored, ${br.offloaded} offloaded, ${br.hits} hits ${c.dim}(${ratio}% hit ratio)${c.reset}`)
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
|
|
61
72
|
return lines.join('\n')
|
|
62
73
|
}
|
|
63
74
|
|