@sliday/tamp 0.8.21 → 0.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +2 -0
- package/compress.js +148 -6
- package/config.js +9 -0
- package/index.js +36 -3
- package/lib/px-render.js +55 -9
- package/lib/replay-memo.js +73 -0
- package/lib/usage-tap.js +360 -0
- package/package.json +2 -2
- package/providers.js +14 -6
package/README.md
CHANGED
|
@@ -301,6 +301,8 @@ Supported on all providers: Anthropic, OpenAI Chat, OpenAI Responses (Codex), Ge
|
|
|
301
301
|
| `TAMP_LOG` | `true` | Enable logging |
|
|
302
302
|
| `TAMP_LOG_FILE` | *(none)* | Append logs to this file path, in addition to stderr |
|
|
303
303
|
| `TAMP_CACHE_SAFE` | `true` | Compress newest only (prompt-cache safe) |
|
|
304
|
+
| `TAMP_REPLAY` | `true` | Re-send older tool results exactly as Tamp first sent them (savings persist, cache prefix stays stable, secrets stay masked) |
|
|
305
|
+
| `TAMP_REPLAY_MAX_BYTES` | `67108864` | Memory bound for the replay memo (bytes, 64 MiB). Least recently used entries go first |
|
|
304
306
|
| `TAMP_REDACT` | `true` | Mask secrets in tool output before anything leaves the machine |
|
|
305
307
|
| `TAMP_REDACT_MODE` | `mask` | `mask` (replace with marker) or `remove` (delete value) |
|
|
306
308
|
| `TAMP_LLMLINGUA_URL` | *(none)* | LLMLingua sidecar URL. Set to `http://localhost:8788` to skip the auto-probe (avoids startup race if sidecar is still loading the model) |
|
package/compress.js
CHANGED
|
@@ -14,7 +14,7 @@ import { evictStaleImages } from './lib/stale-images.js'
|
|
|
14
14
|
import { pruneStaleInputs } from './lib/stale-inputs.js'
|
|
15
15
|
import { summarize, rehydrateReferences } from './lib/disclosure.js'
|
|
16
16
|
import { trimLinesByRelevance } from './lib/bm25.js'
|
|
17
|
-
import { pxRenderTargets } from './lib/px-render.js'
|
|
17
|
+
import { pxRenderTargets, pxModelAllowed } from './lib/px-render.js'
|
|
18
18
|
|
|
19
19
|
const cache = new Map()
|
|
20
20
|
const MAX_CACHE = 500
|
|
@@ -53,6 +53,7 @@ function deduplicateTargets(targets) {
|
|
|
53
53
|
if (ref.length < target.text.length) {
|
|
54
54
|
target.compressed = ref
|
|
55
55
|
target.dedup = true
|
|
56
|
+
target.refTarget = prev
|
|
56
57
|
}
|
|
57
58
|
} else {
|
|
58
59
|
seen.set(key, target)
|
|
@@ -79,6 +80,7 @@ function diffTargets(targets) {
|
|
|
79
80
|
if (diffBody.length < target.text.length * 0.5) {
|
|
80
81
|
target.compressed = `[diff from tool_result in message ${prev.path[1]}, block ${prev.index}]:\n${diffBody}`
|
|
81
82
|
target.diffed = true
|
|
83
|
+
target.refTarget = prev
|
|
82
84
|
break
|
|
83
85
|
}
|
|
84
86
|
}
|
|
@@ -120,7 +122,118 @@ function exemptProtectedTools(targets, body, provider) {
|
|
|
120
122
|
return exempted
|
|
121
123
|
}
|
|
122
124
|
|
|
123
|
-
|
|
125
|
+
// --- Replay memo (see lib/replay-memo.js) ---
|
|
126
|
+
// Key = provider + conversation + position + tool id + hash of the RAW text.
|
|
127
|
+
// Position and conversation keep deterministic tool ids (`functions.read:0`)
|
|
128
|
+
// and identical bodies from sharing an entry; the hash catches a client that
|
|
129
|
+
// compacted or edited the body since.
|
|
130
|
+
function sha(text) {
|
|
131
|
+
return createHash('sha256').update(text).digest('base64')
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
function conversationScope(body) {
|
|
135
|
+
const turns = body?.messages || body?.input || body?.contents
|
|
136
|
+
if (!Array.isArray(turns) || !turns.length) return ''
|
|
137
|
+
// The first turn can be a system prompt every conversation shares (OpenAI
|
|
138
|
+
// chat, Kimi), so the first user turn is part of the identity too.
|
|
139
|
+
const seed = [turns[0], turns.find(t => t?.role === 'user') ?? null]
|
|
140
|
+
// cache_control markers move between requests; they are not identity.
|
|
141
|
+
try { return sha(JSON.stringify(seed, (k, v) => (k === 'cache_control' ? undefined : v))) } catch { return '' }
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function replayKey(provider, scope, target) {
|
|
145
|
+
return `${provider.name}\0${scope}\0${target.path.join('/')}\0${target.id || ''}\0${target.rawHash}`
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
// px-render values are block arrays; never share one object between the memo
|
|
149
|
+
// and a body that later stages (stale-images) may mutate.
|
|
150
|
+
function cloneReplayValue(value) {
|
|
151
|
+
return Array.isArray(value) ? structuredClone(value) : value
|
|
152
|
+
}
|
|
153
|
+
|
|
154
|
+
function replayWrongModel(entry, body, config) {
|
|
155
|
+
return Array.isArray(entry.value) && !pxModelAllowed(body?.model, config.pxModels)
|
|
156
|
+
}
|
|
157
|
+
|
|
158
|
+
// A memo entry is usable only if what it points at is still in this request:
|
|
159
|
+
// image blocks need a model that reads them, and a dedup/diff/read-diff marker
|
|
160
|
+
// needs its referent (a client can clear or drop the block it refers to).
|
|
161
|
+
function replayUsable(entry, body, config, hashByPath, hashes) {
|
|
162
|
+
if (replayWrongModel(entry, body, config)) return false
|
|
163
|
+
const ref = entry.meta.ref
|
|
164
|
+
if (!ref) return true
|
|
165
|
+
return ref.path ? hashByPath.get(ref.path) === ref.hash : hashes.has(ref.hash)
|
|
166
|
+
}
|
|
167
|
+
|
|
168
|
+
function applyReplay(body, provider, target, entry, replay) {
|
|
169
|
+
// Through provider.apply so Gemini's wasObject JSON handling still runs.
|
|
170
|
+
// A null entry means "went out unchanged": nothing to write, not a hit.
|
|
171
|
+
if (entry.value === null) return
|
|
172
|
+
provider.apply(body, [{ ...target, skip: undefined, compressed: cloneReplayValue(entry.value) }])
|
|
173
|
+
replay.hits += 1
|
|
174
|
+
replay.tokensHeld += (entry.meta.originalTokens || 0) - (entry.meta.compressedTokens || 0)
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
// Makes history byte-stable and keeps it redacted. Newest-group targets with a
|
|
178
|
+
// memo hit are replayed and skipped. Older turns (cacheSafe) are never
|
|
179
|
+
// compressed fresh: they get their memo entry, or failing that (restart,
|
|
180
|
+
// eviction, replay off) deterministic redaction only, so a masked secret
|
|
181
|
+
// cannot go out in clear one request later.
|
|
182
|
+
function stabilizeHistory(body, targets, config, provider, memo, replay) {
|
|
183
|
+
// Collect older turns before touching the body: a replayed px-render array
|
|
184
|
+
// would otherwise surface as new sub-text targets.
|
|
185
|
+
let older = []
|
|
186
|
+
if (config.cacheSafe !== false) {
|
|
187
|
+
const latest = new Set(targets.filter(t => t.path).map(t => t.path.join('/')))
|
|
188
|
+
older = provider.extract(body, { ...config, cacheSafe: false })
|
|
189
|
+
exemptProtectedTools(older, body, provider)
|
|
190
|
+
older = older.filter(t => !t.skip && typeof t.text === 'string' && !latest.has(t.path.join('/')))
|
|
191
|
+
}
|
|
192
|
+
if (!memo && (config.redact === false || !older.length)) return
|
|
193
|
+
|
|
194
|
+
const scope = memo ? conversationScope(body) : ''
|
|
195
|
+
const hashByPath = new Map()
|
|
196
|
+
if (memo) {
|
|
197
|
+
for (const t of [...older, ...targets]) {
|
|
198
|
+
if (t.skip || typeof t.text !== 'string') continue
|
|
199
|
+
t.rawHash = sha(t.text)
|
|
200
|
+
hashByPath.set(t.path.join('/'), t.rawHash)
|
|
201
|
+
}
|
|
202
|
+
}
|
|
203
|
+
const hashes = new Set(hashByPath.values())
|
|
204
|
+
|
|
205
|
+
if (memo) {
|
|
206
|
+
for (const target of targets) {
|
|
207
|
+
if (!target.rawHash) continue
|
|
208
|
+
target.replayKey = replayKey(provider, scope, target)
|
|
209
|
+
const entry = memo.get(target.replayKey)
|
|
210
|
+
if (!entry || !replayUsable(entry, body, config, hashByPath, hashes)) {
|
|
211
|
+
// Another model's request must not overwrite images stored for this one.
|
|
212
|
+
if (entry && replayWrongModel(entry, body, config)) target.replayKey = null
|
|
213
|
+
replay.misses += 1
|
|
214
|
+
continue
|
|
215
|
+
}
|
|
216
|
+
applyReplay(body, provider, target, entry, replay)
|
|
217
|
+
target.skip = 'replayed'
|
|
218
|
+
}
|
|
219
|
+
}
|
|
220
|
+
|
|
221
|
+
for (const target of older) {
|
|
222
|
+
const key = memo && replayKey(provider, scope, target)
|
|
223
|
+
const entry = memo?.get(key)
|
|
224
|
+
if (entry && replayUsable(entry, body, config, hashByPath, hashes)) { applyReplay(body, provider, target, entry, replay); continue }
|
|
225
|
+
if (memo) replay.misses += 1
|
|
226
|
+
let value = null
|
|
227
|
+
if (config.redact !== false) {
|
|
228
|
+
const r = redactText(target.text, config.redactMode)
|
|
229
|
+
if (r.count > 0) { value = r.text; provider.apply(body, [{ ...target, compressed: value }]) }
|
|
230
|
+
}
|
|
231
|
+
// Remembering the outcome (deterministic) spares the regex pass next time.
|
|
232
|
+
if (memo && !(entry && replayWrongModel(entry, body, config))) memo.put(key, value, {})
|
|
233
|
+
}
|
|
234
|
+
}
|
|
235
|
+
|
|
236
|
+
function readDiffTargets(targets, body, sessionKey, readCache, memo) {
|
|
124
237
|
if (!readCache || !sessionKey) return
|
|
125
238
|
const paths = extractTargetPaths(body, targets)
|
|
126
239
|
for (let i = 0; i < targets.length; i++) {
|
|
@@ -130,7 +243,10 @@ function readDiffTargets(targets, body, sessionKey, readCache) {
|
|
|
130
243
|
if (target.skip || target.dedup || target.diffed || target.compressed) {
|
|
131
244
|
// Even for already-compressed targets, refresh the cache with the
|
|
132
245
|
// original text so the next request can diff against the latest copy.
|
|
133
|
-
if (typeof target.text === 'string')
|
|
246
|
+
if (typeof target.text === 'string') {
|
|
247
|
+
readCache.put(sessionKey, path, target.text)
|
|
248
|
+
memo?.setAnchor?.(`${sessionKey}:${path}`, target.rawHash)
|
|
249
|
+
}
|
|
134
250
|
continue
|
|
135
251
|
}
|
|
136
252
|
if (typeof target.text !== 'string') continue
|
|
@@ -141,9 +257,11 @@ function readDiffTargets(targets, body, sessionKey, readCache) {
|
|
|
141
257
|
if (patch.length < target.text.length * 0.5) {
|
|
142
258
|
target.compressed = `[read-diff from prior read of ${path}]:\n${patch}`
|
|
143
259
|
target.readDiffed = true
|
|
260
|
+
target.refHash = memo?.getAnchor?.(`${sessionKey}:${path}`)
|
|
144
261
|
}
|
|
145
262
|
}
|
|
146
263
|
readCache.put(sessionKey, path, target.text)
|
|
264
|
+
memo?.setAnchor?.(`${sessionKey}:${path}`, target.rawHash)
|
|
147
265
|
}
|
|
148
266
|
}
|
|
149
267
|
|
|
@@ -710,6 +828,11 @@ export async function compressRequest(body, config, provider) {
|
|
|
710
828
|
// so the forgone compression costs nothing.
|
|
711
829
|
exemptProtectedTools(targets, body, provider)
|
|
712
830
|
|
|
831
|
+
// Before redaction, so memo keys hash the raw text the client sent.
|
|
832
|
+
const memo = config.replayMemo || null
|
|
833
|
+
const replay = memo ? { hits: 0, misses: 0, tokensHeld: 0 } : null
|
|
834
|
+
stabilizeHistory(body, targets, config, provider, memo, replay)
|
|
835
|
+
|
|
713
836
|
// Redaction runs FIRST (security, not compression): mask secrets in every
|
|
714
837
|
// target's text before any stage — dedup/diff/cache/llmlingua/textpress —
|
|
715
838
|
// sees it, so secrets never enter a disk cache or get shipped to a third
|
|
@@ -718,7 +841,9 @@ export async function compressRequest(body, config, provider) {
|
|
|
718
841
|
let redactedCount = 0
|
|
719
842
|
if (config.redact !== false) {
|
|
720
843
|
for (const target of targets) {
|
|
721
|
-
|
|
844
|
+
// Replayed targets are already masked on the wire, but read-diff still
|
|
845
|
+
// caches their text; a raw prior there would leak via a later patch.
|
|
846
|
+
if ((target.skip && target.skip !== 'replayed') || typeof target.text !== 'string') continue
|
|
722
847
|
const r = redactText(target.text, config.redactMode)
|
|
723
848
|
if (r.count > 0) { target.text = r.text; target.redacted = true; redactedCount += r.count }
|
|
724
849
|
}
|
|
@@ -726,7 +851,7 @@ export async function compressRequest(body, config, provider) {
|
|
|
726
851
|
|
|
727
852
|
// Read-diff: session-scoped unified diff for re-reads (lossless, opt-in)
|
|
728
853
|
if (config.stages.includes('read-diff') && config.readCache && config.sessionKey && provider.name === 'anthropic') {
|
|
729
|
-
readDiffTargets(targets, body, config.sessionKey, config.readCache)
|
|
854
|
+
readDiffTargets(targets, body, config.sessionKey, config.readCache, config.replayMemo)
|
|
730
855
|
}
|
|
731
856
|
|
|
732
857
|
// Dedup: replace identical blocks with reference markers
|
|
@@ -817,6 +942,23 @@ export async function compressRequest(body, config, provider) {
|
|
|
817
942
|
}
|
|
818
943
|
provider.apply(body, targets)
|
|
819
944
|
|
|
945
|
+
// Remember what went out. stats[i] belongs to targets[i]: the loop above
|
|
946
|
+
// pushes exactly one entry per target.
|
|
947
|
+
if (memo) {
|
|
948
|
+
for (let i = 0; i < targets.length; i++) {
|
|
949
|
+
const target = targets[i]
|
|
950
|
+
if (!target.replayKey || target.skip) continue
|
|
951
|
+
// A marker is replayable only while its referent can be re-checked.
|
|
952
|
+
// graph refs point into session state the memo cannot see: not stored.
|
|
953
|
+
let ref
|
|
954
|
+
if (target.graphed) continue
|
|
955
|
+
if (target.dedup || target.diffed) ref = { path: target.refTarget.path.join('/'), hash: target.refTarget.rawHash }
|
|
956
|
+
else if (target.readDiffed) { if (!target.refHash) continue; ref = { hash: target.refHash } }
|
|
957
|
+
const value = target.compressed ? cloneReplayValue(target.compressed) : null
|
|
958
|
+
memo.put(target.replayKey, value, { originalTokens: stats[i].originalTokens, compressedTokens: stats[i].compressedTokens, ref })
|
|
959
|
+
}
|
|
960
|
+
}
|
|
961
|
+
|
|
820
962
|
// stale-images: drop base64 images from older turns (opt-in, lossy). Runs
|
|
821
963
|
// after apply so the extractor never sees the notices it leaves behind.
|
|
822
964
|
let staleImages = null
|
|
@@ -832,7 +974,7 @@ export async function compressRequest(body, config, provider) {
|
|
|
832
974
|
if (r.pruned > 0) staleInputs = r
|
|
833
975
|
}
|
|
834
976
|
|
|
835
|
-
return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount, staleImages, staleInputs }
|
|
977
|
+
return { body, stats, targetCount: targets.length, outputHint, disclosure: disclosureStats, redacted: redactedCount, staleImages, staleInputs, replay }
|
|
836
978
|
}
|
|
837
979
|
|
|
838
980
|
export async function compressMessages(body, config) {
|
package/config.js
CHANGED
|
@@ -242,6 +242,10 @@ export function loadConfig(env = process.env, options = {}) {
|
|
|
242
242
|
logFile: get('TAMP_LOG_FILE') || null,
|
|
243
243
|
maxBody: parseInt(get('TAMP_MAX_BODY'), 10) || 10_485_760,
|
|
244
244
|
cacheSafe: parseBoolean(get('TAMP_CACHE_SAFE'), true),
|
|
245
|
+
// Replay memo: re-send each tool_result exactly as tamp first emitted it,
|
|
246
|
+
// so savings, prompt-cache prefixes and redaction survive later requests.
|
|
247
|
+
replay: parseBoolean(get('TAMP_REPLAY'), true),
|
|
248
|
+
replayMaxBytes: parseInt(get('TAMP_REPLAY_MAX_BYTES'), 10) || 67108864,
|
|
245
249
|
// Secret redaction is a security pass, not a compression stage: on by
|
|
246
250
|
// default at every level, masks secrets before any stage (incl. ones that
|
|
247
251
|
// ship text to third parties) sees the body. TAMP_REDACT=false disables it.
|
|
@@ -313,6 +317,11 @@ export const CONFIG_TEMPLATE = `# Tamp configuration
|
|
|
313
317
|
# TAMP_PX_MIN_CHARS=4096
|
|
314
318
|
# TAMP_PX_MAX_IMAGES=6
|
|
315
319
|
|
|
320
|
+
# Replay memo: older tool_results go out exactly as first sent (stable cache
|
|
321
|
+
# prefix, secrets stay masked). In-memory, bounded by TAMP_REPLAY_MAX_BYTES.
|
|
322
|
+
# TAMP_REPLAY=true
|
|
323
|
+
# TAMP_REPLAY_MAX_BYTES=67108864
|
|
324
|
+
|
|
316
325
|
# TAMP_MIN_SIZE=200
|
|
317
326
|
# TAMP_LOG=true
|
|
318
327
|
# TAMP_LOG_FILE=
|
package/index.js
CHANGED
|
@@ -10,6 +10,8 @@ import { createSession, formatRequestLog, redactUrl } from './stats.js'
|
|
|
10
10
|
import { createSessionStore, deriveSessionKey } from './session-graph.js'
|
|
11
11
|
import { createReadCache } from './lib/read-cache.js'
|
|
12
12
|
import { createBrCache } from './lib/br-cache.js'
|
|
13
|
+
import { createReplayMemo } from './lib/replay-memo.js'
|
|
14
|
+
import { createUsageTap, createUsageTotals } from './lib/usage-tap.js'
|
|
13
15
|
import { generateOutputRules, PER_AGENT_OVERRIDES } from './lib/rules-generator.js'
|
|
14
16
|
|
|
15
17
|
// Regex source strings used by the task classifier — surfaced via
|
|
@@ -92,11 +94,12 @@ export function createProxy(overrides = {}, loadOpts = {}) {
|
|
|
92
94
|
: null
|
|
93
95
|
const sessionStore = createSessionStore({ brCache })
|
|
94
96
|
const readCache = createReadCache()
|
|
97
|
+
const replayMemo = config.replay ? createReplayMemo({ maxBytes: config.replayMaxBytes }) : null
|
|
95
98
|
// `tamp -y` starts the sidecar itself and hands the URL down via env, so the
|
|
96
99
|
// isMain probe below never runs and /health kept reporting sidecar:false
|
|
97
100
|
// while llmlingua was in fact wired up. Probe here too, quietly.
|
|
98
101
|
if (config.stages?.includes('llmlingua') && config.llmLinguaUrl) probeSidecar(config, true)
|
|
99
|
-
return { config, session, sessionStore, readCache, brCache, server: _createServer(config, session, sessionStore, readCache, brCache) }
|
|
102
|
+
return { config, session, sessionStore, readCache, brCache, replayMemo, server: _createServer(config, session, sessionStore, readCache, brCache, replayMemo) }
|
|
100
103
|
}
|
|
101
104
|
|
|
102
105
|
// Stream-decompress a zstd body, aborting as soon as the running output size
|
|
@@ -134,9 +137,11 @@ export function sendUpstreamError(res) {
|
|
|
134
137
|
return true
|
|
135
138
|
}
|
|
136
139
|
|
|
137
|
-
function _createServer(config, session, sessionStore, readCache, brCache) {
|
|
140
|
+
function _createServer(config, session, sessionStore, readCache, brCache, replayMemo) {
|
|
138
141
|
const log = createRequestLogger(config)
|
|
139
142
|
const disclosureTotals = { rehydrated: 0, missed: 0, disclosed: 0 }
|
|
143
|
+
const replayTotals = { hits: 0, misses: 0, tokensHeld: 0 }
|
|
144
|
+
const usageTotals = createUsageTotals()
|
|
140
145
|
|
|
141
146
|
function openUpstream(method, upstreamUrl, headers, res) {
|
|
142
147
|
const mod = upstreamUrl.protocol === 'https:' ? https : http
|
|
@@ -149,6 +154,26 @@ function openUpstream(method, upstreamUrl, headers, res) {
|
|
|
149
154
|
}, (upstreamRes) => {
|
|
150
155
|
res.writeHead(upstreamRes.statusCode, upstreamRes.headers)
|
|
151
156
|
upstreamRes.pipe(res)
|
|
157
|
+
// Passive read of provider usage (incl. prompt-cache counters) off the
|
|
158
|
+
// response as it streams past; never alters what the client receives.
|
|
159
|
+
const tap = createUsageTap({
|
|
160
|
+
contentType: upstreamRes.headers['content-type'],
|
|
161
|
+
contentEncoding: upstreamRes.headers['content-encoding'],
|
|
162
|
+
onUsage: (u) => {
|
|
163
|
+
usageTotals.record(u)
|
|
164
|
+
log(`[tamp] usage: in=${u.inputTokens} cache_read=${u.cacheReadTokens} cache_write=${u.cacheCreationTokens} out=${u.outputTokens}`)
|
|
165
|
+
},
|
|
166
|
+
})
|
|
167
|
+
const onData = (chunk) => tap.write(chunk)
|
|
168
|
+
upstreamRes.on('data', onData)
|
|
169
|
+
// 'close' also fires on an aborted upstream, so the tap always gets ended.
|
|
170
|
+
upstreamRes.on('close', () => tap.end())
|
|
171
|
+
// A 'data' listener keeps the source flowing after pipe() unpipes a dead
|
|
172
|
+
// client; detach and pause so an abandoned stream is not drained.
|
|
173
|
+
res.on('close', () => {
|
|
174
|
+
upstreamRes.off('data', onData)
|
|
175
|
+
if (!upstreamRes.readableEnded) upstreamRes.pause()
|
|
176
|
+
})
|
|
152
177
|
upstreamRes.on('error', (err) => {
|
|
153
178
|
log(`[tamp] response stream error: ${err.code || ''} ${err.message}`)
|
|
154
179
|
res.destroy()
|
|
@@ -236,6 +261,8 @@ return http.createServer(async (req, res) => {
|
|
|
236
261
|
blocksCompressed: totals.compressionCount,
|
|
237
262
|
},
|
|
238
263
|
disclosure: disclosureTotals,
|
|
264
|
+
replay: replayMemo ? { ...replayTotals, ...replayMemo.stats() } : null,
|
|
265
|
+
usage: usageTotals.snapshot(),
|
|
239
266
|
})
|
|
240
267
|
res.writeHead(200, { 'Content-Type': 'application/json', 'Content-Length': Buffer.byteLength(body) })
|
|
241
268
|
return res.end(req.method === 'HEAD' ? undefined : body)
|
|
@@ -363,7 +390,7 @@ return http.createServer(async (req, res) => {
|
|
|
363
390
|
const sessionBucket = config.stages?.includes('graph') && sessionKey
|
|
364
391
|
? sessionStore.getBucket(sessionKey)
|
|
365
392
|
: null
|
|
366
|
-
const { body, stats, disclosure } = await compressRequest(parsed, { ...config, sessionBucket, sessionKey, readCache, brCache }, provider)
|
|
393
|
+
const { body, stats, disclosure, replay } = await compressRequest(parsed, { ...config, sessionBucket, sessionKey, readCache, brCache, replayMemo }, provider)
|
|
367
394
|
finalBody = Buffer.from(JSON.stringify(body), 'utf-8')
|
|
368
395
|
// Send uncompressed — simpler and content-length is accurate
|
|
369
396
|
if (decompressed) delete headers['content-encoding']
|
|
@@ -375,6 +402,12 @@ return http.createServer(async (req, res) => {
|
|
|
375
402
|
disclosureTotals.disclosed += disclosure.disclosed || 0
|
|
376
403
|
}
|
|
377
404
|
log(formatRequestLog(stats, session, provider.name, req.url, textBody.length, config.tokenCost, sessionBucket))
|
|
405
|
+
if (replay) {
|
|
406
|
+
replayTotals.hits += replay.hits
|
|
407
|
+
replayTotals.misses += replay.misses
|
|
408
|
+
replayTotals.tokensHeld += replay.tokensHeld
|
|
409
|
+
if (replay.hits > 0) log(`[tamp] replay: ${replay.hits} block${replay.hits !== 1 ? 's' : ''} re-sent as before, ${replay.tokensHeld} tok held`)
|
|
410
|
+
}
|
|
378
411
|
config.onCompress?.(stats, session.getTotals(), { provider: provider.name, url: req.url, bodySize: textBody.length })
|
|
379
412
|
} catch (err) {
|
|
380
413
|
log(`[tamp] passthrough (parse error): ${err.message}`)
|
package/lib/px-render.js
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
// px-render stage: render large DENSE tool_results as PNG image blocks.
|
|
2
2
|
//
|
|
3
3
|
// Borrowed from teamchong/pxpipe (MIT): image input tokens are priced by pixel
|
|
4
|
-
// area (
|
|
4
|
+
// area (⌈w/28⌉×⌈h/28⌉ patches on Anthropic), not text volume, so content that tokenizes
|
|
5
5
|
// badly as text (paths, hashes, hex, grep output at ~1-2 chars/token) packs
|
|
6
6
|
// ~3x more chars per token as a rendered image. Prose tokenizes fine (~3.5+
|
|
7
7
|
// chars/token) and LOSES money imaged — the density gate below excludes it.
|
|
@@ -14,8 +14,10 @@
|
|
|
14
14
|
|
|
15
15
|
import { countTokens } from '@anthropic-ai/tokenizer'
|
|
16
16
|
|
|
17
|
-
// Anthropic vision pricing:
|
|
18
|
-
|
|
17
|
+
// Anthropic vision pricing: one token per 28×28-px patch, ⌈w/28⌉×⌈h/28⌉ per
|
|
18
|
+
// image (pxpipe 0.16 src/core/vision-cost.ts patchTokens; pixels/750 was the
|
|
19
|
+
// older continuous approximation). Fallback when the exported estimator is absent.
|
|
20
|
+
const PATCH_PX = 28
|
|
19
21
|
// Post-render profitability margin: images + factsheet must beat text by 20%.
|
|
20
22
|
const PROFIT_MARGIN = 0.8
|
|
21
23
|
// Density pre-gate: chars-per-text-token at or above this reads as clean
|
|
@@ -49,25 +51,56 @@ export function pxModelAllowed(model, allowCsv) {
|
|
|
49
51
|
// CONST_IDS) so they ride next to the images as verbatim text. Deterministic
|
|
50
52
|
// by construction (fixed pattern order, total sort, no Date/random) so the
|
|
51
53
|
// emitted line is byte-stable and never busts the prompt cache.
|
|
52
|
-
|
|
54
|
+
// Synced with pxpipe 0.16 (LABEL=value, field=value, email, IBAN, currency,
|
|
55
|
+
// dir path, CLI flag, decimal, camelCase, ticket codes). Every pattern must
|
|
56
|
+
// stay linear-time on a whole 200k-char string, not just a 512-char chunk:
|
|
57
|
+
// the email and path patterns start only at the head of a run (lookbehind)
|
|
58
|
+
// instead of retrying at every offset, and the ticket pattern checks its digit
|
|
59
|
+
// after matching instead of with upstream's 120-char lookahead.
|
|
60
|
+
const FS_TICKET = /\b[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+\b/g
|
|
61
|
+
export const FS_PATTERNS = [
|
|
62
|
+
/\b[A-Z][A-Z0-9_]{2,}=[^\s)"'<>]+/g,
|
|
63
|
+
/\b[A-Za-z][A-Za-z0-9_]{2,}=[A-Za-z0-9_.:/+-]{1,64}/g,
|
|
53
64
|
/\bhttps?:\/\/[^\s)"'<>]+/g,
|
|
65
|
+
/(?<![\w.!#$%&'*+/=?^`{|}~-])[\w.!#$%&'*+/=?^`{|}~-]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+\b/g,
|
|
54
66
|
/\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g,
|
|
55
|
-
|
|
67
|
+
/\b[A-Z]{2}\d{2}[A-Z0-9]{8,30}\b/g,
|
|
68
|
+
/(?:[$€£¥]|USD|EUR|GBP|CAD|AUD|CHF|JPY)\d(?:[\d,_]*\d)?(?:\.\d{2})?\b/g,
|
|
69
|
+
/(?<![\w@~+./-])[\w@~+.-]*(?:\/[\w.@+-]+)+\.[A-Za-z]\w{0,8}\b/g,
|
|
70
|
+
/\/[\w.@+-]+(?:\/[\w.@+-]+)+\/?/g,
|
|
56
71
|
/\b(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b/g,
|
|
57
72
|
/\bv?\d+\.\d+(?:\.\d+)?(?:[-+][\w.]+)?\b/g,
|
|
73
|
+
/(?:^|[^\w-])(--?[A-Za-z][\w-]+)/g,
|
|
58
74
|
/\b\d[\d,_]{3,}\b/g,
|
|
75
|
+
/\b\d+\.\d+\b/g,
|
|
59
76
|
/\b[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+\b/g,
|
|
77
|
+
/\b(?:[a-z]+|[A-Z][a-z0-9]+)(?:[A-Z][a-z0-9]*)+\b/g,
|
|
78
|
+
FS_TICKET,
|
|
60
79
|
]
|
|
61
|
-
const FS_MAX_TOKENS =
|
|
80
|
+
const FS_MAX_TOKENS = 96
|
|
62
81
|
const FS_MAX_URLS = 8
|
|
63
82
|
const FS_MAX_SCAN = 262_144
|
|
64
83
|
const FS_URL = /^https?:\/\//
|
|
65
84
|
// Short opaque identifiers (SHAs, UUIDs, numbers, CONST_IDS) outrank long URLs:
|
|
66
85
|
// length is anti-correlated with OCR-risk × consequence.
|
|
67
86
|
const FS_TIER0 = /^(?:(?=[0-9a-f]*\d)[0-9a-f]{7,40}|[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}|[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+|\d[\d,_]*|\d+\.\d+)$/
|
|
87
|
+
// pxpipe 0.16 tier-0 additions. Lowercase field=value stays tier 1: log lines
|
|
88
|
+
// carry one per row and would flood the top tier.
|
|
89
|
+
const FS_TIER0_SHAPES = [
|
|
90
|
+
/^[A-Z][A-Z0-9_]{2,}=\S+$/,
|
|
91
|
+
/^[^@\s]+@[A-Za-z0-9-]+(?:\.[A-Za-z0-9-]+)+$/,
|
|
92
|
+
/^[A-Z]{2}\d{2}[A-Z0-9]{8,30}$/,
|
|
93
|
+
/^(?:[$€£¥]|USD|EUR|GBP|CAD|AUD|CHF|JPY)\d(?:[\d,_]*\d)?(?:\.\d{2})?$/,
|
|
94
|
+
/^--?[A-Za-z][\w-]+$/,
|
|
95
|
+
]
|
|
96
|
+
const FS_TICKET_SHAPE = /^[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+$/
|
|
97
|
+
const FS_CAMEL = /^(?:[a-z]+|[A-Z][a-z0-9]+)(?:[A-Z][a-z0-9]*)+$/
|
|
68
98
|
|
|
69
99
|
function fsTier(tok) {
|
|
70
100
|
if (FS_TIER0.test(tok)) return 0
|
|
101
|
+
if (FS_TIER0_SHAPES.some(re => re.test(tok))) return 0
|
|
102
|
+
if (FS_TICKET_SHAPE.test(tok) && /\d/.test(tok)) return 0
|
|
103
|
+
if (tok.length >= 8 && FS_CAMEL.test(tok)) return 0
|
|
71
104
|
if (FS_URL.test(tok)) return 2
|
|
72
105
|
return 1
|
|
73
106
|
}
|
|
@@ -79,7 +112,8 @@ export function extractFactsheetTokens(text) {
|
|
|
79
112
|
if (chunk.length < 3 || chunk.length > 512) continue
|
|
80
113
|
for (const re of FS_PATTERNS) {
|
|
81
114
|
for (const m of chunk.matchAll(re)) {
|
|
82
|
-
const tok = m[0].replace(/[.,;:!?]+$/, '')
|
|
115
|
+
const tok = (m[1] ?? m[0]).trim().replace(/(?<![.,;:!?])[.,;:!?]+$/, '')
|
|
116
|
+
if (re === FS_TICKET && !/\d/.test(tok)) continue
|
|
83
117
|
if (tok.length >= 3 && tok.length <= 120) seen.add(tok)
|
|
84
118
|
}
|
|
85
119
|
}
|
|
@@ -128,6 +162,16 @@ export async function pxRenderTargets(targets, body, provider, config) {
|
|
|
128
162
|
const px = await loadPxpipe()
|
|
129
163
|
if (!px?.renderTextToImages) return 0
|
|
130
164
|
|
|
165
|
+
// pxpipe 0.16 resolves a Claude id to its render profile (claude-model-profiles.ts):
|
|
166
|
+
// the original Fable 5 keeps bare 5×8 rows, Fable 5.1+ and other Claude ids get
|
|
167
|
+
// +2px row spacing. Only Claude ids are passed: anything else would resolve to a
|
|
168
|
+
// GPT/Gemini geometry, so it keeps the Anthropic defaults.
|
|
169
|
+
const base = body.model.replace(/\[[^\]]*\]/g, '')
|
|
170
|
+
const pxModel = /claude|anthropic/i.test(base) ? base : undefined
|
|
171
|
+
const imageTokensOf = pages => pages.reduce((sum, p) => sum + (pxModel && typeof px.openAIVisionTokens === 'function'
|
|
172
|
+
? px.openAIVisionTokens(pxModel, p.width, p.height)
|
|
173
|
+
: Math.ceil(p.width / PATCH_PX) * Math.ceil(p.height / PATCH_PX)), 0)
|
|
174
|
+
|
|
131
175
|
let rendered = 0
|
|
132
176
|
for (const target of targets) {
|
|
133
177
|
if (target.skip || target.dedup || target.diffed || target.readDiffed || target.graphed || target.disclosed || target.bm25Trimmed || target.compressed) continue
|
|
@@ -140,7 +184,7 @@ export async function pxRenderTargets(targets, body, provider, config) {
|
|
|
140
184
|
|
|
141
185
|
let result
|
|
142
186
|
try {
|
|
143
|
-
result = await px.renderTextToImages(target.text, { reflow: true })
|
|
187
|
+
result = await px.renderTextToImages(target.text, pxModel ? { reflow: true, model: pxModel } : { reflow: true })
|
|
144
188
|
} catch {
|
|
145
189
|
continue
|
|
146
190
|
}
|
|
@@ -149,7 +193,9 @@ export async function pxRenderTargets(targets, body, provider, config) {
|
|
|
149
193
|
if (result.droppedChars > 0) continue
|
|
150
194
|
|
|
151
195
|
const sheet = factsheetText(target.text)
|
|
152
|
-
|
|
196
|
+
// Priced from the real rendered page sizes (Σ w×h = result.pixels), so a
|
|
197
|
+
// spaced profile's taller pages are charged in full.
|
|
198
|
+
const imageTokens = imageTokensOf(result.pages)
|
|
153
199
|
const sheetTokens = sheet ? countTokens(sheet) : 0
|
|
154
200
|
if (imageTokens + sheetTokens >= textTokens * PROFIT_MARGIN) continue
|
|
155
201
|
|
|
@@ -0,0 +1,73 @@
|
|
|
1
|
+
// Replay memo: remembers what tamp emitted for each tool_result so the same
|
|
2
|
+
// raw tool_result, re-sent as history in a later request, goes out
|
|
3
|
+
// byte-identical. Without it cacheSafe compresses (and redacts) a turn only
|
|
4
|
+
// while it is newest, so request N+1 forwards that turn raw: savings vanish,
|
|
5
|
+
// the prompt-cache prefix breaks, and a masked secret leaves unmasked.
|
|
6
|
+
//
|
|
7
|
+
// In-memory LRU bounded by the bytes of stored values. A value is a string,
|
|
8
|
+
// an array of content blocks (px-render), or null ("was sent unchanged").
|
|
9
|
+
// Raw text never enters the memo; callers key on its hash.
|
|
10
|
+
|
|
11
|
+
export const DEFAULT_REPLAY_MAX_BYTES = 64 * 1024 * 1024
|
|
12
|
+
const MAX_ANCHORS = 20_000
|
|
13
|
+
|
|
14
|
+
function valueBytes(value) {
|
|
15
|
+
if (value === null || value === undefined) return 0
|
|
16
|
+
if (typeof value === 'string') return Buffer.byteLength(value, 'utf8')
|
|
17
|
+
return Buffer.byteLength(JSON.stringify(value), 'utf8')
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
export function createReplayMemo({ maxBytes = DEFAULT_REPLAY_MAX_BYTES } = {}) {
|
|
21
|
+
const store = new Map()
|
|
22
|
+
let bytes = 0
|
|
23
|
+
let evictions = 0
|
|
24
|
+
|
|
25
|
+
function drop(key) {
|
|
26
|
+
const entry = store.get(key)
|
|
27
|
+
if (!entry) return
|
|
28
|
+
bytes -= entry.size
|
|
29
|
+
store.delete(key)
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function get(key) {
|
|
33
|
+
const entry = store.get(key)
|
|
34
|
+
if (!entry) return undefined
|
|
35
|
+
// Map insertion order is the LRU order: re-insert to mark as recent.
|
|
36
|
+
store.delete(key)
|
|
37
|
+
store.set(key, entry)
|
|
38
|
+
return entry
|
|
39
|
+
}
|
|
40
|
+
|
|
41
|
+
function put(key, value, meta = {}) {
|
|
42
|
+
drop(key)
|
|
43
|
+
// Count the key too, so a flood of null entries stays bounded.
|
|
44
|
+
const size = valueBytes(value) + key.length
|
|
45
|
+
if (size > maxBytes) return false
|
|
46
|
+
store.set(key, { value, meta, size })
|
|
47
|
+
bytes += size
|
|
48
|
+
while (bytes > maxBytes) {
|
|
49
|
+
drop(store.keys().next().value)
|
|
50
|
+
evictions += 1
|
|
51
|
+
}
|
|
52
|
+
return true
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
function stats() {
|
|
56
|
+
return { entries: store.size, bytes, maxBytes, evictions }
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
// read-diff anchors: raw hash of the read that each cached file copy came
|
|
60
|
+
// from, so a replayed diff can check its base is still in the request.
|
|
61
|
+
const anchors = new Map()
|
|
62
|
+
function setAnchor(key, hash) {
|
|
63
|
+
anchors.delete(key)
|
|
64
|
+
if (!hash) return
|
|
65
|
+
anchors.set(key, hash)
|
|
66
|
+
if (anchors.size > MAX_ANCHORS) anchors.delete(anchors.keys().next().value)
|
|
67
|
+
}
|
|
68
|
+
function getAnchor(key) {
|
|
69
|
+
return anchors.get(key)
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
return { get, put, stats, setAnchor, getAnchor }
|
|
73
|
+
}
|
package/lib/usage-tap.js
ADDED
|
@@ -0,0 +1,360 @@
|
|
|
1
|
+
// Passive token-usage telemetry for upstream responses.
|
|
2
|
+
//
|
|
3
|
+
// The proxy pipes upstream bytes to the client untouched. This tap sees the
|
|
4
|
+
// same chunks on a side path and pulls the provider's usage counters out of
|
|
5
|
+
// them, prompt-cache counters included, so a cache-breaking rewrite shows up as
|
|
6
|
+
// a falling hit rate instead of a silent bill. It never writes to, holds up, or
|
|
7
|
+
// alters what the client gets: write() copies only the bytes it must keep, and
|
|
8
|
+
// any internal failure turns the tap into a no-op.
|
|
9
|
+
//
|
|
10
|
+
// The format comes from the payload shape, not from a provider argument:
|
|
11
|
+
// Anthropic Messages message_start.message.usage + message_delta.usage, or top-level usage
|
|
12
|
+
// OpenAI Chat usage.prompt_tokens / prompt_tokens_details.cached_tokens
|
|
13
|
+
// OpenAI Responses response.*.response.usage, or top-level usage with input_tokens_details
|
|
14
|
+
// Gemini usageMetadata (last one wins)
|
|
15
|
+
//
|
|
16
|
+
// Normalized inputTokens means UNCACHED input: inputTokens + cacheReadTokens +
|
|
17
|
+
// cacheCreationTokens is the full prompt. Anthropic already reports it that
|
|
18
|
+
// way; OpenAI and Gemini include cached tokens in the prompt count, so we
|
|
19
|
+
// subtract.
|
|
20
|
+
|
|
21
|
+
import zlib from 'node:zlib'
|
|
22
|
+
|
|
23
|
+
const DEFAULT_MAX_BUFFER = 1024 * 1024
|
|
24
|
+
|
|
25
|
+
const num = v => (typeof v === 'number' && Number.isFinite(v) && v > 0 ? v : 0)
|
|
26
|
+
const isObj = v => v !== null && typeof v === 'object' && !Array.isArray(v)
|
|
27
|
+
|
|
28
|
+
function normalized(provider, inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens) {
|
|
29
|
+
return { provider, inputTokens, outputTokens, cacheReadTokens, cacheCreationTokens }
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
function fromOpenAIChat(u) {
|
|
33
|
+
const d = isObj(u.prompt_tokens_details) ? u.prompt_tokens_details : {}
|
|
34
|
+
const read = num(d.cached_tokens)
|
|
35
|
+
const write = num(d.cache_write_tokens)
|
|
36
|
+
return normalized('openai', Math.max(0, num(u.prompt_tokens) - read - write), num(u.completion_tokens), read, write)
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function fromOpenAIResponses(u) {
|
|
40
|
+
const d = isObj(u.input_tokens_details) ? u.input_tokens_details : {}
|
|
41
|
+
const read = num(d.cached_tokens)
|
|
42
|
+
const write = num(d.cache_write_tokens)
|
|
43
|
+
return normalized('openai-responses', Math.max(0, num(u.input_tokens) - read - write), num(u.output_tokens), read, write)
|
|
44
|
+
}
|
|
45
|
+
|
|
46
|
+
function fromGemini(m) {
|
|
47
|
+
const read = num(m.cachedContentTokenCount)
|
|
48
|
+
// candidatesTokenCount excludes thinking; the other providers bill reasoning
|
|
49
|
+
// as output, so fold thoughtsTokenCount in to keep outputTokens comparable.
|
|
50
|
+
const output = num(m.candidatesTokenCount) + num(m.thoughtsTokenCount)
|
|
51
|
+
return normalized('gemini', Math.max(0, num(m.promptTokenCount) - read), output, read, 0)
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
function fromAnthropic(a) {
|
|
55
|
+
return normalized('anthropic', num(a.input_tokens), num(a.output_tokens),
|
|
56
|
+
num(a.cache_read_input_tokens), num(a.cache_creation_input_tokens))
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const ANTHROPIC_FIELDS = ['input_tokens', 'output_tokens', 'cache_read_input_tokens', 'cache_creation_input_tokens']
|
|
60
|
+
|
|
61
|
+
// Collects usage across a response. Later values win; Anthropic fields merge
|
|
62
|
+
// because message_start and message_delta each carry part of the picture.
|
|
63
|
+
function createExtractor() {
|
|
64
|
+
let result = null
|
|
65
|
+
let anthropic = null
|
|
66
|
+
|
|
67
|
+
function mergeAnthropic(u) {
|
|
68
|
+
if (!anthropic) anthropic = {}
|
|
69
|
+
for (const k of ANTHROPIC_FIELDS) if (u[k] != null) anthropic[k] = u[k]
|
|
70
|
+
result = fromAnthropic(anthropic)
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
function take(obj) {
|
|
74
|
+
if (Array.isArray(obj)) {
|
|
75
|
+
for (const item of obj) take(item)
|
|
76
|
+
return
|
|
77
|
+
}
|
|
78
|
+
if (!isObj(obj)) return
|
|
79
|
+
const type = typeof obj.type === 'string' ? obj.type : ''
|
|
80
|
+
|
|
81
|
+
if (type === 'message_start' && isObj(obj.message) && isObj(obj.message.usage)) return mergeAnthropic(obj.message.usage)
|
|
82
|
+
if (type === 'message_delta' && isObj(obj.usage)) return mergeAnthropic(obj.usage)
|
|
83
|
+
if (type.startsWith('response.') && isObj(obj.response) && isObj(obj.response.usage)) {
|
|
84
|
+
result = fromOpenAIResponses(obj.response.usage)
|
|
85
|
+
return
|
|
86
|
+
}
|
|
87
|
+
if (isObj(obj.usageMetadata)) {
|
|
88
|
+
result = fromGemini(obj.usageMetadata)
|
|
89
|
+
return
|
|
90
|
+
}
|
|
91
|
+
const u = obj.usage
|
|
92
|
+
if (!isObj(u)) return
|
|
93
|
+
if (u.prompt_tokens != null) {
|
|
94
|
+
result = fromOpenAIChat(u)
|
|
95
|
+
} else if (type === 'message' || u.cache_read_input_tokens != null || u.cache_creation_input_tokens != null) {
|
|
96
|
+
anthropic = null
|
|
97
|
+
mergeAnthropic(u)
|
|
98
|
+
} else if (obj.object === 'response' || isObj(u.input_tokens_details)) {
|
|
99
|
+
result = fromOpenAIResponses(u)
|
|
100
|
+
}
|
|
101
|
+
}
|
|
102
|
+
|
|
103
|
+
return { take, result: () => result }
|
|
104
|
+
}
|
|
105
|
+
|
|
106
|
+
// Incremental SSE reader. Splits on LF bytes (a UTF-8 multi-byte sequence never
|
|
107
|
+
// contains 0x0A, so decoding whole lines is safe), strips a trailing CR, joins
|
|
108
|
+
// data: lines per event, and hands each event's payload to onData. At most one
|
|
109
|
+
// partial line and one event's data stay in memory, each capped at max bytes;
|
|
110
|
+
// anything over the cap is dropped and parsing resumes at the next line/event.
|
|
111
|
+
function createSseReader(max, onData) {
|
|
112
|
+
let parts = []
|
|
113
|
+
let partLen = 0
|
|
114
|
+
let skipping = false
|
|
115
|
+
let data = []
|
|
116
|
+
let dataLen = 0
|
|
117
|
+
let dropEvent = false
|
|
118
|
+
|
|
119
|
+
const resetPartial = () => { parts = []; partLen = 0 }
|
|
120
|
+
|
|
121
|
+
function dispatch() {
|
|
122
|
+
if (!dropEvent && data.length) onData(data.join('\n'))
|
|
123
|
+
data = []
|
|
124
|
+
dataLen = 0
|
|
125
|
+
dropEvent = false
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function line(str) {
|
|
129
|
+
if (str.endsWith('\r')) str = str.slice(0, -1)
|
|
130
|
+
if (str === '') return dispatch()
|
|
131
|
+
if (!str.startsWith('data:')) return
|
|
132
|
+
if (dropEvent) return
|
|
133
|
+
let v = str.slice(5)
|
|
134
|
+
if (v.startsWith(' ')) v = v.slice(1)
|
|
135
|
+
dataLen += v.length + 1
|
|
136
|
+
if (dataLen > max) {
|
|
137
|
+
data = []
|
|
138
|
+
dropEvent = true
|
|
139
|
+
return
|
|
140
|
+
}
|
|
141
|
+
data.push(v)
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
function endLine(seg) {
|
|
145
|
+
if (skipping) {
|
|
146
|
+
skipping = false
|
|
147
|
+
return
|
|
148
|
+
}
|
|
149
|
+
const total = partLen + seg.length
|
|
150
|
+
if (total > max) {
|
|
151
|
+
resetPartial()
|
|
152
|
+
return
|
|
153
|
+
}
|
|
154
|
+
if (partLen === 0) return line(seg.toString('utf8'))
|
|
155
|
+
parts.push(seg)
|
|
156
|
+
const full = Buffer.concat(parts, total)
|
|
157
|
+
resetPartial()
|
|
158
|
+
line(full.toString('utf8'))
|
|
159
|
+
}
|
|
160
|
+
|
|
161
|
+
function addPartial(seg) {
|
|
162
|
+
if (skipping || seg.length === 0) return
|
|
163
|
+
if (partLen + seg.length > max) {
|
|
164
|
+
resetPartial()
|
|
165
|
+
skipping = true
|
|
166
|
+
return
|
|
167
|
+
}
|
|
168
|
+
parts.push(Buffer.from(seg))
|
|
169
|
+
partLen += seg.length
|
|
170
|
+
}
|
|
171
|
+
|
|
172
|
+
return {
|
|
173
|
+
push(buf) {
|
|
174
|
+
let pos = 0
|
|
175
|
+
while (pos < buf.length) {
|
|
176
|
+
const nl = buf.indexOf(10, pos)
|
|
177
|
+
if (nl === -1) {
|
|
178
|
+
addPartial(buf.subarray(pos))
|
|
179
|
+
return
|
|
180
|
+
}
|
|
181
|
+
endLine(buf.subarray(pos, nl))
|
|
182
|
+
pos = nl + 1
|
|
183
|
+
}
|
|
184
|
+
},
|
|
185
|
+
end() {
|
|
186
|
+
if (!skipping && partLen) {
|
|
187
|
+
const full = Buffer.concat(parts, partLen)
|
|
188
|
+
resetPartial()
|
|
189
|
+
line(full.toString('utf8'))
|
|
190
|
+
}
|
|
191
|
+
dispatch()
|
|
192
|
+
},
|
|
193
|
+
}
|
|
194
|
+
}
|
|
195
|
+
|
|
196
|
+
// Bounded whole-body collector for non-streaming JSON. Over the cap it drops
|
|
197
|
+
// everything and reports overflow so the caller can give up.
|
|
198
|
+
function createJsonReader(max, onData) {
|
|
199
|
+
let parts = []
|
|
200
|
+
let len = 0
|
|
201
|
+
let over = false
|
|
202
|
+
return {
|
|
203
|
+
push(buf) {
|
|
204
|
+
if (over) return
|
|
205
|
+
if (len + buf.length > max) {
|
|
206
|
+
over = true
|
|
207
|
+
parts = []
|
|
208
|
+
return
|
|
209
|
+
}
|
|
210
|
+
parts.push(Buffer.from(buf))
|
|
211
|
+
len += buf.length
|
|
212
|
+
},
|
|
213
|
+
end() {
|
|
214
|
+
if (over || len === 0) return
|
|
215
|
+
const text = Buffer.concat(parts, len).toString('utf8')
|
|
216
|
+
parts = []
|
|
217
|
+
onData(text)
|
|
218
|
+
},
|
|
219
|
+
overflowed: () => over,
|
|
220
|
+
}
|
|
221
|
+
}
|
|
222
|
+
|
|
223
|
+
function pickDecoder(encoding, firstChunk) {
|
|
224
|
+
if (encoding === 'gzip' || encoding === 'x-gzip') return zlib.createGunzip()
|
|
225
|
+
if (encoding === 'br') return zlib.createBrotliDecompress()
|
|
226
|
+
if (encoding === 'deflate') {
|
|
227
|
+
// HTTP "deflate" is meant to be zlib-wrapped, but some servers send raw
|
|
228
|
+
// deflate. A zlib header has CM=8 and a header checksum divisible by 31.
|
|
229
|
+
const b0 = firstChunk[0]
|
|
230
|
+
const b1 = firstChunk.length > 1 ? firstChunk[1] : 0
|
|
231
|
+
const wrapped = (b0 & 0x0f) === 8 && ((b0 << 8) | b1) % 31 === 0
|
|
232
|
+
return wrapped ? zlib.createInflate() : zlib.createInflateRaw()
|
|
233
|
+
}
|
|
234
|
+
return null
|
|
235
|
+
}
|
|
236
|
+
|
|
237
|
+
function parseEncoding(value) {
|
|
238
|
+
if (value == null) return 'identity'
|
|
239
|
+
if (typeof value !== 'string') return null
|
|
240
|
+
const parts = value.split(',').map(s => s.trim().toLowerCase()).filter(Boolean)
|
|
241
|
+
if (parts.length === 0) return 'identity'
|
|
242
|
+
if (parts.length > 1) return null // stacked encodings: not worth the risk
|
|
243
|
+
return parts[0]
|
|
244
|
+
}
|
|
245
|
+
|
|
246
|
+
const NOOP = Object.freeze({ write() {}, end() {} })
|
|
247
|
+
|
|
248
|
+
export function createUsageTap({ contentType, contentEncoding, onUsage, maxBufferBytes } = {}) {
|
|
249
|
+
try {
|
|
250
|
+
if (typeof onUsage !== 'function' || typeof contentType !== 'string') return NOOP
|
|
251
|
+
const ct = contentType.toLowerCase()
|
|
252
|
+
const isSse = ct.includes('text/event-stream')
|
|
253
|
+
const isJson = !isSse && ct.includes('json')
|
|
254
|
+
if (!isSse && !isJson) return NOOP
|
|
255
|
+
|
|
256
|
+
const encoding = parseEncoding(contentEncoding)
|
|
257
|
+
if (encoding !== 'identity' && encoding !== 'gzip' && encoding !== 'x-gzip' && encoding !== 'br' && encoding !== 'deflate') return NOOP
|
|
258
|
+
|
|
259
|
+
const max = Number.isFinite(maxBufferBytes) && maxBufferBytes > 0 ? maxBufferBytes : DEFAULT_MAX_BUFFER
|
|
260
|
+
const extractor = createExtractor()
|
|
261
|
+
let dead = false
|
|
262
|
+
let finished = false
|
|
263
|
+
let decoder = null
|
|
264
|
+
|
|
265
|
+
const kill = () => {
|
|
266
|
+
dead = true
|
|
267
|
+
if (decoder) {
|
|
268
|
+
try { decoder.destroy() } catch {}
|
|
269
|
+
decoder = null
|
|
270
|
+
}
|
|
271
|
+
}
|
|
272
|
+
|
|
273
|
+
const parse = (text) => {
|
|
274
|
+
// Cheap filter: content deltas outnumber usage events by orders of magnitude.
|
|
275
|
+
if (!text.includes('"usage')) return
|
|
276
|
+
let obj
|
|
277
|
+
try { obj = JSON.parse(text) } catch { return }
|
|
278
|
+
extractor.take(obj)
|
|
279
|
+
}
|
|
280
|
+
|
|
281
|
+
const reader = isSse ? createSseReader(max, parse) : createJsonReader(max, parse)
|
|
282
|
+
|
|
283
|
+
const finish = () => {
|
|
284
|
+
if (dead || finished) return
|
|
285
|
+
finished = true
|
|
286
|
+
try {
|
|
287
|
+
reader.end()
|
|
288
|
+
const usage = extractor.result()
|
|
289
|
+
if (usage) onUsage(usage)
|
|
290
|
+
} catch {}
|
|
291
|
+
kill()
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
const feed = (buf) => {
|
|
295
|
+
if (dead) return
|
|
296
|
+
reader.push(buf)
|
|
297
|
+
if (reader.overflowed && reader.overflowed()) kill()
|
|
298
|
+
}
|
|
299
|
+
|
|
300
|
+
let decoderPicked = encoding === 'identity'
|
|
301
|
+
|
|
302
|
+
return {
|
|
303
|
+
write(chunk) {
|
|
304
|
+
if (dead || finished) return
|
|
305
|
+
try {
|
|
306
|
+
if (!Buffer.isBuffer(chunk) && !(chunk instanceof Uint8Array)) return
|
|
307
|
+
if (chunk.length === 0) return
|
|
308
|
+
if (!decoderPicked) {
|
|
309
|
+
decoderPicked = true
|
|
310
|
+
decoder = pickDecoder(encoding, chunk)
|
|
311
|
+
if (!decoder) return kill()
|
|
312
|
+
decoder.on('data', (out) => { try { feed(out) } catch { kill() } })
|
|
313
|
+
decoder.on('error', kill)
|
|
314
|
+
decoder.on('end', finish)
|
|
315
|
+
}
|
|
316
|
+
if (decoder) {
|
|
317
|
+
// The decoder works off-thread; if compressed input piles up faster
|
|
318
|
+
// than it drains, stop rather than buffer the whole response.
|
|
319
|
+
if (decoder.writableLength + chunk.length > max) return kill()
|
|
320
|
+
decoder.write(Buffer.from(chunk))
|
|
321
|
+
} else {
|
|
322
|
+
feed(chunk)
|
|
323
|
+
}
|
|
324
|
+
} catch {
|
|
325
|
+
kill()
|
|
326
|
+
}
|
|
327
|
+
},
|
|
328
|
+
end() {
|
|
329
|
+
if (dead || finished) return
|
|
330
|
+
try {
|
|
331
|
+
if (decoder) decoder.end()
|
|
332
|
+
else if (decoderPicked) finish()
|
|
333
|
+
else kill() // encoded response with no body
|
|
334
|
+
} catch {
|
|
335
|
+
kill()
|
|
336
|
+
}
|
|
337
|
+
},
|
|
338
|
+
}
|
|
339
|
+
} catch {
|
|
340
|
+
return NOOP
|
|
341
|
+
}
|
|
342
|
+
}
|
|
343
|
+
|
|
344
|
+
export function createUsageTotals() {
|
|
345
|
+
const t = { requests: 0, inputTokens: 0, outputTokens: 0, cacheReadTokens: 0, cacheCreationTokens: 0 }
|
|
346
|
+
return {
|
|
347
|
+
record(usage) {
|
|
348
|
+
if (!isObj(usage)) return
|
|
349
|
+
t.requests++
|
|
350
|
+
t.inputTokens += num(usage.inputTokens)
|
|
351
|
+
t.outputTokens += num(usage.outputTokens)
|
|
352
|
+
t.cacheReadTokens += num(usage.cacheReadTokens)
|
|
353
|
+
t.cacheCreationTokens += num(usage.cacheCreationTokens)
|
|
354
|
+
},
|
|
355
|
+
snapshot() {
|
|
356
|
+
const denom = t.inputTokens + t.cacheReadTokens + t.cacheCreationTokens
|
|
357
|
+
return { ...t, cacheHitRate: denom > 0 ? t.cacheReadTokens / denom : null }
|
|
358
|
+
},
|
|
359
|
+
}
|
|
360
|
+
}
|
package/package.json
CHANGED
|
@@ -14,7 +14,7 @@
|
|
|
14
14
|
"sidecar/server.py",
|
|
15
15
|
"sidecar/requirements.txt"
|
|
16
16
|
],
|
|
17
|
-
"version": "0.
|
|
17
|
+
"version": "0.9.0",
|
|
18
18
|
"description": "Token compression proxy for coding agents. Works with Claude Code, Aider, Cursor, Cline, Windsurf. 60-70% combined input+output savings with Caveman-integrated presets and per-command rewriters.",
|
|
19
19
|
"type": "module",
|
|
20
20
|
"main": "index.js",
|
|
@@ -56,6 +56,6 @@
|
|
|
56
56
|
"react": "^18.2.0"
|
|
57
57
|
},
|
|
58
58
|
"optionalDependencies": {
|
|
59
|
-
"pxpipe-proxy": "^0.
|
|
59
|
+
"pxpipe-proxy": "^0.16.0"
|
|
60
60
|
}
|
|
61
61
|
}
|
package/providers.js
CHANGED
|
@@ -8,6 +8,13 @@ function applyTargets(body, targets) {
|
|
|
8
8
|
}
|
|
9
9
|
}
|
|
10
10
|
|
|
11
|
+
// Stable identity for a target so the replay memo can find it again in a
|
|
12
|
+
// later request. Sub-blocks of array content get `#j`. No id, no replay.
|
|
13
|
+
function targetId(id, j) {
|
|
14
|
+
if (typeof id !== 'string' || !id) return undefined
|
|
15
|
+
return j === undefined ? id : `${id}#${j}`
|
|
16
|
+
}
|
|
17
|
+
|
|
11
18
|
function findLatestEligibleGroup(items, extractTargets) {
|
|
12
19
|
let fallback = null
|
|
13
20
|
for (let i = items.length - 1; i >= 0; i--) {
|
|
@@ -35,12 +42,12 @@ function extractAnthropicMessageTargets(msg, mi) {
|
|
|
35
42
|
}
|
|
36
43
|
|
|
37
44
|
if (typeof block.content === 'string') {
|
|
38
|
-
targets.push({ path: ['messages', mi, 'content', i, 'content'], text: block.content, index: i })
|
|
45
|
+
targets.push({ path: ['messages', mi, 'content', i, 'content'], text: block.content, index: i, id: targetId(block.tool_use_id) })
|
|
39
46
|
} else if (Array.isArray(block.content)) {
|
|
40
47
|
for (let j = 0; j < block.content.length; j++) {
|
|
41
48
|
const sub = block.content[j]
|
|
42
49
|
if (sub && sub.type === 'text') {
|
|
43
|
-
targets.push({ path: ['messages', mi, 'content', i, 'content', j, 'text'], text: sub.text, index: i })
|
|
50
|
+
targets.push({ path: ['messages', mi, 'content', i, 'content', j, 'text'], text: sub.text, index: i, id: targetId(block.tool_use_id, j) })
|
|
44
51
|
}
|
|
45
52
|
}
|
|
46
53
|
}
|
|
@@ -260,14 +267,14 @@ function extractOpenAIChatTargets(msg, i) {
|
|
|
260
267
|
if (!block || typeof block !== 'object') continue
|
|
261
268
|
if (OPENAI_COMPAT_SKIP_BLOCK_TYPES.has(block.type)) continue
|
|
262
269
|
if (block.type === 'text' && typeof block.text === 'string') {
|
|
263
|
-
targets.push({ path: ['messages', i, 'content', j, 'text'], text: block.text, index: i })
|
|
270
|
+
targets.push({ path: ['messages', i, 'content', j, 'text'], text: block.text, index: i, id: targetId(msg.tool_call_id, j) })
|
|
264
271
|
}
|
|
265
272
|
}
|
|
266
273
|
return targets
|
|
267
274
|
}
|
|
268
275
|
return []
|
|
269
276
|
}
|
|
270
|
-
return [{ path: ['messages', i, 'content'], text: msg.content, index: i }]
|
|
277
|
+
return [{ path: ['messages', i, 'content'], text: msg.content, index: i, id: targetId(msg.tool_call_id) }]
|
|
271
278
|
}
|
|
272
279
|
|
|
273
280
|
const openai = {
|
|
@@ -403,6 +410,7 @@ function extractGeminiContentTargets(content, ci) {
|
|
|
403
410
|
text,
|
|
404
411
|
index: pi,
|
|
405
412
|
wasObject: typeof resp !== 'string',
|
|
413
|
+
id: targetId(part.functionResponse.id),
|
|
406
414
|
})
|
|
407
415
|
}
|
|
408
416
|
|
|
@@ -484,13 +492,13 @@ function isResponsesToolOutput(type) {
|
|
|
484
492
|
function extractOpenAIResponsesTargets(item, i) {
|
|
485
493
|
if (!isResponsesToolOutput(item?.type)) return []
|
|
486
494
|
if (typeof item.output === 'string') {
|
|
487
|
-
return [{ path: ['input', i, 'output'], text: item.output, index: i }]
|
|
495
|
+
return [{ path: ['input', i, 'output'], text: item.output, index: i, id: targetId(item.call_id) }]
|
|
488
496
|
}
|
|
489
497
|
if (Array.isArray(item.output)) {
|
|
490
498
|
const targets = []
|
|
491
499
|
for (const [j, part] of item.output.entries()) {
|
|
492
500
|
if (part?.type === 'input_text' && typeof part.text === 'string') {
|
|
493
|
-
targets.push({ path: ['input', i, 'output', j, 'text'], text: part.text, index: i })
|
|
501
|
+
targets.push({ path: ['input', i, 'output', j, 'text'], text: part.text, index: i, id: targetId(item.call_id, j) })
|
|
494
502
|
}
|
|
495
503
|
}
|
|
496
504
|
return targets
|