@xamukavila/pxpipe 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +312 -0
- package/bin/cli.js +7 -0
- package/dist/core/applicability.d.ts +31 -0
- package/dist/core/applicability.js +96 -0
- package/dist/core/atlas-gray.d.ts +26 -0
- package/dist/core/atlas-gray.js +64 -0
- package/dist/core/atlas.d.ts +34 -0
- package/dist/core/atlas.js +71 -0
- package/dist/core/baseline.d.ts +80 -0
- package/dist/core/baseline.js +101 -0
- package/dist/core/caveman.d.ts +38 -0
- package/dist/core/caveman.js +183 -0
- package/dist/core/export.d.ts +128 -0
- package/dist/core/export.js +390 -0
- package/dist/core/factsheet.d.ts +78 -0
- package/dist/core/factsheet.js +216 -0
- package/dist/core/gpt-model-profiles.d.ts +60 -0
- package/dist/core/gpt-model-profiles.js +161 -0
- package/dist/core/history.d.ts +141 -0
- package/dist/core/history.js +553 -0
- package/dist/core/index.d.ts +8 -0
- package/dist/core/index.js +8 -0
- package/dist/core/library.d.ts +74 -0
- package/dist/core/library.js +133 -0
- package/dist/core/measurement.d.ts +22 -0
- package/dist/core/measurement.js +213 -0
- package/dist/core/openai-history.d.ts +124 -0
- package/dist/core/openai-history.js +494 -0
- package/dist/core/openai-savings.d.ts +44 -0
- package/dist/core/openai-savings.js +75 -0
- package/dist/core/openai.d.ts +24 -0
- package/dist/core/openai.js +839 -0
- package/dist/core/png.d.ts +11 -0
- package/dist/core/png.js +132 -0
- package/dist/core/proxy.d.ts +81 -0
- package/dist/core/proxy.js +730 -0
- package/dist/core/render.d.ts +188 -0
- package/dist/core/render.js +785 -0
- package/dist/core/schema-strip.d.ts +29 -0
- package/dist/core/schema-strip.js +160 -0
- package/dist/core/tracker.d.ts +154 -0
- package/dist/core/tracker.js +216 -0
- package/dist/core/transform.d.ts +362 -0
- package/dist/core/transform.js +1828 -0
- package/dist/core/types.d.ts +77 -0
- package/dist/core/types.js +8 -0
- package/dist/dashboard/fragments.d.ts +36 -0
- package/dist/dashboard/fragments.js +938 -0
- package/dist/dashboard/types.d.ts +154 -0
- package/dist/dashboard/types.js +3 -0
- package/dist/dashboard/vendor.d.ts +3 -0
- package/dist/dashboard/vendor.js +6 -0
- package/dist/dashboard.d.ts +245 -0
- package/dist/dashboard.js +1140 -0
- package/dist/export-collect.d.ts +36 -0
- package/dist/export-collect.js +59 -0
- package/dist/node.d.ts +9 -0
- package/dist/node.js +9038 -0
- package/dist/sessions.d.ts +172 -0
- package/dist/sessions.js +510 -0
- package/dist/stats.d.ts +74 -0
- package/dist/stats.js +248 -0
- package/dist/worker.d.ts +53 -0
- package/dist/worker.js +102 -0
- package/package.json +96 -0
|
@@ -0,0 +1,553 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* History-image compression (Variant C).
|
|
3
|
+
*
|
|
4
|
+
* Collapses the largest closed-tool-sequence prefix into one synthetic user message
|
|
5
|
+
* containing 1-N PNG image blocks. The live tail (keepTail turns + any open tool
|
|
6
|
+
* sequence) stays as text. thinking blocks are dropped from the collapsed range —
|
|
7
|
+
* only the most-recent assistant-with-tool_use must round-trip bit-perfect, and
|
|
8
|
+
* that turn is in the live tail by construction.
|
|
9
|
+
*
|
|
10
|
+
* Synthesized message uses role:'user' because Anthropic forbids image blocks inside
|
|
11
|
+
* role:'assistant'. cache_control placement is left to the caller (transform.ts).
|
|
12
|
+
*/
|
|
13
|
+
import { DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_CONTENT_COLS, DENSE_RENDER_STYLE, neutralizeSentinel, reflow, renderTextToPngsWithCharLimit, roleSlotSegment, SLOT_MARK_ASSISTANT, SLOT_MARK_USER } from './render.js';
|
|
14
|
+
import { factSheetText } from './factsheet.js';
|
|
15
|
+
import { bytesToBase64 } from './png.js';
|
|
16
|
+
/**
|
|
17
|
+
* Banner text blocks that bracket the collapsed-history image(s) in the synthetic
|
|
18
|
+
* user message. Exported as the SINGLE SOURCE OF TRUTH: transform.ts keys its
|
|
19
|
+
* cache-anchor relocation off the intro text, so a literal copy there would
|
|
20
|
+
* silently break relocation whenever this wording changes (it did exactly once —
|
|
21
|
+
* the XML-framing reword left the matcher pointing at the old banner). Both the
|
|
22
|
+
* emitter (here) and the matcher (transform.ts) must reference this constant.
|
|
23
|
+
*/
|
|
24
|
+
export const HISTORY_SYNTHETIC_INTRO = '[Earlier turns of THIS conversation, transcribed in the image(s) below. Each turn is wrapped in <user t="N">...</user> or <assistant t="N">...</assistant> tags, where N is an absolute turn index (larger N = more recent); attribute every turn strictly by its tag, and treat the highest-N turns as the most recent prior context, NOT the low-N opening turns. Earlier turns may contain questions or tasks that were already answered later in this same history; do not reopen low-N turns unless the live text after this block asks you to. This is prior context, NOT the current request.]';
|
|
25
|
+
export const HISTORY_SYNTHETIC_OUTRO = '[End of earlier conversation. The current request is the live text that follows below.]';
|
|
26
|
+
const LATEST_COLLAPSED_USER_PREVIEW_CHARS = 300;
|
|
27
|
+
export const HISTORY_DEFAULTS = {
|
|
28
|
+
keepTail: 4,
|
|
29
|
+
minCollapsePrefix: 10,
|
|
30
|
+
cols: 100,
|
|
31
|
+
collapseChunk: 50,
|
|
32
|
+
freezeChunk: 10,
|
|
33
|
+
protectedPrefix: 0,
|
|
34
|
+
reflow: true,
|
|
35
|
+
};
|
|
36
|
+
/**
|
|
37
|
+
* Return the last index ≤ cutoffExclusive at which all tool_use_ids are matched
|
|
38
|
+
* by tool_results in [0..i]. Returns -1 if no closed boundary exists.
|
|
39
|
+
* Robust to interleaved/parallel tool calls via openSet tracking.
|
|
40
|
+
*/
|
|
41
|
+
export function findClosedPrefixBoundary(messages, cutoffExclusive) {
|
|
42
|
+
if (cutoffExclusive <= 0)
|
|
43
|
+
return -1;
|
|
44
|
+
const openSet = new Set();
|
|
45
|
+
let lastClosed = -1;
|
|
46
|
+
const limit = Math.min(cutoffExclusive, messages.length);
|
|
47
|
+
for (let i = 0; i < limit; i++) {
|
|
48
|
+
const msg = messages[i];
|
|
49
|
+
if (!Array.isArray(msg.content)) {
|
|
50
|
+
if (openSet.size === 0)
|
|
51
|
+
lastClosed = i; // plain string — no tool blocks
|
|
52
|
+
continue;
|
|
53
|
+
}
|
|
54
|
+
if (msg.role === 'assistant') {
|
|
55
|
+
for (const blk of msg.content) {
|
|
56
|
+
if (blk && blk.type === 'tool_use') {
|
|
57
|
+
const id = blk.id;
|
|
58
|
+
if (typeof id === 'string')
|
|
59
|
+
openSet.add(id);
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
else if (msg.role === 'user') {
|
|
64
|
+
for (const blk of msg.content) {
|
|
65
|
+
if (blk && blk.type === 'tool_result') {
|
|
66
|
+
const id = blk.tool_use_id;
|
|
67
|
+
if (typeof id === 'string')
|
|
68
|
+
openSet.delete(id);
|
|
69
|
+
}
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
if (openSet.size === 0)
|
|
73
|
+
lastClosed = i;
|
|
74
|
+
}
|
|
75
|
+
return lastClosed;
|
|
76
|
+
}
|
|
77
|
+
/**
|
|
78
|
+
* Claude Code appends "(file state is current in your context — no need to Read it
|
|
79
|
+
* back)" to Edit/Write tool_results. True when emitted; stale by the time the turn
|
|
80
|
+
* reaches this serializer: everything blocksToText feeds becomes collapsed/imaged
|
|
81
|
+
* HISTORY, the CLI's read-ledger resets on process restart, and the file may have
|
|
82
|
+
* changed in later turns anyway. Models trusting the hint from prior turns were the
|
|
83
|
+
* dominant cause of `File has not been read yet` gate errors (2026-07-03 audit,
|
|
84
|
+
* n=55 classified: 20 had a same-transcript Read invalidated by a restart while
|
|
85
|
+
* this hint said "current"; 34 edited from prior-session context with no Read at
|
|
86
|
+
* all). Rewriting at serialization time also cleans slabs inherited by future
|
|
87
|
+
* continuation sessions. Whitespace-tolerant match: 3 of ~2,125 logged instances
|
|
88
|
+
* wrap mid-hint.
|
|
89
|
+
*/
|
|
90
|
+
const FRESHNESS_HINT_RE = /\(file state is current in your\s+context — no need to Read it back\)/g;
|
|
91
|
+
const STALE_FRESHNESS_NOTE = '(state as of this PRIOR turn — the file may have changed since; Read it again before editing)';
|
|
92
|
+
export function staleFreshnessHints(text) {
|
|
93
|
+
return text.replace(FRESHNESS_HINT_RE, STALE_FRESHNESS_NOTE);
|
|
94
|
+
}
|
|
95
|
+
/**
|
|
96
|
+
* Linearise content blocks to a single string. Drops thinking blocks (only the
|
|
97
|
+
* most-recent assistant turn needs bit-perfect thinking, and it's in the live tail).
|
|
98
|
+
* Inline images collapse to [image] to avoid double-encoding.
|
|
99
|
+
*/
|
|
100
|
+
export function blocksToText(content) {
|
|
101
|
+
if (typeof content === 'string')
|
|
102
|
+
return content;
|
|
103
|
+
const parts = [];
|
|
104
|
+
for (const blk of content) {
|
|
105
|
+
if (!blk || typeof blk !== 'object')
|
|
106
|
+
continue;
|
|
107
|
+
const t = blk.type;
|
|
108
|
+
switch (t) {
|
|
109
|
+
case 'text':
|
|
110
|
+
parts.push(blk.text);
|
|
111
|
+
break;
|
|
112
|
+
case 'tool_use': {
|
|
113
|
+
const tu = blk;
|
|
114
|
+
// Compact JSON (no indent) — pretty-printing bloats text ~5× and the renderer is row-aware.
|
|
115
|
+
let argsStr;
|
|
116
|
+
try {
|
|
117
|
+
argsStr = JSON.stringify(tu.input);
|
|
118
|
+
}
|
|
119
|
+
catch {
|
|
120
|
+
argsStr = String(tu.input);
|
|
121
|
+
}
|
|
122
|
+
parts.push(`[tool_use ${tu.name}]\n${argsStr}`);
|
|
123
|
+
break;
|
|
124
|
+
}
|
|
125
|
+
case 'tool_result': {
|
|
126
|
+
const tr = blk;
|
|
127
|
+
const inner = tr.content;
|
|
128
|
+
let innerText;
|
|
129
|
+
if (typeof inner === 'string') {
|
|
130
|
+
innerText = inner;
|
|
131
|
+
}
|
|
132
|
+
else if (Array.isArray(inner)) {
|
|
133
|
+
const subParts = [];
|
|
134
|
+
for (const sub of inner) {
|
|
135
|
+
if (!sub || typeof sub !== 'object')
|
|
136
|
+
continue;
|
|
137
|
+
if (sub.type === 'text') {
|
|
138
|
+
subParts.push(sub.text);
|
|
139
|
+
}
|
|
140
|
+
else if (sub.type === 'image') {
|
|
141
|
+
subParts.push('[image]');
|
|
142
|
+
}
|
|
143
|
+
}
|
|
144
|
+
innerText = subParts.join('\n');
|
|
145
|
+
}
|
|
146
|
+
else {
|
|
147
|
+
innerText = '';
|
|
148
|
+
}
|
|
149
|
+
const errMark = tr.is_error === true ? ' (error)' : '';
|
|
150
|
+
parts.push(`[tool_result${errMark}]\n${staleFreshnessHints(innerText)}`);
|
|
151
|
+
break;
|
|
152
|
+
}
|
|
153
|
+
case 'image':
|
|
154
|
+
parts.push('[image]');
|
|
155
|
+
break;
|
|
156
|
+
// 'thinking' and any other block type → drop silently.
|
|
157
|
+
default:
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
}
|
|
161
|
+
return parts.join('\n\n');
|
|
162
|
+
}
|
|
163
|
+
/** Return the caller's cache_control marker on a message, if any block carries one.
|
|
164
|
+
* Used to align freeze-chunk boundaries to roaming breakpoints so a marked segment
|
|
165
|
+
* stays independently cacheable instead of being silently flattened into the image. */
|
|
166
|
+
export function messageCacheControl(m) {
|
|
167
|
+
if (!Array.isArray(m.content))
|
|
168
|
+
return undefined;
|
|
169
|
+
for (let i = m.content.length - 1; i >= 0; i--) {
|
|
170
|
+
const b = m.content[i];
|
|
171
|
+
if (b && b.cache_control !== undefined)
|
|
172
|
+
return b.cache_control;
|
|
173
|
+
}
|
|
174
|
+
return undefined;
|
|
175
|
+
}
|
|
176
|
+
/** Serialize messages [fromInclusive..upToExclusive) to a text blob with
|
|
177
|
+
* `<role>…</role>` XML wrappers. Open+close tags bracket each turn so a misread
|
|
178
|
+
* boundary self-corrects and the model attributes speakers reliably even off a
|
|
179
|
+
* lossy image — bare `--- role ---` start-dividers let one role bleed into the
|
|
180
|
+
* next when a divider is missed. */
|
|
181
|
+
export function messagesToHistoryText(messages, upToExclusive, fromInclusive = 0) {
|
|
182
|
+
return messagesToHistorySegments(messages, upToExclusive, fromInclusive).text;
|
|
183
|
+
}
|
|
184
|
+
/** Like {@link messagesToHistoryText} but also returns the parallel slot string for
|
|
185
|
+
* colorByRole: a width-identical copy where each `<role>` tag is replaced by its
|
|
186
|
+
* role marker and the body is copied verbatim (slot 0). Role attribution is decided
|
|
187
|
+
* HERE, where the message role is known — never re-parsed out of flattened text.
|
|
188
|
+
* A tool_result block sits inside its user message and a tool_use block inside its
|
|
189
|
+
* assistant message, so each is owned by the turn that carries it. */
|
|
190
|
+
export function messagesToHistorySegments(messages, upToExclusive, fromInclusive = 0) {
|
|
191
|
+
const textOut = [];
|
|
192
|
+
const slotOut = [];
|
|
193
|
+
for (let i = fromInclusive; i < upToExclusive; i++) {
|
|
194
|
+
const m = messages[i];
|
|
195
|
+
const body = blocksToText(m.content);
|
|
196
|
+
if (!body.trim())
|
|
197
|
+
continue;
|
|
198
|
+
const isAssistant = m.role === 'assistant';
|
|
199
|
+
const tag = isAssistant ? 'assistant' : 'user';
|
|
200
|
+
const mark = isAssistant ? SLOT_MARK_ASSISTANT : SLOT_MARK_USER;
|
|
201
|
+
// Absolute turn index = message position from conversation start. Gives the model an
|
|
202
|
+
// explicit recency anchor so it can tell turn 1 from turn 60, instead of pattern-matching
|
|
203
|
+
// the most salient turn — primacy was resurrecting the OPENING turn as if it were the live
|
|
204
|
+
// request. MUST stay absolute (never "N ago" or "i/total"): a per-turn value that's stable
|
|
205
|
+
// once the turn closes keeps each frozen chunk byte-identical, so cache_read survives.
|
|
206
|
+
const attr = ` t="${i}"`;
|
|
207
|
+
textOut.push(`<${tag}${attr}>\n${body}\n</${tag}>`);
|
|
208
|
+
slotOut.push(roleSlotSegment(tag, body, mark, attr));
|
|
209
|
+
}
|
|
210
|
+
return { text: textOut.join('\n\n'), slotText: slotOut.join('\n\n') };
|
|
211
|
+
}
|
|
212
|
+
function compactPreview(text) {
|
|
213
|
+
const compact = text.replace(/\s+/g, ' ').trim();
|
|
214
|
+
if (compact.length <= LATEST_COLLAPSED_USER_PREVIEW_CHARS)
|
|
215
|
+
return compact;
|
|
216
|
+
return compact.slice(0, LATEST_COLLAPSED_USER_PREVIEW_CHARS).trimEnd() + '...';
|
|
217
|
+
}
|
|
218
|
+
// User-typed words must never survive ONLY as a truncated preview (#7: the EC demo's
|
|
219
|
+
// 577-char task lost its questions and "Reply as:" format at the 300-char preview cap,
|
|
220
|
+
// because no later turn restated them). Task-defining text is carried verbatim up to
|
|
221
|
+
// this cap; beyond it, head+tail elision keeps both the setup AND the trailing output
|
|
222
|
+
// format, which real prompts put at the end.
|
|
223
|
+
const LATEST_COLLAPSED_USER_VERBATIM_CHARS = 4000;
|
|
224
|
+
const VERBATIM_HEAD_CHARS = 2600;
|
|
225
|
+
const VERBATIM_TAIL_CHARS = 1400;
|
|
226
|
+
function verbatimTaskText(text) {
|
|
227
|
+
const t = text.trim();
|
|
228
|
+
if (t.length <= LATEST_COLLAPSED_USER_VERBATIM_CHARS)
|
|
229
|
+
return t;
|
|
230
|
+
const elided = t.length - VERBATIM_HEAD_CHARS - VERBATIM_TAIL_CHARS;
|
|
231
|
+
return (t.slice(0, VERBATIM_HEAD_CHARS) +
|
|
232
|
+
`\n[… middle elided (${elided} chars) …]\n` +
|
|
233
|
+
t.slice(t.length - VERBATIM_TAIL_CHARS));
|
|
234
|
+
}
|
|
235
|
+
/**
|
|
236
|
+
* The user's typed words in a user message: text blocks only, excluding
|
|
237
|
+
* <system-reminder> wrappers and (in the opening slab message) everything at or
|
|
238
|
+
* before the '[End of rendered context.]' boundary — same rule as
|
|
239
|
+
* demoteProtectedHeadText, so pxpipe scaffolding is never mistaken for the task.
|
|
240
|
+
*/
|
|
241
|
+
function typedUserText(content) {
|
|
242
|
+
if (typeof content === 'string')
|
|
243
|
+
return content.trim();
|
|
244
|
+
if (!Array.isArray(content))
|
|
245
|
+
return '';
|
|
246
|
+
const boundaryIdx = content.findIndex((b) => b && typeof b === 'object' &&
|
|
247
|
+
b.type === 'text' &&
|
|
248
|
+
b.text === '[End of rendered context.]');
|
|
249
|
+
const parts = [];
|
|
250
|
+
for (let i = 0; i < content.length; i++) {
|
|
251
|
+
if (boundaryIdx >= 0 && i <= boundaryIdx)
|
|
252
|
+
continue;
|
|
253
|
+
const blk = content[i];
|
|
254
|
+
if (!blk || typeof blk !== 'object')
|
|
255
|
+
continue;
|
|
256
|
+
if (blk.type !== 'text')
|
|
257
|
+
continue;
|
|
258
|
+
const text = blk.text.trim();
|
|
259
|
+
if (!text)
|
|
260
|
+
continue;
|
|
261
|
+
if (text.startsWith('<system-reminder>'))
|
|
262
|
+
continue;
|
|
263
|
+
parts.push(text);
|
|
264
|
+
}
|
|
265
|
+
return parts.join('\n\n');
|
|
266
|
+
}
|
|
267
|
+
/**
|
|
268
|
+
* Demote request TEXT in the protected head (slab anchor) to a marked PRIOR-CONTEXT
|
|
269
|
+
* tombstone. The session's OPENING user turn rides in the SAME message as the slab
|
|
270
|
+
* images (transform.ts sets protectedPrefix = firstUserIdx + 1 to keep that message
|
|
271
|
+
* from collapsing into [image] placeholders). Protecting it for the cache anchor also
|
|
272
|
+
* passed its request text through as clean native text at the very TOP — ahead of the
|
|
273
|
+
* synthetic history block — where the model reads it as the LIVE request. It never is:
|
|
274
|
+
* the live request is always in the tail (tail = messages.slice(collapseLen),
|
|
275
|
+
* keepTail >= 1), so any text in the protected head is, by construction, stale.
|
|
276
|
+
*
|
|
277
|
+
* Image/tool blocks (the slab) pass through byte-identical so the cache anchor and any
|
|
278
|
+
* cache_control breakpoint survive; the demotion is a pure function of the message, so
|
|
279
|
+
* the protected prefix stays byte-stable across turns (one-time re-cache on deploy).
|
|
280
|
+
*/
|
|
281
|
+
function demoteProtectedHeadText(head) {
|
|
282
|
+
return head.map((m, idx) => {
|
|
283
|
+
if (m.role !== 'user')
|
|
284
|
+
return m;
|
|
285
|
+
const tomb = (preview, cc) => {
|
|
286
|
+
const t = {
|
|
287
|
+
type: 'text',
|
|
288
|
+
text: `[Opening turn <user t="${idx}"> of this session — PRIOR CONTEXT ONLY, ` +
|
|
289
|
+
`superseded by later turns; NOT the current request and must not be acted ` +
|
|
290
|
+
`on. Preview: "${preview}"]`,
|
|
291
|
+
};
|
|
292
|
+
if (cc !== undefined) {
|
|
293
|
+
t.cache_control = cc;
|
|
294
|
+
}
|
|
295
|
+
return t;
|
|
296
|
+
};
|
|
297
|
+
if (typeof m.content === 'string') {
|
|
298
|
+
const preview = compactPreview(m.content);
|
|
299
|
+
return preview ? { ...m, content: [tomb(preview)] } : m;
|
|
300
|
+
}
|
|
301
|
+
if (!Array.isArray(m.content))
|
|
302
|
+
return m;
|
|
303
|
+
// pxpipe's own slab scaffolding (the rendered images, the fact-sheet, and the
|
|
304
|
+
// '[End of rendered context.]' boundary) is NOT the user's request and must
|
|
305
|
+
// survive byte-identical: relocateAnchorToHistoryImage keys on that boundary
|
|
306
|
+
// text to locate the slab cache anchor. Only the user's stale opening turn —
|
|
307
|
+
// the blocks AFTER the boundary — gets demoted. With no boundary (the slab did
|
|
308
|
+
// not image) boundaryIdx is -1 and the whole message demotes, exactly as before.
|
|
309
|
+
const boundaryIdx = m.content.findIndex((b) => b && typeof b === 'object' &&
|
|
310
|
+
b.type === 'text' &&
|
|
311
|
+
b.text === '[End of rendered context.]');
|
|
312
|
+
let changed = false;
|
|
313
|
+
const out = [];
|
|
314
|
+
for (let i = 0; i < m.content.length; i++) {
|
|
315
|
+
const blk = m.content[i];
|
|
316
|
+
if (boundaryIdx >= 0 && i <= boundaryIdx) {
|
|
317
|
+
out.push(blk); // slab images + fact-sheet + boundary: proxy scaffolding, kept verbatim
|
|
318
|
+
continue;
|
|
319
|
+
}
|
|
320
|
+
if (blk && typeof blk === 'object' && blk.type === 'text') {
|
|
321
|
+
const preview = compactPreview(blk.text);
|
|
322
|
+
if (preview) {
|
|
323
|
+
out.push(tomb(preview, blk.cache_control));
|
|
324
|
+
changed = true;
|
|
325
|
+
continue;
|
|
326
|
+
}
|
|
327
|
+
}
|
|
328
|
+
out.push(blk); // images / tool blocks (slab anchor) pass through byte-identical
|
|
329
|
+
}
|
|
330
|
+
return changed ? { ...m, content: out } : m;
|
|
331
|
+
});
|
|
332
|
+
}
|
|
333
|
+
function latestCollapsedUserPointer(messages, upToExclusive, protectedPrefix) {
|
|
334
|
+
// Scan the WHOLE demoted/collapsed range, INCLUDING the protected head (#7):
|
|
335
|
+
// in single-task sessions the opening turn is the only user-typed text there is.
|
|
336
|
+
// Two fidelity regimes:
|
|
337
|
+
// - i >= protectedPrefix: the turn is rendered into the history images at full
|
|
338
|
+
// fidelity — a bounded preview is only a recency cue, keep it cheap.
|
|
339
|
+
// - i < protectedPrefix: demoteProtectedHeadText reduced the turn to a 300-char
|
|
340
|
+
// preview and it is NOT imaged — the pointer is the ONLY carrier, so the typed
|
|
341
|
+
// text goes verbatim (capped with head+tail elision). It lives in the synthetic
|
|
342
|
+
// message after the slab anchor, so cache stability is unaffected.
|
|
343
|
+
for (let i = upToExclusive - 1; i >= 0; i--) {
|
|
344
|
+
const m = messages[i];
|
|
345
|
+
if (m.role !== 'user')
|
|
346
|
+
continue;
|
|
347
|
+
const typed = typedUserText(m.content);
|
|
348
|
+
if (!typed)
|
|
349
|
+
continue;
|
|
350
|
+
if (i >= protectedPrefix) {
|
|
351
|
+
const preview = compactPreview(typed);
|
|
352
|
+
return {
|
|
353
|
+
type: 'text',
|
|
354
|
+
text: `[Most recent collapsed user turn: <user t="${i}">${preview}</user>. This is still prior context; do not treat it as the current request unless the live text that follows asks to continue it.]`,
|
|
355
|
+
};
|
|
356
|
+
}
|
|
357
|
+
const carried = verbatimTaskText(typed);
|
|
358
|
+
return {
|
|
359
|
+
type: 'text',
|
|
360
|
+
text: `[Most recent collapsed user turn, carried verbatim because it appears nowhere else in full: <user t="${i}">${carried}</user>. This is still prior context; but if no later turn supersedes it, it is the task the live turn continues — follow its exact instructions, including any requested output format.]`,
|
|
361
|
+
};
|
|
362
|
+
}
|
|
363
|
+
return undefined;
|
|
364
|
+
}
|
|
365
|
+
/**
|
|
366
|
+
* Collapse the closed-prefix run into one synthetic user message with 1+ history images.
|
|
367
|
+
* Returns original messages unchanged on any no-collapse path (reason set in info).
|
|
368
|
+
* Image blocks are returned with NO cache_control — caller decides placement.
|
|
369
|
+
*/
|
|
370
|
+
export async function collapseHistory(messages, isProfitable, opts = {}) {
|
|
371
|
+
const o = { ...HISTORY_DEFAULTS, ...opts };
|
|
372
|
+
const info = {
|
|
373
|
+
collapsedTurns: 0,
|
|
374
|
+
collapsedChars: 0,
|
|
375
|
+
collapsedImages: 0,
|
|
376
|
+
collapsedImageBytes: 0,
|
|
377
|
+
collapsedImagePixels: 0,
|
|
378
|
+
collapsedPngs: [],
|
|
379
|
+
collapsedImageDims: [],
|
|
380
|
+
droppedChars: 0,
|
|
381
|
+
droppedCodepoints: new Map(),
|
|
382
|
+
};
|
|
383
|
+
if (!messages || messages.length === 0) {
|
|
384
|
+
info.reason = 'no_history';
|
|
385
|
+
return { messages: messages ?? [], info };
|
|
386
|
+
}
|
|
387
|
+
// Protected leading messages (slab) pass through untouched; collapse starts after them.
|
|
388
|
+
const protectedPrefix = Math.max(0, Math.min(o.protectedPrefix ?? 0, messages.length));
|
|
389
|
+
// Snap the cutoff to a collapseChunk grid so the rendered PNG stays byte-identical
|
|
390
|
+
// across turns and keeps hitting Anthropic's prompt cache. See docs/HISTORY_CACHE_MODEL.md.
|
|
391
|
+
// Floor at minCollapsePrefix + protectedPrefix so short histories still collapse.
|
|
392
|
+
const rawCutoff = messages.length - o.keepTail;
|
|
393
|
+
const cutoff = o.collapseChunk > 0
|
|
394
|
+
? Math.min(rawCutoff, Math.max(o.minCollapsePrefix + protectedPrefix, Math.floor(rawCutoff / o.collapseChunk) * o.collapseChunk))
|
|
395
|
+
: rawCutoff;
|
|
396
|
+
const boundary = findClosedPrefixBoundary(messages, cutoff);
|
|
397
|
+
if (boundary < 0) {
|
|
398
|
+
info.reason = 'no_closed_prefix';
|
|
399
|
+
return { messages, info };
|
|
400
|
+
}
|
|
401
|
+
// Need at least minCollapsePrefix turns in [protectedPrefix..boundary] — collapsing
|
|
402
|
+
// 2-3 turns is net cost (cache-amortization math doesn't work at small scale).
|
|
403
|
+
const collapseLen = boundary + 1;
|
|
404
|
+
if (collapseLen - protectedPrefix < o.minCollapsePrefix) {
|
|
405
|
+
info.reason = 'prefix_too_short';
|
|
406
|
+
return { messages, info };
|
|
407
|
+
}
|
|
408
|
+
// Exclude slab messages (protectedPrefix) from serialization.
|
|
409
|
+
const text = messagesToHistoryText(messages, collapseLen, protectedPrefix);
|
|
410
|
+
if (!text || text.length === 0) {
|
|
411
|
+
info.reason = 'render_empty';
|
|
412
|
+
return { messages, info };
|
|
413
|
+
}
|
|
414
|
+
// Reflow for RENDERING ONLY: pack short lines + mark hard breaks with ↵ so the
|
|
415
|
+
// newline-heavy transcript fills full rows instead of one line per row. Same
|
|
416
|
+
// glyph size (cols unchanged) → identical legibility, fewer images, more saved.
|
|
417
|
+
// `text` stays original — it backs `collapsedChars` and the cache byte-stability.
|
|
418
|
+
const safeText = neutralizeSentinel(text);
|
|
419
|
+
const renderText = o.reflow ? reflow(safeText) ?? safeText : text;
|
|
420
|
+
if (!isProfitable(renderText, o.cols)) { // pass string, not length — see ProfitableFn
|
|
421
|
+
info.reason = 'not_profitable';
|
|
422
|
+
info.collapsedChars = text.length; // surface what we DIDN'T compress
|
|
423
|
+
return { messages, info };
|
|
424
|
+
}
|
|
425
|
+
// APPEND-ONLY rendering. Render the collapse range [protectedPrefix..collapseLen)
|
|
426
|
+
// as independent image blocks on an ABSOLUTE message grid anchored at
|
|
427
|
+
// protectedPrefix (step = freezeChunk). A completed chunk's bytes are fixed by
|
|
428
|
+
// its message range alone, so old chunks stay byte-identical as the conversation
|
|
429
|
+
// grows (cache_read forever); only the newest partial chunk re-renders.
|
|
430
|
+
//
|
|
431
|
+
// Chunk-end positions = the absolute grid ∪ caller cache_control marks: a marked
|
|
432
|
+
// message forces a split right after it, and that chunk's LAST image carries the
|
|
433
|
+
// caller's marker — so a roaming breakpoint survives as an aligned, independently
|
|
434
|
+
// cacheable image boundary instead of being silently flattened (count conserved,
|
|
435
|
+
// never added). Each chunk is reflowed and rendered on its own, which is what
|
|
436
|
+
// makes the bytes a pure function of the chunk's messages.
|
|
437
|
+
const step = o.freezeChunk > 0 ? o.freezeChunk : collapseLen - protectedPrefix;
|
|
438
|
+
const ends = new Set();
|
|
439
|
+
for (let e = protectedPrefix + step; e < collapseLen; e += step)
|
|
440
|
+
ends.add(e);
|
|
441
|
+
const markerByEnd = new Map();
|
|
442
|
+
for (let i = protectedPrefix; i < collapseLen; i++) {
|
|
443
|
+
const cc = messageCacheControl(messages[i]);
|
|
444
|
+
if (cc !== undefined) {
|
|
445
|
+
ends.add(i + 1);
|
|
446
|
+
markerByEnd.set(i + 1, cc);
|
|
447
|
+
}
|
|
448
|
+
}
|
|
449
|
+
ends.add(collapseLen);
|
|
450
|
+
const sortedEnds = [...ends].filter((e) => e > protectedPrefix && e <= collapseLen).sort((a, b) => a - b);
|
|
451
|
+
// Carry-over anchor end: the largest FULLY grid-aligned chunk boundary strictly
|
|
452
|
+
// before collapseLen. That chunk's bytes are frozen across window advances, unlike
|
|
453
|
+
// the newest partial chunk — so it's the stable place to pin the cache breakpoint (#11).
|
|
454
|
+
let carryOverEnd = -1;
|
|
455
|
+
for (let e = protectedPrefix + step; e < collapseLen; e += step)
|
|
456
|
+
carryOverEnd = e;
|
|
457
|
+
let carryOverOrdinal = -1;
|
|
458
|
+
const imageBlocks = [];
|
|
459
|
+
let chunkStart = protectedPrefix;
|
|
460
|
+
for (const chunkEnd of sortedEnds) {
|
|
461
|
+
const seg = messagesToHistorySegments(messages, chunkEnd, chunkStart);
|
|
462
|
+
chunkStart = chunkEnd;
|
|
463
|
+
if (!seg.text || seg.text.length === 0)
|
|
464
|
+
continue;
|
|
465
|
+
// Reflow the text and its parallel slot string in lockstep so role attribution
|
|
466
|
+
// stays codepoint-aligned with the rendered text. The two have identical newline
|
|
467
|
+
// structure (slot bodies are verbatim copies), so minify/reflow mutate them the
|
|
468
|
+
// same way; reflow() only bails on a ↵ collision, which hits both identically.
|
|
469
|
+
let chunkRender = seg.text;
|
|
470
|
+
let chunkSlot = seg.slotText;
|
|
471
|
+
if (o.reflow) {
|
|
472
|
+
// Neutralize pre-existing ↵ first (1:1 swap at identical positions in text+slot, so
|
|
473
|
+
// they stay codepoint-aligned) — otherwise reflow bails and the chunk renders raw,
|
|
474
|
+
// unpacked. This conversation's transcript literally contains ↵, which would defeat
|
|
475
|
+
// packing on exactly the long sessions where collapse matters most.
|
|
476
|
+
const safeText = neutralizeSentinel(seg.text);
|
|
477
|
+
const safeSlot = neutralizeSentinel(seg.slotText);
|
|
478
|
+
const rt = reflow(safeText);
|
|
479
|
+
const rs = reflow(safeSlot);
|
|
480
|
+
if (rt !== null && rs !== null) {
|
|
481
|
+
chunkRender = rt;
|
|
482
|
+
chunkSlot = rs;
|
|
483
|
+
}
|
|
484
|
+
else {
|
|
485
|
+
chunkRender = safeText;
|
|
486
|
+
chunkSlot = safeSlot;
|
|
487
|
+
}
|
|
488
|
+
}
|
|
489
|
+
// Use the dense readable profile (not full-canvas) to keep code/config legible.
|
|
490
|
+
// colorByRole tints the structural <role> tags so turn boundaries are scannable
|
|
491
|
+
// in the history image; it's token-free (vision cost is by pixel dims, not PNG
|
|
492
|
+
// byte depth) and carries the serialize-time slot string instead of re-parsing.
|
|
493
|
+
const imgs = await renderTextToPngsWithCharLimit(chunkRender, DENSE_CONTENT_COLS, DENSE_CONTENT_CHARS_PER_IMAGE, { ...DENSE_RENDER_STYLE, colorByRole: true }, undefined, chunkSlot);
|
|
494
|
+
const markerCC = markerByEnd.get(chunkEnd);
|
|
495
|
+
for (let k = 0; k < imgs.length; k++) {
|
|
496
|
+
const img = imgs[k];
|
|
497
|
+
const block = {
|
|
498
|
+
type: 'image',
|
|
499
|
+
source: {
|
|
500
|
+
type: 'base64',
|
|
501
|
+
media_type: 'image/png',
|
|
502
|
+
data: bytesToBase64(img.png),
|
|
503
|
+
},
|
|
504
|
+
};
|
|
505
|
+
// Mark the LAST image of a marked segment — the caller's breakpoint anchor.
|
|
506
|
+
if (markerCC !== undefined && k === imgs.length - 1)
|
|
507
|
+
block.cache_control = markerCC;
|
|
508
|
+
imageBlocks.push(block);
|
|
509
|
+
info.collapsedImageBytes += img.png.length;
|
|
510
|
+
info.collapsedImagePixels += img.width * img.height;
|
|
511
|
+
info.collapsedPngs.push(img.png);
|
|
512
|
+
info.collapsedImageDims.push({ width: img.width, height: img.height });
|
|
513
|
+
info.droppedChars += img.droppedChars;
|
|
514
|
+
for (const [cp, n] of img.droppedCodepoints) {
|
|
515
|
+
info.droppedCodepoints.set(cp, (info.droppedCodepoints.get(cp) ?? 0) + n);
|
|
516
|
+
}
|
|
517
|
+
}
|
|
518
|
+
// The carry-over chunk's LAST image is the newest byte-stable history image.
|
|
519
|
+
// Record its ordinal so the relocator pins the cache breakpoint here instead of
|
|
520
|
+
// on the still-growing newest chunk, which busts every window advance (#11).
|
|
521
|
+
if (chunkEnd === carryOverEnd)
|
|
522
|
+
carryOverOrdinal = imageBlocks.length - 1;
|
|
523
|
+
}
|
|
524
|
+
if (imageBlocks.length === 0) {
|
|
525
|
+
info.reason = 'render_empty';
|
|
526
|
+
return { messages, info };
|
|
527
|
+
}
|
|
528
|
+
const latestUserPointer = latestCollapsedUserPointer(messages, collapseLen, protectedPrefix);
|
|
529
|
+
const historyFactSheet = factSheetText(text);
|
|
530
|
+
const syntheticContent = [
|
|
531
|
+
{ type: 'text', text: HISTORY_SYNTHETIC_INTRO },
|
|
532
|
+
...imageBlocks,
|
|
533
|
+
...(latestUserPointer ? [latestUserPointer] : []),
|
|
534
|
+
...(historyFactSheet ? [{ type: 'text', text: historyFactSheet }] : []),
|
|
535
|
+
{ type: 'text', text: HISTORY_SYNTHETIC_OUTRO },
|
|
536
|
+
];
|
|
537
|
+
const syntheticUser = {
|
|
538
|
+
role: 'user',
|
|
539
|
+
content: syntheticContent,
|
|
540
|
+
};
|
|
541
|
+
// Demote stale request text in the protected head so the session's opening turn
|
|
542
|
+
// can't surface as clean native text ahead of the history image and read as live.
|
|
543
|
+
const head = demoteProtectedHeadText(messages.slice(0, protectedPrefix));
|
|
544
|
+
const tail = messages.slice(collapseLen);
|
|
545
|
+
info.collapsedTurns = collapseLen - protectedPrefix;
|
|
546
|
+
info.collapsedChars = text.length;
|
|
547
|
+
info.collapsedImages = imageBlocks.length;
|
|
548
|
+
if (carryOverOrdinal >= 0)
|
|
549
|
+
info.carryOverImageOrdinal = carryOverOrdinal;
|
|
550
|
+
// [slab, history image, live tail] — slab cache_control anchor stays at the front.
|
|
551
|
+
return { messages: [...head, syntheticUser, ...tail], info };
|
|
552
|
+
}
|
|
553
|
+
//# sourceMappingURL=history.js.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { getAllowedModelBases, getConfiguredModelBases, isPxpipeSupportedGptModel, isPxpipeSupportedModel, setAllowedModelBases, shouldTransformAnthropicMessages, type PxpipeApplicabilityInput, type PxpipeApplicabilityReason, } from './applicability.js';
|
|
2
|
+
export { buildCountTokensBodies, buildBaselineCountTokensBody, buildCacheablePrefixCountTokensBody, countCacheControlMarkers, type CountTokensBodies, } from './measurement.js';
|
|
3
|
+
export { transformAnthropicMessages, renderTextToImages, type PxpipeOptions, type PxpipeReason, type PxpipeTransformInput, type PxpipeTransformResult, type RenderTextToImagesOptions, type RenderedTextImage, type RenderTextToImagesResult, } from './library.js';
|
|
4
|
+
export { transformRequest, type TransformInfo as PxpipeTransformInfo, type TransformOptions, type KeepSharpBlock, type RecoverableBlock, } from './transform.js';
|
|
5
|
+
export { transformOpenAIChatCompletions, transformOpenAIResponses, resolveVisionCost, openAIVisionTokens } from './openai.js';
|
|
6
|
+
export { createProxy, type ProxyConfig, type ProxyEvent } from './proxy.js';
|
|
7
|
+
export { computeActualInputEff, computeBaselineInputEff, CACHE_CREATE_RATE, CACHE_READ_RATE, } from './baseline.js';
|
|
8
|
+
//# sourceMappingURL=index.d.ts.map
|
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
export { getAllowedModelBases, getConfiguredModelBases, isPxpipeSupportedGptModel, isPxpipeSupportedModel, setAllowedModelBases, shouldTransformAnthropicMessages, } from './applicability.js';
|
|
2
|
+
export { buildCountTokensBodies, buildBaselineCountTokensBody, buildCacheablePrefixCountTokensBody, countCacheControlMarkers, } from './measurement.js';
|
|
3
|
+
export { transformAnthropicMessages, renderTextToImages, } from './library.js';
|
|
4
|
+
export { transformRequest, } from './transform.js';
|
|
5
|
+
export { transformOpenAIChatCompletions, transformOpenAIResponses, resolveVisionCost, openAIVisionTokens } from './openai.js';
|
|
6
|
+
export { createProxy } from './proxy.js';
|
|
7
|
+
export { computeActualInputEff, computeBaselineInputEff, CACHE_CREATE_RATE, CACHE_READ_RATE, } from './baseline.js';
|
|
8
|
+
//# sourceMappingURL=index.js.map
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { type RenderStyle } from './render.js';
|
|
2
|
+
import { type TransformInfo, type TransformOptions, type KeepSharpBlock, type RecoverableBlock } from './transform.js';
|
|
3
|
+
export type { KeepSharpBlock, RecoverableBlock };
|
|
4
|
+
export type BytesLike = Uint8Array | ArrayBuffer | ArrayBufferView;
|
|
5
|
+
export interface PxpipeOptions extends Pick<TransformOptions, 'charsPerToken' | 'historyAmortizationHorizon' | 'keepSharp' | 'emitRecoverable'> {
|
|
6
|
+
/** Test/debug-only bypass. Product hosts should prefer their dashboard setting. */
|
|
7
|
+
readonly compress?: boolean;
|
|
8
|
+
}
|
|
9
|
+
export interface PxpipeTransformInput {
|
|
10
|
+
readonly body: BytesLike;
|
|
11
|
+
/** Resolved upstream model when available; aliases are accepted for applicability checks. */
|
|
12
|
+
readonly model?: string | null;
|
|
13
|
+
readonly requestId?: string;
|
|
14
|
+
readonly options?: PxpipeOptions;
|
|
15
|
+
}
|
|
16
|
+
export type PxpipeReason = 'applied' | 'unsupported_model' | 'parse_error' | 'below_min_chars' | 'below_min_tokens' | 'not_profitable' | 'compress_disabled' | 'image_limit' | 'transform_error' | 'passthrough';
|
|
17
|
+
export interface PxpipeTransformResult {
|
|
18
|
+
readonly body: Uint8Array;
|
|
19
|
+
readonly applied: boolean;
|
|
20
|
+
readonly reason: PxpipeReason;
|
|
21
|
+
readonly detail?: string;
|
|
22
|
+
readonly info: TransformInfo;
|
|
23
|
+
readonly cache: {
|
|
24
|
+
readonly ownsCacheControl: boolean;
|
|
25
|
+
readonly markerCount: number;
|
|
26
|
+
};
|
|
27
|
+
}
|
|
28
|
+
/**
|
|
29
|
+
* Library wrapper for the Anthropic Messages transformer: model gate, machine-readable
|
|
30
|
+
* reasons, and cache_control ownership flag (prevents hosts stacking a second injector).
|
|
31
|
+
*/
|
|
32
|
+
export declare function transformAnthropicMessages(input: PxpipeTransformInput): Promise<PxpipeTransformResult>;
|
|
33
|
+
export interface RenderTextToImagesOptions {
|
|
34
|
+
/** Wrap-width cap in cols. Default DENSE_CONTENT_COLS (384). */
|
|
35
|
+
readonly cols?: number;
|
|
36
|
+
/** Shrink the canvas to the widest actual line (default true). `false` keeps the
|
|
37
|
+
* full `cols` width — the proxy's eval-backed full-canvas behavior. */
|
|
38
|
+
readonly shrink?: boolean;
|
|
39
|
+
/** Columns to pack side-by-side. `'auto'` (default) packs as many as fit the width
|
|
40
|
+
* cap; a number forces that count (clamped to what fits). */
|
|
41
|
+
readonly multiCol?: number | 'auto';
|
|
42
|
+
/** Reflow the text before rendering (minify + join hard newlines with the ↵ sentinel so
|
|
43
|
+
* short lines pack into full-width rows). This is the proxy's dense history format and is
|
|
44
|
+
* what `pxpipe export` uses. Default false (raw one-line-per-row). */
|
|
45
|
+
readonly reflow?: boolean;
|
|
46
|
+
/** Max source chars per page. Default DENSE_CONTENT_CHARS_PER_IMAGE. */
|
|
47
|
+
readonly maxCharsPerImage?: number;
|
|
48
|
+
/** Render style. Default DENSE_RENDER_STYLE (bare 5×8 cell, anti-aliased). */
|
|
49
|
+
readonly style?: RenderStyle;
|
|
50
|
+
/** Max page height in px. Default MAX_HEIGHT_PX (728 — Anthropic 1568-edge / ~1.15 MP safe). */
|
|
51
|
+
readonly maxHeightPx?: number;
|
|
52
|
+
}
|
|
53
|
+
export interface RenderedTextImage {
|
|
54
|
+
readonly png: Uint8Array;
|
|
55
|
+
readonly width: number;
|
|
56
|
+
readonly height: number;
|
|
57
|
+
}
|
|
58
|
+
export interface RenderTextToImagesResult {
|
|
59
|
+
readonly pages: RenderedTextImage[];
|
|
60
|
+
/** Codepoints absent from the glyph atlas (rendered as blank cells). */
|
|
61
|
+
readonly droppedChars: number;
|
|
62
|
+
/** Σ width×height across all pages. */
|
|
63
|
+
readonly pixels: number;
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Render arbitrary text to dense PNG pages — the public, documented entry for the
|
|
67
|
+
* renderer the proxy uses internally. Sizes a narrow canvas to the content (`shrink`)
|
|
68
|
+
* and packs multiple columns (`multiCol`) so short-line content isn't priced at full
|
|
69
|
+
* width. Returns raw PNG bytes + pixel dimensions, ready to write to disk or wrap in
|
|
70
|
+
* image blocks. This is the surface SDK consumers should use instead of reaching into
|
|
71
|
+
* the internal leaf renderers in `render.ts`.
|
|
72
|
+
*/
|
|
73
|
+
export declare function renderTextToImages(text: string, opts?: RenderTextToImagesOptions): Promise<RenderTextToImagesResult>;
|
|
74
|
+
//# sourceMappingURL=library.d.ts.map
|