@xamukavila/pxpipe 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +312 -0
- package/bin/cli.js +7 -0
- package/dist/core/applicability.d.ts +31 -0
- package/dist/core/applicability.js +96 -0
- package/dist/core/atlas-gray.d.ts +26 -0
- package/dist/core/atlas-gray.js +64 -0
- package/dist/core/atlas.d.ts +34 -0
- package/dist/core/atlas.js +71 -0
- package/dist/core/baseline.d.ts +80 -0
- package/dist/core/baseline.js +101 -0
- package/dist/core/caveman.d.ts +38 -0
- package/dist/core/caveman.js +183 -0
- package/dist/core/export.d.ts +128 -0
- package/dist/core/export.js +390 -0
- package/dist/core/factsheet.d.ts +78 -0
- package/dist/core/factsheet.js +216 -0
- package/dist/core/gpt-model-profiles.d.ts +60 -0
- package/dist/core/gpt-model-profiles.js +161 -0
- package/dist/core/history.d.ts +141 -0
- package/dist/core/history.js +553 -0
- package/dist/core/index.d.ts +8 -0
- package/dist/core/index.js +8 -0
- package/dist/core/library.d.ts +74 -0
- package/dist/core/library.js +133 -0
- package/dist/core/measurement.d.ts +22 -0
- package/dist/core/measurement.js +213 -0
- package/dist/core/openai-history.d.ts +124 -0
- package/dist/core/openai-history.js +494 -0
- package/dist/core/openai-savings.d.ts +44 -0
- package/dist/core/openai-savings.js +75 -0
- package/dist/core/openai.d.ts +24 -0
- package/dist/core/openai.js +839 -0
- package/dist/core/png.d.ts +11 -0
- package/dist/core/png.js +132 -0
- package/dist/core/proxy.d.ts +81 -0
- package/dist/core/proxy.js +730 -0
- package/dist/core/render.d.ts +188 -0
- package/dist/core/render.js +785 -0
- package/dist/core/schema-strip.d.ts +29 -0
- package/dist/core/schema-strip.js +160 -0
- package/dist/core/tracker.d.ts +154 -0
- package/dist/core/tracker.js +216 -0
- package/dist/core/transform.d.ts +362 -0
- package/dist/core/transform.js +1828 -0
- package/dist/core/types.d.ts +77 -0
- package/dist/core/types.js +8 -0
- package/dist/dashboard/fragments.d.ts +36 -0
- package/dist/dashboard/fragments.js +938 -0
- package/dist/dashboard/types.d.ts +154 -0
- package/dist/dashboard/types.js +3 -0
- package/dist/dashboard/vendor.d.ts +3 -0
- package/dist/dashboard/vendor.js +6 -0
- package/dist/dashboard.d.ts +245 -0
- package/dist/dashboard.js +1140 -0
- package/dist/export-collect.d.ts +36 -0
- package/dist/export-collect.js +59 -0
- package/dist/node.d.ts +9 -0
- package/dist/node.js +9038 -0
- package/dist/sessions.d.ts +172 -0
- package/dist/sessions.js +510 -0
- package/dist/stats.d.ts +74 -0
- package/dist/stats.js +248 -0
- package/dist/worker.d.ts +53 -0
- package/dist/worker.js +102 -0
- package/package.json +96 -0
|
@@ -0,0 +1,216 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Verbatim fact-sheet for imaged content.
|
|
3
|
+
*
|
|
4
|
+
* When pxpipe renders a block (system slab, history, tool_result, reminder) to a PNG,
|
|
5
|
+
* the precision-critical, hard-to-OCR strings inside it — file paths, URLs, SHAs/UUIDs,
|
|
6
|
+
* version numbers, CLI flags, large numbers, CONST_IDS — are exactly what a model is
|
|
7
|
+
* most likely to misread off the image yet most likely to need quoted verbatim. This
|
|
8
|
+
* module extracts those tokens so they ride next to the image as plain text: the model
|
|
9
|
+
* quotes them without re-reading the PNG, and they stay in the cached prefix.
|
|
10
|
+
*
|
|
11
|
+
* Deterministic by construction (fixed pattern order, length-desc/lexical sort, no
|
|
12
|
+
* Date/random) → the emitted text is byte-stable across turns and never busts the
|
|
13
|
+
* Anthropic prompt cache. Empirically ~5% of source chars on production history
|
|
14
|
+
* (median 4.9%, max 12.1%, N=10), which preserves the imaging token win.
|
|
15
|
+
*/
|
|
16
|
+
/** ReDoS-safe extraction patterns (each global). Ordered most- to least-specific so the
|
|
17
|
+
* longest, most-identifying tokens are kept first when the substring filter runs. */
|
|
18
|
+
const PATTERNS = [
|
|
19
|
+
/\bhttps?:\/\/[^\s)"'<>]+/g, // URLs
|
|
20
|
+
/\b[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}\b/g, // UUID
|
|
21
|
+
/(?:[\w@~+-]+)?(?:\/[\w.@+-]+)+\.[A-Za-z]\w{0,8}\b/g, // path with a file extension (multi-dot ok: .test.ts)
|
|
22
|
+
/\/[\w.@+-]+(?:\/[\w.@+-]+)+\/?/g, // dir path (>=2 segments)
|
|
23
|
+
/\b(?=[0-9a-f]*\d)[0-9a-f]{7,40}\b/g, // git sha / long hex (must contain a digit)
|
|
24
|
+
/\bv?\d+\.\d+(?:\.\d+)?(?:[-+][\w.]+)?\b/g, // version string
|
|
25
|
+
/(?:^|[^\w-])(--?[A-Za-z][\w-]+)/g, // CLI flag (token in capture group 1)
|
|
26
|
+
/\b\d[\d,_]{3,}\b/g, // large / separated number
|
|
27
|
+
/\b\d+\.\d+\b/g, // decimal
|
|
28
|
+
/\b[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+\b/g, // CONST_IDS / env var names
|
|
29
|
+
// Ticket/advisory-style codes: uppercase hyphenated with ≥1 digit (PROJ-1482,
|
|
30
|
+
// CVE-2024-30078, AUDIT-ZX9). Digit lookahead is bounded → no backtracking blowup.
|
|
31
|
+
/\b(?=[A-Z0-9-]{0,119}\d)[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+\b/g,
|
|
32
|
+
];
|
|
33
|
+
const MIN_LEN = 3;
|
|
34
|
+
const MAX_LEN = 120;
|
|
35
|
+
/** Budget cap: highest-priority tokens kept first. Exported so consumers can report drops. */
|
|
36
|
+
export const MAX_TOKENS = 64;
|
|
37
|
+
// At most this many URL exemplars: URLs are long, structured, low OCR-risk, and usually
|
|
38
|
+
// reconstructable, so they must never crowd out short zero-redundancy tokens.
|
|
39
|
+
const MAX_URLS = 8;
|
|
40
|
+
const MAX_SEEN = 2048; // defensive bound on distinct tokens entering substring-collapse
|
|
41
|
+
const MAX_SCAN = 262_144; // defensive input bound; tool_results are already paged
|
|
42
|
+
const MAX_CHUNK = 512; // whitespace-free chunks longer than this are blobs (base64, minified) — skip
|
|
43
|
+
/** Budget priority by token SHAPE, not length — length is anti-correlated with
|
|
44
|
+
* OCR-risk×consequence: a 70-char URL is structured and reconstructable, while a 7-char
|
|
45
|
+
* hex SHA or a port has zero redundancy and fails silently when misread. So short opaque
|
|
46
|
+
* identifiers outrank long URLs when the budget is tight. Pure + total → deterministic →
|
|
47
|
+
* cache-stable. Tiers: 0 = protect always, 1 = paths/versions/misc, 2 = URLs (cap + last). */
|
|
48
|
+
const SHAPE_UUID = /^[0-9a-fA-F]{8}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{4}-[0-9a-fA-F]{12}$/;
|
|
49
|
+
const SHAPE_HEX = /^(?=[0-9a-f]*\d)[0-9a-f]{7,40}$/; // git sha / opaque hex
|
|
50
|
+
const SHAPE_CONST = /^[A-Z][A-Z0-9]{2,}(?:_[A-Z0-9]+)+$/; // CONST_IDS / env vars
|
|
51
|
+
const SHAPE_TICKET = /^(?=[A-Z0-9-]*\d)[A-Z][A-Z0-9]+(?:-[A-Z0-9]+)+$/; // PROJ-1482 / CVE-2024-30078
|
|
52
|
+
const SHAPE_FLAG = /^--?[A-Za-z][\w-]+$/; // CLI flag
|
|
53
|
+
const SHAPE_NUM = /^\d[\d,_]*$|^\d+\.\d+$/; // port / large or separated number / decimal
|
|
54
|
+
const SHAPE_URL = /^https?:\/\//;
|
|
55
|
+
/** Lower tier = higher keep-priority. Pure function of the token → deterministic. */
|
|
56
|
+
function priorityTier(tok) {
|
|
57
|
+
if (SHAPE_HEX.test(tok) ||
|
|
58
|
+
SHAPE_UUID.test(tok) ||
|
|
59
|
+
SHAPE_CONST.test(tok) ||
|
|
60
|
+
SHAPE_TICKET.test(tok) ||
|
|
61
|
+
SHAPE_FLAG.test(tok) ||
|
|
62
|
+
SHAPE_NUM.test(tok)) {
|
|
63
|
+
return 0;
|
|
64
|
+
}
|
|
65
|
+
if (SHAPE_URL.test(tok))
|
|
66
|
+
return 2;
|
|
67
|
+
return 1;
|
|
68
|
+
}
|
|
69
|
+
/**
|
|
70
|
+
* Extract deduped, precision-critical tokens from `text`. Substrings of a longer kept
|
|
71
|
+
* token are dropped (so `/github.com` inside the full URL, `lib/x.ts` inside
|
|
72
|
+
* `src/lib/x.ts`, etc. collapse to the most specific form); the 64-token budget is then
|
|
73
|
+
* filled by priority tier (see `priorityTier`) so short, high-consequence tokens are never
|
|
74
|
+
* evicted by long low-risk URLs.
|
|
75
|
+
*
|
|
76
|
+
* Every token class is whitespace-free, so we split on whitespace first and skip
|
|
77
|
+
* blob-length chunks. That bounds each regex to a short chunk and keeps extraction
|
|
78
|
+
* strictly O(n) — no quadratic backtracking on delimiter-heavy input like base64 or
|
|
79
|
+
* minified bundles (which embed `/` and would otherwise make the path patterns blow up).
|
|
80
|
+
*/
|
|
81
|
+
export function extractFactSheetTokens(text) {
|
|
82
|
+
return extractFactSheetEntries(text).map((e) => e.token);
|
|
83
|
+
}
|
|
84
|
+
/**
|
|
85
|
+
* Like `extractFactSheetTokens`, but each kept token carries its occurrence count.
|
|
86
|
+
* Counts make the fact sheet a *quantitative* index: tally questions over imaged
|
|
87
|
+
* content ("how many lines mention CODE-X?") become answerable from text instead
|
|
88
|
+
* of from counting rows of 5×8 px glyphs — the one operation page images are worst
|
|
89
|
+
* at. The kept-token SET and its order are byte-identical to the pre-count
|
|
90
|
+
* behaviour; only counts are new. Same-token spans matched by two patterns are
|
|
91
|
+
* deduped by offset so a token is never double-counted.
|
|
92
|
+
*/
|
|
93
|
+
export function extractFactSheetEntries(text) {
|
|
94
|
+
const scan = text.length > MAX_SCAN ? text.slice(0, MAX_SCAN) : text;
|
|
95
|
+
const counts = new Map();
|
|
96
|
+
for (const chunk of scan.split(/\s+/)) {
|
|
97
|
+
if (chunk.length < MIN_LEN || chunk.length > MAX_CHUNK)
|
|
98
|
+
continue;
|
|
99
|
+
// Offset-level dedup WITHIN this chunk: two patterns matching the identical
|
|
100
|
+
// token at the same position must count once. Keyed by token+start offset.
|
|
101
|
+
const spanSeen = new Set();
|
|
102
|
+
for (const re of PATTERNS) {
|
|
103
|
+
for (const m of chunk.matchAll(re)) {
|
|
104
|
+
// Strip trailing sentence punctuation pulled in from prose (`pull/93.` → `pull/93`);
|
|
105
|
+
// no real identifier we extract ends in these.
|
|
106
|
+
const tok = (m[1] ?? m[0]).trim().replace(/[.,;:!?]+$/, '');
|
|
107
|
+
if (tok.length < MIN_LEN || tok.length > MAX_LEN)
|
|
108
|
+
continue;
|
|
109
|
+
const key = `${m.index ?? 0}\x00${tok}`;
|
|
110
|
+
if (spanSeen.has(key))
|
|
111
|
+
continue;
|
|
112
|
+
spanSeen.add(key);
|
|
113
|
+
counts.set(tok, (counts.get(tok) ?? 0) + 1);
|
|
114
|
+
}
|
|
115
|
+
}
|
|
116
|
+
if (counts.size >= MAX_SEEN)
|
|
117
|
+
break;
|
|
118
|
+
}
|
|
119
|
+
// Phase 1 — substring collapse (length-desc): keep the most-specific form, folding e.g.
|
|
120
|
+
// a URL's path-portion into the full URL. Cross-tier on purpose. Total order (length,
|
|
121
|
+
// then lexical) so the result is independent of Set iteration order.
|
|
122
|
+
const ordered = [...counts.keys()].sort((a, b) => b.length - a.length || (a < b ? -1 : a > b ? 1 : 0));
|
|
123
|
+
const specific = [];
|
|
124
|
+
for (const t of ordered) {
|
|
125
|
+
if (!specific.some((k) => k.includes(t)))
|
|
126
|
+
specific.push(t);
|
|
127
|
+
}
|
|
128
|
+
// Phase 2 — allocate the budget by priority tier (shape, not length) so short,
|
|
129
|
+
// zero-redundancy tokens (SHAs, ports, flags) can never be evicted by long low-risk URLs.
|
|
130
|
+
// URLs are kept only as a few exemplars. Comparator is total → byte-stable output.
|
|
131
|
+
const ranked = specific
|
|
132
|
+
.map((t) => ({ t, tier: priorityTier(t) }))
|
|
133
|
+
.sort((a, b) => a.tier - b.tier || b.t.length - a.t.length || (a.t < b.t ? -1 : a.t > b.t ? 1 : 0));
|
|
134
|
+
const kept = [];
|
|
135
|
+
let urls = 0;
|
|
136
|
+
for (const { t, tier } of ranked) {
|
|
137
|
+
if (kept.length >= MAX_TOKENS)
|
|
138
|
+
break;
|
|
139
|
+
if (tier === 2 && urls++ >= MAX_URLS)
|
|
140
|
+
continue;
|
|
141
|
+
kept.push(t);
|
|
142
|
+
}
|
|
143
|
+
return kept.map((t) => ({ token: t, count: counts.get(t) ?? 1 }));
|
|
144
|
+
}
|
|
145
|
+
/**
|
|
146
|
+
* Page-aware variant of `extractFactSheetTokens` for large source texts.
|
|
147
|
+
*
|
|
148
|
+
* Splits `text` into chunks of `charsPerPage` (use `DENSE_CONTENT_CHARS_PER_IMAGE`
|
|
149
|
+
* from render.ts for the export pipeline), calls `extractFactSheetTokens` on each
|
|
150
|
+
* chunk (each chunk is smaller than MAX_SCAN so no truncation occurs), merges the
|
|
151
|
+
* results across all chunks with first-seen deduplication, then applies a single
|
|
152
|
+
* global priority-budget pass to select the best MAX_TOKENS identifiers.
|
|
153
|
+
*
|
|
154
|
+
* Returns `{ kept, dropped }` where `dropped` is the count of identifiers that
|
|
155
|
+
* survived extraction across all pages but did not fit in the MAX_TOKENS budget.
|
|
156
|
+
*
|
|
157
|
+
* Does NOT mutate the behaviour of `extractFactSheetTokens` or `factSheetText`.
|
|
158
|
+
*/
|
|
159
|
+
export function extractFactSheetTokensAllPages(text, charsPerPage) {
|
|
160
|
+
const { kept, dropped } = extractFactSheetEntriesAllPages(text, charsPerPage);
|
|
161
|
+
return { kept: kept.map((e) => e.token), dropped };
|
|
162
|
+
}
|
|
163
|
+
/** Entry-carrying variant of `extractFactSheetTokensAllPages`: same kept set and
|
|
164
|
+
* order, with per-token occurrence counts summed across all pages. */
|
|
165
|
+
export function extractFactSheetEntriesAllPages(text, charsPerPage) {
|
|
166
|
+
const counts = new Map();
|
|
167
|
+
const all = [];
|
|
168
|
+
// Walk the text in page-sized chunks. Each chunk is ≤ charsPerPage < MAX_SCAN,
|
|
169
|
+
// so extractFactSheetEntries will not truncate within a chunk.
|
|
170
|
+
const pageCount = Math.max(1, Math.ceil(text.length / charsPerPage));
|
|
171
|
+
for (let i = 0; i < pageCount; i++) {
|
|
172
|
+
const chunk = text.slice(i * charsPerPage, (i + 1) * charsPerPage);
|
|
173
|
+
for (const { token, count } of extractFactSheetEntries(chunk)) {
|
|
174
|
+
if (!counts.has(token))
|
|
175
|
+
all.push(token);
|
|
176
|
+
counts.set(token, (counts.get(token) ?? 0) + count);
|
|
177
|
+
}
|
|
178
|
+
}
|
|
179
|
+
// Re-apply the global priority budget so a tier-0 identifier on page 5
|
|
180
|
+
// is never evicted by many tier-1 tokens from page 1.
|
|
181
|
+
const ranked = all
|
|
182
|
+
.map((t) => ({ t, tier: priorityTier(t) }))
|
|
183
|
+
.sort((a, b) => a.tier - b.tier || b.t.length - a.t.length || (a.t < b.t ? -1 : a.t > b.t ? 1 : 0));
|
|
184
|
+
const kept = [];
|
|
185
|
+
let urls = 0;
|
|
186
|
+
for (const { t, tier } of ranked) {
|
|
187
|
+
if (kept.length >= MAX_TOKENS)
|
|
188
|
+
break;
|
|
189
|
+
if (tier === 2 && urls++ >= MAX_URLS)
|
|
190
|
+
continue;
|
|
191
|
+
kept.push({ token: t, count: counts.get(t) ?? 1 });
|
|
192
|
+
}
|
|
193
|
+
return { kept, dropped: all.length - kept.length };
|
|
194
|
+
}
|
|
195
|
+
const OPEN = '[Exact identifiers from the rendered context above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image: ';
|
|
196
|
+
/** Variant used when at least one token repeats — explains the ×N annotation so the
|
|
197
|
+
* model can answer tally questions from the sheet instead of counting glyph rows. */
|
|
198
|
+
const OPEN_COUNTS = '[Exact identifiers from the rendered context above (paths, ids, versions, numbers) — quote these verbatim instead of transcribing them from the image; ×N marks a token that occurs N times within the imaged content: ';
|
|
199
|
+
/** Build the one-line fact-sheet string from a pre-extracted token list. */
|
|
200
|
+
export function factSheetTextFromTokens(tokens) {
|
|
201
|
+
return tokens.length > 0 ? OPEN + tokens.join(' · ') + ']' : '';
|
|
202
|
+
}
|
|
203
|
+
/** Build the one-line fact-sheet string from token+count entries. Byte-identical to
|
|
204
|
+
* `factSheetTextFromTokens` when no token repeats, so existing sheets stay cache-stable. */
|
|
205
|
+
export function factSheetTextFromEntries(entries) {
|
|
206
|
+
if (entries.length === 0)
|
|
207
|
+
return '';
|
|
208
|
+
const anyRepeat = entries.some((e) => e.count >= 2);
|
|
209
|
+
const body = entries.map((e) => (e.count >= 2 ? `${e.token} ×${e.count}` : e.token)).join(' · ');
|
|
210
|
+
return (anyRepeat ? OPEN_COUNTS : OPEN) + body + ']';
|
|
211
|
+
}
|
|
212
|
+
/** One-line fact-sheet string for `text`, or `''` when nothing notable was found. */
|
|
213
|
+
export function factSheetText(text) {
|
|
214
|
+
return factSheetTextFromEntries(extractFactSheetEntries(text));
|
|
215
|
+
}
|
|
216
|
+
//# sourceMappingURL=factsheet.js.map
|
|
@@ -0,0 +1,60 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-model GPT rendering + vision-cost profiles.
|
|
3
|
+
*
|
|
4
|
+
* One place to retune when a new model ships with different image tokenization,
|
|
5
|
+
* a different downscale threshold (max safe portrait-strip width), or a different
|
|
6
|
+
* max image height. Every built-in profile is BEHAVIOR-IDENTICAL to the old
|
|
7
|
+
* hardcoded `resolveVisionCost` + `GPT_STRIP_COLS` + `MAX_HEIGHT_PX`, so existing
|
|
8
|
+
* cost numbers (1190 / 1445 / 2372 / 1464 / 630 …) are unchanged.
|
|
9
|
+
*
|
|
10
|
+
* Retune without a code change via the PXPIPE_GPT_PROFILES env var (JSON map of
|
|
11
|
+
* model-id PREFIX -> partial profile; longest matching prefix wins, checked
|
|
12
|
+
* BEFORE the built-in table). Partial fields fall back to the built-in match, so
|
|
13
|
+
* you can override just one knob:
|
|
14
|
+
*
|
|
15
|
+
* PXPIPE_GPT_PROFILES='{"gpt-5.6":{"vision":{"regime":"patch","multiplier":1,"patchCap":12000},"stripCols":200,"maxHeightPx":2400}}'
|
|
16
|
+
* PXPIPE_GPT_PROFILES='{"gpt-5.6":{"stripCols":176}}' # widen only
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* GPT strip height, DECOUPLED from render.ts's MAX_HEIGHT_PX (which is Anthropic's
|
|
20
|
+
* 1568-edge / ~1.15 MP clamp). OpenAI's pre-tokenize resize is different: fit within
|
|
21
|
+
* 2048×2048, then shortest side → 768. A 768-px-wide portrait strip up to 2048 px tall
|
|
22
|
+
* survives un-resampled, so GPT keeps the taller page. Every built-in cost number below
|
|
23
|
+
* (1190 / 1445 / 2372 / 1464 / 630 …) was calibrated at this height — do not re-link to
|
|
24
|
+
* the Anthropic constant.
|
|
25
|
+
*/
|
|
26
|
+
export declare const GPT_MAX_HEIGHT_PX = 1932;
|
|
27
|
+
/** Image-token cost model (mirrors OpenAI's mandatory pre-tokenize resize). */
|
|
28
|
+
export type GptVisionCost = {
|
|
29
|
+
regime: 'tile';
|
|
30
|
+
base: number;
|
|
31
|
+
perTile: number;
|
|
32
|
+
} | {
|
|
33
|
+
regime: 'patch';
|
|
34
|
+
multiplier: number;
|
|
35
|
+
patchCap: number;
|
|
36
|
+
};
|
|
37
|
+
export interface GptModelProfile {
|
|
38
|
+
/** How OpenAI bills the rendered images as input tokens. */
|
|
39
|
+
vision: GptVisionCost;
|
|
40
|
+
/** Max portrait-strip width in COLUMNS before the API downscales (destroying
|
|
41
|
+
* 5px glyphs). 152 cols x 5px + 8px pad = 768px = OpenAI's shortest-side floor. */
|
|
42
|
+
stripCols: number;
|
|
43
|
+
/** Max rendered image height in px. Threaded into the renderer so the gate's
|
|
44
|
+
* cost estimate and the actual page split agree. */
|
|
45
|
+
maxHeightPx: number;
|
|
46
|
+
}
|
|
47
|
+
/** Default downscale-safe strip width (768px). Exported as the global cols default. */
|
|
48
|
+
export declare const DEFAULT_GPT_STRIP_COLS = 152;
|
|
49
|
+
/**
|
|
50
|
+
* Conservative fallback for unrecognized models: tile 85/170 over-states cost,
|
|
51
|
+
* which biases the gate toward pass-through (safe). Matches gpt-4o/4.1/4.5.
|
|
52
|
+
*/
|
|
53
|
+
export declare const DEFAULT_GPT_PROFILE: GptModelProfile;
|
|
54
|
+
/**
|
|
55
|
+
* Resolve the full rendering + vision-cost profile for a model id. Env overrides
|
|
56
|
+
* (longest matching prefix) win over the built-in table; unknown models get the
|
|
57
|
+
* conservative `DEFAULT_GPT_PROFILE`.
|
|
58
|
+
*/
|
|
59
|
+
export declare function resolveGptProfile(model: string | null | undefined): GptModelProfile;
|
|
60
|
+
//# sourceMappingURL=gpt-model-profiles.d.ts.map
|
|
@@ -0,0 +1,161 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Per-model GPT rendering + vision-cost profiles.
|
|
3
|
+
*
|
|
4
|
+
* One place to retune when a new model ships with different image tokenization,
|
|
5
|
+
* a different downscale threshold (max safe portrait-strip width), or a different
|
|
6
|
+
* max image height. Every built-in profile is BEHAVIOR-IDENTICAL to the old
|
|
7
|
+
* hardcoded `resolveVisionCost` + `GPT_STRIP_COLS` + `MAX_HEIGHT_PX`, so existing
|
|
8
|
+
* cost numbers (1190 / 1445 / 2372 / 1464 / 630 …) are unchanged.
|
|
9
|
+
*
|
|
10
|
+
* Retune without a code change via the PXPIPE_GPT_PROFILES env var (JSON map of
|
|
11
|
+
* model-id PREFIX -> partial profile; longest matching prefix wins, checked
|
|
12
|
+
* BEFORE the built-in table). Partial fields fall back to the built-in match, so
|
|
13
|
+
* you can override just one knob:
|
|
14
|
+
*
|
|
15
|
+
* PXPIPE_GPT_PROFILES='{"gpt-5.6":{"vision":{"regime":"patch","multiplier":1,"patchCap":12000},"stripCols":200,"maxHeightPx":2400}}'
|
|
16
|
+
* PXPIPE_GPT_PROFILES='{"gpt-5.6":{"stripCols":176}}' # widen only
|
|
17
|
+
*/
|
|
18
|
+
/**
|
|
19
|
+
* GPT strip height, DECOUPLED from render.ts's MAX_HEIGHT_PX (which is Anthropic's
|
|
20
|
+
* 1568-edge / ~1.15 MP clamp). OpenAI's pre-tokenize resize is different: fit within
|
|
21
|
+
* 2048×2048, then shortest side → 768. A 768-px-wide portrait strip up to 2048 px tall
|
|
22
|
+
* survives un-resampled, so GPT keeps the taller page. Every built-in cost number below
|
|
23
|
+
* (1190 / 1445 / 2372 / 1464 / 630 …) was calibrated at this height — do not re-link to
|
|
24
|
+
* the Anthropic constant.
|
|
25
|
+
*/
|
|
26
|
+
export const GPT_MAX_HEIGHT_PX = 1932;
|
|
27
|
+
/** Default downscale-safe strip width (768px). Exported as the global cols default. */
|
|
28
|
+
export const DEFAULT_GPT_STRIP_COLS = 152;
|
|
29
|
+
const C = DEFAULT_GPT_STRIP_COLS;
|
|
30
|
+
const H = GPT_MAX_HEIGHT_PX;
|
|
31
|
+
/**
|
|
32
|
+
* Conservative fallback for unrecognized models: tile 85/170 over-states cost,
|
|
33
|
+
* which biases the gate toward pass-through (safe). Matches gpt-4o/4.1/4.5.
|
|
34
|
+
*/
|
|
35
|
+
export const DEFAULT_GPT_PROFILE = {
|
|
36
|
+
vision: { regime: 'tile', base: 85, perTile: 170 },
|
|
37
|
+
stripCols: C,
|
|
38
|
+
maxHeightPx: H,
|
|
39
|
+
};
|
|
40
|
+
/** True for the patch-billed mini/nano family (incl. o4-mini). */
|
|
41
|
+
const isMiniNanoPatch = (m) => /^(?:gpt-5(?:\.\d+)?|gpt-4\.1)-(?:mini|nano)/.test(m) || /^o4-mini/.test(m);
|
|
42
|
+
/**
|
|
43
|
+
* Built-in profiles, evaluated in order (first match wins). Precedence and
|
|
44
|
+
* numbers reproduce the previous hardcoded `resolveVisionCost` EXACTLY:
|
|
45
|
+
* mini/nano -> patch (nano 2.46 / mini 1.62, cap 1536), BEFORE 5.x flagship.
|
|
46
|
+
*/
|
|
47
|
+
const BUILTIN_RULES = [
|
|
48
|
+
// nano patch models: ceil(patches * 2.46), cap 1536
|
|
49
|
+
{
|
|
50
|
+
test: (m) => isMiniNanoPatch(m) && /nano/.test(m),
|
|
51
|
+
profile: { vision: { regime: 'patch', multiplier: 2.46, patchCap: 1536 }, stripCols: C, maxHeightPx: H },
|
|
52
|
+
},
|
|
53
|
+
// mini / o4-mini patch models: ceil(patches * 1.62), cap 1536
|
|
54
|
+
{
|
|
55
|
+
test: (m) => isMiniNanoPatch(m) && !/nano/.test(m),
|
|
56
|
+
profile: { vision: { regime: 'patch', multiplier: 1.62, patchCap: 1536 }, stripCols: C, maxHeightPx: H },
|
|
57
|
+
},
|
|
58
|
+
// gpt-5.6 flagship — EXPLICIT slot so the day-one retune is one line (or one env
|
|
59
|
+
// var). Identical to the generic 5.x rule below until the real numbers land.
|
|
60
|
+
{
|
|
61
|
+
test: (m) => /^gpt-5\.6/.test(m),
|
|
62
|
+
profile: { vision: { regime: 'patch', multiplier: 1, patchCap: 10000 }, stripCols: C, maxHeightPx: H },
|
|
63
|
+
},
|
|
64
|
+
// 5.x flagship (gpt-5.4/5.5/…, no -mini/-nano): patch, multiplier 1, detail:original cap
|
|
65
|
+
{
|
|
66
|
+
test: (m) => /^gpt-5\.\d/.test(m),
|
|
67
|
+
profile: { vision: { regime: 'patch', multiplier: 1, patchCap: 10000 }, stripCols: C, maxHeightPx: H },
|
|
68
|
+
},
|
|
69
|
+
// gpt-5 / gpt-5-chat-latest: tile 70/140
|
|
70
|
+
{
|
|
71
|
+
test: (m) => /^gpt-5/.test(m),
|
|
72
|
+
profile: { vision: { regime: 'tile', base: 70, perTile: 140 }, stripCols: C, maxHeightPx: H },
|
|
73
|
+
},
|
|
74
|
+
// o1 / o3 reasoning: tile 75/150
|
|
75
|
+
{
|
|
76
|
+
test: (m) => /^o[13]/.test(m),
|
|
77
|
+
profile: { vision: { regime: 'tile', base: 75, perTile: 150 }, stripCols: C, maxHeightPx: H },
|
|
78
|
+
},
|
|
79
|
+
];
|
|
80
|
+
function resolveBuiltin(m) {
|
|
81
|
+
for (const rule of BUILTIN_RULES)
|
|
82
|
+
if (rule.test(m))
|
|
83
|
+
return rule.profile;
|
|
84
|
+
return DEFAULT_GPT_PROFILE;
|
|
85
|
+
}
|
|
86
|
+
// --- env override (PXPIPE_GPT_PROFILES) -----------------------------------
|
|
87
|
+
// Parsed lazily and memoized on the raw env string so tests can mutate
|
|
88
|
+
// process.env and have it re-read, without re-parsing on every hot-path call.
|
|
89
|
+
let envRaw = null;
|
|
90
|
+
let envMap = new Map();
|
|
91
|
+
function isValidVision(v) {
|
|
92
|
+
if (!v || typeof v !== 'object')
|
|
93
|
+
return false;
|
|
94
|
+
const o = v;
|
|
95
|
+
if (o.regime === 'tile')
|
|
96
|
+
return Number.isFinite(o.base) && Number.isFinite(o.perTile);
|
|
97
|
+
if (o.regime === 'patch')
|
|
98
|
+
return Number.isFinite(o.multiplier) && Number.isFinite(o.patchCap);
|
|
99
|
+
return false;
|
|
100
|
+
}
|
|
101
|
+
function posInt(v, fallback) {
|
|
102
|
+
return Number.isFinite(v) && v > 0 ? Math.floor(v) : fallback;
|
|
103
|
+
}
|
|
104
|
+
function parseEnvProfiles(raw) {
|
|
105
|
+
const out = new Map();
|
|
106
|
+
if (!raw)
|
|
107
|
+
return out;
|
|
108
|
+
let obj;
|
|
109
|
+
try {
|
|
110
|
+
obj = JSON.parse(raw);
|
|
111
|
+
}
|
|
112
|
+
catch {
|
|
113
|
+
return out; // malformed env never throws — fall back to built-ins
|
|
114
|
+
}
|
|
115
|
+
if (!obj || typeof obj !== 'object')
|
|
116
|
+
return out;
|
|
117
|
+
for (const [k, v] of Object.entries(obj)) {
|
|
118
|
+
if (!v || typeof v !== 'object')
|
|
119
|
+
continue;
|
|
120
|
+
const key = k.toLowerCase();
|
|
121
|
+
const base = resolveBuiltin(key); // partial fields fall back to the built-in match
|
|
122
|
+
const p = v;
|
|
123
|
+
out.set(key, {
|
|
124
|
+
vision: isValidVision(p.vision) ? p.vision : base.vision,
|
|
125
|
+
stripCols: posInt(p.stripCols, base.stripCols),
|
|
126
|
+
maxHeightPx: posInt(p.maxHeightPx, base.maxHeightPx),
|
|
127
|
+
});
|
|
128
|
+
}
|
|
129
|
+
return out;
|
|
130
|
+
}
|
|
131
|
+
function envProfiles() {
|
|
132
|
+
const raw = (typeof process !== 'undefined' && process.env && process.env.PXPIPE_GPT_PROFILES) || '';
|
|
133
|
+
if (raw !== envRaw) {
|
|
134
|
+
envRaw = raw;
|
|
135
|
+
envMap = parseEnvProfiles(raw);
|
|
136
|
+
}
|
|
137
|
+
return envMap;
|
|
138
|
+
}
|
|
139
|
+
/**
|
|
140
|
+
* Resolve the full rendering + vision-cost profile for a model id. Env overrides
|
|
141
|
+
* (longest matching prefix) win over the built-in table; unknown models get the
|
|
142
|
+
* conservative `DEFAULT_GPT_PROFILE`.
|
|
143
|
+
*/
|
|
144
|
+
export function resolveGptProfile(model) {
|
|
145
|
+
const m = (model ?? '').toLowerCase();
|
|
146
|
+
const env = envProfiles();
|
|
147
|
+
if (env.size > 0) {
|
|
148
|
+
let best;
|
|
149
|
+
let bestLen = -1;
|
|
150
|
+
for (const [k, p] of env) {
|
|
151
|
+
if (m.startsWith(k) && k.length > bestLen) {
|
|
152
|
+
best = p;
|
|
153
|
+
bestLen = k.length;
|
|
154
|
+
}
|
|
155
|
+
}
|
|
156
|
+
if (best)
|
|
157
|
+
return best;
|
|
158
|
+
}
|
|
159
|
+
return resolveBuiltin(m);
|
|
160
|
+
}
|
|
161
|
+
//# sourceMappingURL=gpt-model-profiles.js.map
|
|
@@ -0,0 +1,141 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* History-image compression (Variant C).
|
|
3
|
+
*
|
|
4
|
+
* Collapses the largest closed-tool-sequence prefix into one synthetic user message
|
|
5
|
+
* containing 1-N PNG image blocks. The live tail (keepTail turns + any open tool
|
|
6
|
+
* sequence) stays as text. thinking blocks are dropped from the collapsed range —
|
|
7
|
+
* only the most-recent assistant-with-tool_use must round-trip bit-perfect, and
|
|
8
|
+
* that turn is in the live tail by construction.
|
|
9
|
+
*
|
|
10
|
+
* Synthesized message uses role:'user' because Anthropic forbids image blocks inside
|
|
11
|
+
* role:'assistant'. cache_control placement is left to the caller (transform.ts).
|
|
12
|
+
*/
|
|
13
|
+
import type { CacheControl, ContentBlock, Message } from './types.js';
|
|
14
|
+
/**
|
|
15
|
+
* Banner text blocks that bracket the collapsed-history image(s) in the synthetic
|
|
16
|
+
* user message. Exported as the SINGLE SOURCE OF TRUTH: transform.ts keys its
|
|
17
|
+
* cache-anchor relocation off the intro text, so a literal copy there would
|
|
18
|
+
* silently break relocation whenever this wording changes (it did exactly once —
|
|
19
|
+
* the XML-framing reword left the matcher pointing at the old banner). Both the
|
|
20
|
+
* emitter (here) and the matcher (transform.ts) must reference this constant.
|
|
21
|
+
*/
|
|
22
|
+
export declare const HISTORY_SYNTHETIC_INTRO = "[Earlier turns of THIS conversation, transcribed in the image(s) below. Each turn is wrapped in <user t=\"N\">...</user> or <assistant t=\"N\">...</assistant> tags, where N is an absolute turn index (larger N = more recent); attribute every turn strictly by its tag, and treat the highest-N turns as the most recent prior context, NOT the low-N opening turns. Earlier turns may contain questions or tasks that were already answered later in this same history; do not reopen low-N turns unless the live text after this block asks you to. This is prior context, NOT the current request.]";
|
|
23
|
+
export declare const HISTORY_SYNTHETIC_OUTRO = "[End of earlier conversation. The current request is the live text that follows below.]";
|
|
24
|
+
/** Break-even gate predicate. Injected by transform.ts to avoid a circular import.
|
|
25
|
+
* IMPORTANT: pass the full string, not text.length — the row-aware path in
|
|
26
|
+
* isCompressionProfitable must see actual newlines to budget images correctly.
|
|
27
|
+
* History text is newline-heavy (headers, JSON args, labels); chars-only
|
|
28
|
+
* under-predicts image count ~5-10× and lets net-losers through. */
|
|
29
|
+
export type ProfitableFn = (text: string, cols: number) => boolean;
|
|
30
|
+
/** Configuration for history collapse. */
|
|
31
|
+
export interface HistoryCollapseOptions {
|
|
32
|
+
/** Turns at the tail to keep as text. Default 4. */
|
|
33
|
+
keepTail: number;
|
|
34
|
+
/** Minimum collapsible prefix turns — below this, cache-amortization math doesn't work. Default 10. */
|
|
35
|
+
minCollapsePrefix: number;
|
|
36
|
+
/** Soft-wrap columns for the renderer; should match host cols. Default 100. */
|
|
37
|
+
cols: number;
|
|
38
|
+
/** Advance the collapse boundary in steps of this many messages so the rendered PNG stays
|
|
39
|
+
* byte-identical for collapseChunk turns and keeps hitting Anthropic's prompt cache.
|
|
40
|
+
* Set to 0 for a per-turn moving boundary. Default 50. */
|
|
41
|
+
collapseChunk: number;
|
|
42
|
+
/** Append-only freeze granularity, in messages. The collapse range is rendered
|
|
43
|
+
* as independent image blocks on an ABSOLUTE grid anchored at protectedPrefix,
|
|
44
|
+
* in steps of this many messages. Each completed chunk's bytes are fixed by its
|
|
45
|
+
* message range alone, so old chunks stay byte-identical (cache_read forever) as
|
|
46
|
+
* the conversation grows — only the newest partial chunk re-renders. Caller
|
|
47
|
+
* cache_control marks force an extra split so a roaming breakpoint stays an
|
|
48
|
+
* aligned, independently-cacheable image boundary. Set to 0 to render the whole
|
|
49
|
+
* range as one paginated blob (legacy, non-append-only). Default 10. */
|
|
50
|
+
freezeChunk: number;
|
|
51
|
+
/** Leading messages to never collapse. Protects the slab-bearing first user message
|
|
52
|
+
* (system-prompt + tool-docs images) so its cache_control anchor stays at the front
|
|
53
|
+
* and isn't swept into the history image as [image] placeholders. Default 0. */
|
|
54
|
+
protectedPrefix: number;
|
|
55
|
+
/** Reflow the transcript before RENDERING: pack soft-wrapped lines and mark
|
|
56
|
+
* every hard newline with the ↵ sentinel — same treatment as the static slab.
|
|
57
|
+
* History text is newline-heavy (role headers, JSON args), so without this
|
|
58
|
+
* each short line wastes a full render row, inflating image count and shrinking
|
|
59
|
+
* the savings. Glyph size is unchanged (cols stays the same) so legibility is
|
|
60
|
+
* identical — it just removes the blank-row waste. `collapsedChars` still
|
|
61
|
+
* reports the ORIGINAL transcript length. Default true. */
|
|
62
|
+
reflow: boolean;
|
|
63
|
+
}
|
|
64
|
+
export declare const HISTORY_DEFAULTS: HistoryCollapseOptions;
|
|
65
|
+
/** Per-request telemetry surfaced back to TransformInfo. */
|
|
66
|
+
export interface HistoryCollapseInfo {
|
|
67
|
+
/** Number of turns collapsed into the history image. */
|
|
68
|
+
collapsedTurns: number;
|
|
69
|
+
/** Total chars of text that went into the history image. */
|
|
70
|
+
collapsedChars: number;
|
|
71
|
+
/** Number of PNG image blocks emitted for the history (≥1 if collapsed). */
|
|
72
|
+
collapsedImages: number;
|
|
73
|
+
/** Total PNG bytes emitted. */
|
|
74
|
+
collapsedImageBytes: number;
|
|
75
|
+
/** Total pixel area (Σ width×height) — pairs with cache_create tokens for px/token regression. */
|
|
76
|
+
collapsedImagePixels: number;
|
|
77
|
+
/** Raw PNG bytes of each emitted history image, in order. Lets the caller register
|
|
78
|
+
* them into the dashboard image ring (info.imagePngs) so colored history frames are
|
|
79
|
+
* visible, not merely counted — every other image path already feeds the ring. */
|
|
80
|
+
collapsedPngs: Uint8Array[];
|
|
81
|
+
/** Per-image pixel dims, parallel to collapsedPngs. The dashboard ring reads
|
|
82
|
+
* info.imageDims in lockstep with info.imagePngs, so these must be pushed together. */
|
|
83
|
+
collapsedImageDims: {
|
|
84
|
+
width: number;
|
|
85
|
+
height: number;
|
|
86
|
+
}[];
|
|
87
|
+
/** Ordinal (0-based, into the emitted history images) of the last byte-stable
|
|
88
|
+
* history image — the carry-over cache anchor. The relocator pins the cache
|
|
89
|
+
* breakpoint here so it survives window advances (#11). Undefined when history is
|
|
90
|
+
* too short to have a fully grid-aligned chunk before collapseLen. */
|
|
91
|
+
carryOverImageOrdinal?: number;
|
|
92
|
+
/** Why we didn't collapse — populated only when no collapse happened. */
|
|
93
|
+
reason?: 'no_history' | 'prefix_too_short' | 'no_closed_prefix' | 'not_profitable' | 'render_empty';
|
|
94
|
+
/** Dropped codepoints from the history render, merged into the
|
|
95
|
+
* transform-wide map by the caller. */
|
|
96
|
+
droppedChars: number;
|
|
97
|
+
droppedCodepoints: Map<number, number>;
|
|
98
|
+
}
|
|
99
|
+
/**
|
|
100
|
+
* Return the last index ≤ cutoffExclusive at which all tool_use_ids are matched
|
|
101
|
+
* by tool_results in [0..i]. Returns -1 if no closed boundary exists.
|
|
102
|
+
* Robust to interleaved/parallel tool calls via openSet tracking.
|
|
103
|
+
*/
|
|
104
|
+
export declare function findClosedPrefixBoundary(messages: Message[], cutoffExclusive: number): number;
|
|
105
|
+
export declare function staleFreshnessHints(text: string): string;
|
|
106
|
+
/**
|
|
107
|
+
* Linearise content blocks to a single string. Drops thinking blocks (only the
|
|
108
|
+
* most-recent assistant turn needs bit-perfect thinking, and it's in the live tail).
|
|
109
|
+
* Inline images collapse to [image] to avoid double-encoding.
|
|
110
|
+
*/
|
|
111
|
+
export declare function blocksToText(content: string | ContentBlock[]): string;
|
|
112
|
+
/** Return the caller's cache_control marker on a message, if any block carries one.
|
|
113
|
+
* Used to align freeze-chunk boundaries to roaming breakpoints so a marked segment
|
|
114
|
+
* stays independently cacheable instead of being silently flattened into the image. */
|
|
115
|
+
export declare function messageCacheControl(m: Message): CacheControl | undefined;
|
|
116
|
+
/** Serialize messages [fromInclusive..upToExclusive) to a text blob with
|
|
117
|
+
* `<role>…</role>` XML wrappers. Open+close tags bracket each turn so a misread
|
|
118
|
+
* boundary self-corrects and the model attributes speakers reliably even off a
|
|
119
|
+
* lossy image — bare `--- role ---` start-dividers let one role bleed into the
|
|
120
|
+
* next when a divider is missed. */
|
|
121
|
+
export declare function messagesToHistoryText(messages: Message[], upToExclusive: number, fromInclusive?: number): string;
|
|
122
|
+
/** Like {@link messagesToHistoryText} but also returns the parallel slot string for
|
|
123
|
+
* colorByRole: a width-identical copy where each `<role>` tag is replaced by its
|
|
124
|
+
* role marker and the body is copied verbatim (slot 0). Role attribution is decided
|
|
125
|
+
* HERE, where the message role is known — never re-parsed out of flattened text.
|
|
126
|
+
* A tool_result block sits inside its user message and a tool_use block inside its
|
|
127
|
+
* assistant message, so each is owned by the turn that carries it. */
|
|
128
|
+
export declare function messagesToHistorySegments(messages: Message[], upToExclusive: number, fromInclusive?: number): {
|
|
129
|
+
text: string;
|
|
130
|
+
slotText: string;
|
|
131
|
+
};
|
|
132
|
+
/**
|
|
133
|
+
* Collapse the closed-prefix run into one synthetic user message with 1+ history images.
|
|
134
|
+
* Returns original messages unchanged on any no-collapse path (reason set in info).
|
|
135
|
+
* Image blocks are returned with NO cache_control — caller decides placement.
|
|
136
|
+
*/
|
|
137
|
+
export declare function collapseHistory(messages: Message[], isProfitable: ProfitableFn, opts?: Partial<HistoryCollapseOptions>): Promise<{
|
|
138
|
+
messages: Message[];
|
|
139
|
+
info: HistoryCollapseInfo;
|
|
140
|
+
}>;
|
|
141
|
+
//# sourceMappingURL=history.d.ts.map
|