@xamukavila/pxpipe 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +21 -0
- package/README.md +312 -0
- package/bin/cli.js +7 -0
- package/dist/core/applicability.d.ts +31 -0
- package/dist/core/applicability.js +96 -0
- package/dist/core/atlas-gray.d.ts +26 -0
- package/dist/core/atlas-gray.js +64 -0
- package/dist/core/atlas.d.ts +34 -0
- package/dist/core/atlas.js +71 -0
- package/dist/core/baseline.d.ts +80 -0
- package/dist/core/baseline.js +101 -0
- package/dist/core/caveman.d.ts +38 -0
- package/dist/core/caveman.js +183 -0
- package/dist/core/export.d.ts +128 -0
- package/dist/core/export.js +390 -0
- package/dist/core/factsheet.d.ts +78 -0
- package/dist/core/factsheet.js +216 -0
- package/dist/core/gpt-model-profiles.d.ts +60 -0
- package/dist/core/gpt-model-profiles.js +161 -0
- package/dist/core/history.d.ts +141 -0
- package/dist/core/history.js +553 -0
- package/dist/core/index.d.ts +8 -0
- package/dist/core/index.js +8 -0
- package/dist/core/library.d.ts +74 -0
- package/dist/core/library.js +133 -0
- package/dist/core/measurement.d.ts +22 -0
- package/dist/core/measurement.js +213 -0
- package/dist/core/openai-history.d.ts +124 -0
- package/dist/core/openai-history.js +494 -0
- package/dist/core/openai-savings.d.ts +44 -0
- package/dist/core/openai-savings.js +75 -0
- package/dist/core/openai.d.ts +24 -0
- package/dist/core/openai.js +839 -0
- package/dist/core/png.d.ts +11 -0
- package/dist/core/png.js +132 -0
- package/dist/core/proxy.d.ts +81 -0
- package/dist/core/proxy.js +730 -0
- package/dist/core/render.d.ts +188 -0
- package/dist/core/render.js +785 -0
- package/dist/core/schema-strip.d.ts +29 -0
- package/dist/core/schema-strip.js +160 -0
- package/dist/core/tracker.d.ts +154 -0
- package/dist/core/tracker.js +216 -0
- package/dist/core/transform.d.ts +362 -0
- package/dist/core/transform.js +1828 -0
- package/dist/core/types.d.ts +77 -0
- package/dist/core/types.js +8 -0
- package/dist/dashboard/fragments.d.ts +36 -0
- package/dist/dashboard/fragments.js +938 -0
- package/dist/dashboard/types.d.ts +154 -0
- package/dist/dashboard/types.js +3 -0
- package/dist/dashboard/vendor.d.ts +3 -0
- package/dist/dashboard/vendor.js +6 -0
- package/dist/dashboard.d.ts +245 -0
- package/dist/dashboard.js +1140 -0
- package/dist/export-collect.d.ts +36 -0
- package/dist/export-collect.js +59 -0
- package/dist/node.d.ts +9 -0
- package/dist/node.js +9038 -0
- package/dist/sessions.d.ts +172 -0
- package/dist/sessions.js +510 -0
- package/dist/stats.d.ts +74 -0
- package/dist/stats.js +248 -0
- package/dist/worker.d.ts +53 -0
- package/dist/worker.js +102 -0
- package/package.json +96 -0
|
@@ -0,0 +1,80 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache-aware baseline math for the unproxied counterfactual.
|
|
3
|
+
* Workers-safe: no node:, no Buffer, no process.*. Pure number math.
|
|
4
|
+
* See docs/CACHING_AND_SAVINGS.md for the full derivation and audit history.
|
|
5
|
+
*/
|
|
6
|
+
/** Documented Anthropic price ratios: cc_5m = 1.25×, cr = 0.1× base input. One-line change if rates change. */
|
|
7
|
+
export declare const CACHE_CREATE_RATE = 1.25;
|
|
8
|
+
export declare const CACHE_READ_RATE = 0.1;
|
|
9
|
+
/** Anthropic prompt-cache TTL (seconds). Kept for callers that display provider
|
|
10
|
+
* docs, but savings math does not use TTL to infer a hypothetical text-cache
|
|
11
|
+
* hit: text is considered warm only when the actual request reports cr > 0. */
|
|
12
|
+
export declare const CACHE_TTL_SEC = 300;
|
|
13
|
+
/** This session's previous usage-bearing turn, used only for warm split sizing. */
|
|
14
|
+
export interface BaselineWarmthPrev {
|
|
15
|
+
/** Completion time of that turn, in wall-clock seconds. */
|
|
16
|
+
ts: number;
|
|
17
|
+
/** Cacheable-prefix tokens measured that turn (0 if the probe missed). */
|
|
18
|
+
cacheable: number;
|
|
19
|
+
/** Hash of the image-bound/static text prefix. If it changes, do not reuse the
|
|
20
|
+
* prior prefix size for this row's text reused/grown split. */
|
|
21
|
+
prefixSha?: string;
|
|
22
|
+
}
|
|
23
|
+
/**
|
|
24
|
+
* Decide whether the TEXT counterfactual's prefix was warm this turn.
|
|
25
|
+
*
|
|
26
|
+
* Strict accounting rule: the imagined text path gets the same observed cache
|
|
27
|
+
* state as the real image path. `cr > 0` is server proof that the request read a
|
|
28
|
+
* warm prefix, so the text baseline is warm too. `cr === 0` means the actual
|
|
29
|
+
* request did not read cache, so the text baseline is priced cold too. We do not
|
|
30
|
+
* use wall-clock TTL to claim that text would have been warm while images were
|
|
31
|
+
* cold; that would be an unobservable counterfactual and can create negative
|
|
32
|
+
* rows from cache assumptions rather than token savings.
|
|
33
|
+
*
|
|
34
|
+
* When cr proves warmth, a completed same-prefix prior is used only to estimate
|
|
35
|
+
* how much of the text prefix was reused vs grown. If none is available, assume
|
|
36
|
+
* full reuse of this turn's cacheable prefix; this is conservative for savings.
|
|
37
|
+
*
|
|
38
|
+
* @param prev this session's previous usage-bearing turn, or undefined.
|
|
39
|
+
* @param nowSec request-start wall-clock seconds, used only to reject prior
|
|
40
|
+
* rows that had not completed before this request was sent.
|
|
41
|
+
* @param cacheable this turn's cacheable-prefix tokens (the full-reuse credit
|
|
42
|
+
* when warm only via cr, since cr proves a read but not the split).
|
|
43
|
+
* @param cr observed cache-read tokens this turn; the only warm/cold signal.
|
|
44
|
+
* @param ttlSec legacy parameter; no longer decides warm/cold. It only
|
|
45
|
+
* bounds whether a prior prefix size is used for reused/grown
|
|
46
|
+
* splitting after cr > 0 has already proved warmth.
|
|
47
|
+
* @param prefixSha stable-prefix fingerprint for the text counterfactual. A
|
|
48
|
+
* prior prefix size is reused only when this matches.
|
|
49
|
+
*/
|
|
50
|
+
export declare function deriveBaselineWarmth(prev: BaselineWarmthPrev | undefined, nowSec: number, cacheable: number, cr: number, ttlSec?: number, prefixSha?: string): {
|
|
51
|
+
warm: boolean;
|
|
52
|
+
prevCacheable: number;
|
|
53
|
+
};
|
|
54
|
+
/**
|
|
55
|
+
* Weighted input cost for the unproxied TEXT counterfactual (see docs/CACHING_AND_SAVINGS.md).
|
|
56
|
+
*
|
|
57
|
+
* Warmth matters: a TEXT prefix is only a cheap cache-read when a warm cache
|
|
58
|
+
* actually existed this turn. The previous warmth-FREE version always priced
|
|
59
|
+
* the cacheable prefix at CACHE_READ_RATE, which fabricated a "free read" on
|
|
60
|
+
* cold/TTL-expiry turns where text would in fact have paid a 1.25× create —
|
|
61
|
+
* that produced a phantom loss vs the imaged path (which DOES pay the create).
|
|
62
|
+
*
|
|
63
|
+
* cold turn (first turn / >5min since this session's last turn):
|
|
64
|
+
* text has no warm cache either ⇒ cacheable×CACHE_CREATE_RATE + coldTail×1.0
|
|
65
|
+
* warm turn (a prior turn cached the prefix within TTL):
|
|
66
|
+
* text append-caches ⇒ reused×CACHE_READ_RATE + grown×CACHE_CREATE_RATE + coldTail×1.0
|
|
67
|
+
* where reused = min(prevCacheable, cacheable), grown = cacheable − reused.
|
|
68
|
+
* This is what TEXT pays regardless of whether pxpipe's image busted its
|
|
69
|
+
* own cache on a growth turn — so the real growth loss is preserved.
|
|
70
|
+
*
|
|
71
|
+
* Saving = baseline_eff − actual_eff; can be negative (honestly reported, not floored).
|
|
72
|
+
*
|
|
73
|
+
* @param baselineCacheable tokens up to the last cache_control marker. ≤0 ⇒ credit nothing.
|
|
74
|
+
* @param warm was a warm cache available for this session this turn?
|
|
75
|
+
* @param prevCacheable cacheable prefix size on this session's previous turn (warm only).
|
|
76
|
+
*/
|
|
77
|
+
export declare function computeBaselineInputEff(baseline: number, baselineCacheable: number, inputTokens: number, cc: number, cr: number, warm?: boolean, prevCacheable?: number): number;
|
|
78
|
+
/** Weighted input cost pxpipe actually paid this turn. */
|
|
79
|
+
export declare function computeActualInputEff(inputTokens: number, cc: number, cr: number): number;
|
|
80
|
+
//# sourceMappingURL=baseline.d.ts.map
|
|
@@ -0,0 +1,101 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Cache-aware baseline math for the unproxied counterfactual.
|
|
3
|
+
* Workers-safe: no node:, no Buffer, no process.*. Pure number math.
|
|
4
|
+
* See docs/CACHING_AND_SAVINGS.md for the full derivation and audit history.
|
|
5
|
+
*/
|
|
6
|
+
/** Documented Anthropic price ratios: cc_5m = 1.25×, cr = 0.1× base input. One-line change if rates change. */
|
|
7
|
+
export const CACHE_CREATE_RATE = 1.25;
|
|
8
|
+
export const CACHE_READ_RATE = 0.1;
|
|
9
|
+
/** Anthropic prompt-cache TTL (seconds). Kept for callers that display provider
|
|
10
|
+
* docs, but savings math does not use TTL to infer a hypothetical text-cache
|
|
11
|
+
* hit: text is considered warm only when the actual request reports cr > 0. */
|
|
12
|
+
export const CACHE_TTL_SEC = 300;
|
|
13
|
+
/**
|
|
14
|
+
* Decide whether the TEXT counterfactual's prefix was warm this turn.
|
|
15
|
+
*
|
|
16
|
+
* Strict accounting rule: the imagined text path gets the same observed cache
|
|
17
|
+
* state as the real image path. `cr > 0` is server proof that the request read a
|
|
18
|
+
* warm prefix, so the text baseline is warm too. `cr === 0` means the actual
|
|
19
|
+
* request did not read cache, so the text baseline is priced cold too. We do not
|
|
20
|
+
* use wall-clock TTL to claim that text would have been warm while images were
|
|
21
|
+
* cold; that would be an unobservable counterfactual and can create negative
|
|
22
|
+
* rows from cache assumptions rather than token savings.
|
|
23
|
+
*
|
|
24
|
+
* When cr proves warmth, a completed same-prefix prior is used only to estimate
|
|
25
|
+
* how much of the text prefix was reused vs grown. If none is available, assume
|
|
26
|
+
* full reuse of this turn's cacheable prefix; this is conservative for savings.
|
|
27
|
+
*
|
|
28
|
+
* @param prev this session's previous usage-bearing turn, or undefined.
|
|
29
|
+
* @param nowSec request-start wall-clock seconds, used only to reject prior
|
|
30
|
+
* rows that had not completed before this request was sent.
|
|
31
|
+
* @param cacheable this turn's cacheable-prefix tokens (the full-reuse credit
|
|
32
|
+
* when warm only via cr, since cr proves a read but not the split).
|
|
33
|
+
* @param cr observed cache-read tokens this turn; the only warm/cold signal.
|
|
34
|
+
* @param ttlSec legacy parameter; no longer decides warm/cold. It only
|
|
35
|
+
* bounds whether a prior prefix size is used for reused/grown
|
|
36
|
+
* splitting after cr > 0 has already proved warmth.
|
|
37
|
+
* @param prefixSha stable-prefix fingerprint for the text counterfactual. A
|
|
38
|
+
* prior prefix size is reused only when this matches.
|
|
39
|
+
*/
|
|
40
|
+
export function deriveBaselineWarmth(prev, nowSec, cacheable, cr, ttlSec = CACHE_TTL_SEC, prefixSha) {
|
|
41
|
+
const age = prev !== undefined ? nowSec - prev.ts : Number.POSITIVE_INFINITY;
|
|
42
|
+
const samePrefix = prev === undefined
|
|
43
|
+
|| prev.prefixSha === undefined
|
|
44
|
+
|| prefixSha === undefined
|
|
45
|
+
|| prev.prefixSha === prefixSha;
|
|
46
|
+
// cr is the only warm/cold signal. A prior only refines the warm split.
|
|
47
|
+
if (!(cr > 0))
|
|
48
|
+
return { warm: false, prevCacheable: 0 };
|
|
49
|
+
// Fresh prior: use its real prefix size for the reused/grown split. Without
|
|
50
|
+
// one, cr proves warmth but not the split, so assume full reuse.
|
|
51
|
+
const freshPrior = prev !== undefined && age >= 0 && age < ttlSec && samePrefix;
|
|
52
|
+
return { warm: true, prevCacheable: freshPrior ? prev.cacheable : cacheable };
|
|
53
|
+
}
|
|
54
|
+
/**
|
|
55
|
+
* Weighted input cost for the unproxied TEXT counterfactual (see docs/CACHING_AND_SAVINGS.md).
|
|
56
|
+
*
|
|
57
|
+
* Warmth matters: a TEXT prefix is only a cheap cache-read when a warm cache
|
|
58
|
+
* actually existed this turn. The previous warmth-FREE version always priced
|
|
59
|
+
* the cacheable prefix at CACHE_READ_RATE, which fabricated a "free read" on
|
|
60
|
+
* cold/TTL-expiry turns where text would in fact have paid a 1.25× create —
|
|
61
|
+
* that produced a phantom loss vs the imaged path (which DOES pay the create).
|
|
62
|
+
*
|
|
63
|
+
* cold turn (first turn / >5min since this session's last turn):
|
|
64
|
+
* text has no warm cache either ⇒ cacheable×CACHE_CREATE_RATE + coldTail×1.0
|
|
65
|
+
* warm turn (a prior turn cached the prefix within TTL):
|
|
66
|
+
* text append-caches ⇒ reused×CACHE_READ_RATE + grown×CACHE_CREATE_RATE + coldTail×1.0
|
|
67
|
+
* where reused = min(prevCacheable, cacheable), grown = cacheable − reused.
|
|
68
|
+
* This is what TEXT pays regardless of whether pxpipe's image busted its
|
|
69
|
+
* own cache on a growth turn — so the real growth loss is preserved.
|
|
70
|
+
*
|
|
71
|
+
* Saving = baseline_eff − actual_eff; can be negative (honestly reported, not floored).
|
|
72
|
+
*
|
|
73
|
+
* @param baselineCacheable tokens up to the last cache_control marker. ≤0 ⇒ credit nothing.
|
|
74
|
+
* @param warm was a warm cache available for this session this turn?
|
|
75
|
+
* @param prevCacheable cacheable prefix size on this session's previous turn (warm only).
|
|
76
|
+
*/
|
|
77
|
+
export function computeBaselineInputEff(baseline, baselineCacheable, inputTokens, cc, cr, warm = false, prevCacheable = 0) {
|
|
78
|
+
if (baseline <= 0)
|
|
79
|
+
return 0;
|
|
80
|
+
// Probe miss: can't split prefix from tail, so credit nothing (same as actual).
|
|
81
|
+
if (baselineCacheable <= 0)
|
|
82
|
+
return computeActualInputEff(inputTokens, cc, cr);
|
|
83
|
+
const cacheable = Math.min(baselineCacheable, baseline);
|
|
84
|
+
const coldTail = baseline - cacheable;
|
|
85
|
+
if (warm) {
|
|
86
|
+
// Text reads the prefix it already had cached (0.10×) and creates only the
|
|
87
|
+
// growth since last turn (1.25×). Independent of the image path's cache.
|
|
88
|
+
const reused = Math.min(Math.max(prevCacheable, 0), cacheable);
|
|
89
|
+
const grown = cacheable - reused;
|
|
90
|
+
return reused * CACHE_READ_RATE + grown * CACHE_CREATE_RATE + coldTail * 1.0;
|
|
91
|
+
}
|
|
92
|
+
// Cold (first turn / TTL expiry): no warm cache for text either, so it
|
|
93
|
+
// re-creates the whole cacheable prefix at the create rate — same event the
|
|
94
|
+
// imaged path pays. Removes the phantom "free read" that fabricated a loss.
|
|
95
|
+
return cacheable * CACHE_CREATE_RATE + coldTail * 1.0;
|
|
96
|
+
}
|
|
97
|
+
/** Weighted input cost pxpipe actually paid this turn. */
|
|
98
|
+
export function computeActualInputEff(inputTokens, cc, cr) {
|
|
99
|
+
return inputTokens + cc * CACHE_CREATE_RATE + cr * CACHE_READ_RATE;
|
|
100
|
+
}
|
|
101
|
+
//# sourceMappingURL=baseline.js.map
|
|
@@ -0,0 +1,38 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caveman pass — deterministic, rule-based prose compression for text bound
|
|
3
|
+
* to the optical (PNG) channel.
|
|
4
|
+
*
|
|
5
|
+
* Why: image cost is pixel-area billing (`width × height / 750`), and pixel
|
|
6
|
+
* area at fixed density is proportional to char count — so every char dropped
|
|
7
|
+
* BEFORE rendering compounds with the render savings. Articles, filler
|
|
8
|
+
* adverbs, and politeness phrases carry near-zero gist, which is the only
|
|
9
|
+
* fidelity level the optical channel promises anyway (FINDINGS.md: pxpipe is
|
|
10
|
+
* a lossy gist-compressor).
|
|
11
|
+
*
|
|
12
|
+
* The trade-off (FINDINGS.md capacity argument): natural-language redundancy
|
|
13
|
+
* is the error-correcting code of the lossy optical read — telegraphic text
|
|
14
|
+
* gives the language prior less signal to repair under-resolved glyphs. This
|
|
15
|
+
* pass is therefore an opt-in EXPERIMENT (`PXPIPE_CAVEMAN=1`) until the A/B
|
|
16
|
+
* harness shows gist/verbatim recall does not regress.
|
|
17
|
+
*
|
|
18
|
+
* Hard requirements:
|
|
19
|
+
* - DETERMINISTIC and idempotent. The slab image bytes are the prompt-cache
|
|
20
|
+
* key; any nondeterminism here busts the cache every turn. No LLM, no
|
|
21
|
+
* randomness, no locale/config dependence.
|
|
22
|
+
* - VERBATIM-SAFE. Never touches: fenced/indented code, markdown table
|
|
23
|
+
* rows, inline `code` spans, double-quoted spans, or any token that is
|
|
24
|
+
* not purely alphabetic — ids, hashes, paths, URLs, env vars, numbers,
|
|
25
|
+
* ALL-CAPS and CamelCase tokens all pass through untouched. Only whole
|
|
26
|
+
* lowercase (or Capitalized) words from the curated EN/PT lists drop.
|
|
27
|
+
*/
|
|
28
|
+
/**
|
|
29
|
+
* Deterministically strip low-information words from prose. Code (fenced,
|
|
30
|
+
* indented, inline), table rows, quoted spans, and every non-alphabetic
|
|
31
|
+
* token pass through byte-exact. Line structure is preserved (no line merges
|
|
32
|
+
* or splits) so renderer row accounting and paging stay comparable.
|
|
33
|
+
*
|
|
34
|
+
* Callers gate on `classifyContent(text) === 'other'` — running this over
|
|
35
|
+
* JSON/logs would be a no-op waste of CPU at best.
|
|
36
|
+
*/
|
|
37
|
+
export declare function cavemanize(text: string): string;
|
|
38
|
+
//# sourceMappingURL=caveman.d.ts.map
|
|
@@ -0,0 +1,183 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Caveman pass — deterministic, rule-based prose compression for text bound
|
|
3
|
+
* to the optical (PNG) channel.
|
|
4
|
+
*
|
|
5
|
+
* Why: image cost is pixel-area billing (`width × height / 750`), and pixel
|
|
6
|
+
* area at fixed density is proportional to char count — so every char dropped
|
|
7
|
+
* BEFORE rendering compounds with the render savings. Articles, filler
|
|
8
|
+
* adverbs, and politeness phrases carry near-zero gist, which is the only
|
|
9
|
+
* fidelity level the optical channel promises anyway (FINDINGS.md: pxpipe is
|
|
10
|
+
* a lossy gist-compressor).
|
|
11
|
+
*
|
|
12
|
+
* The trade-off (FINDINGS.md capacity argument): natural-language redundancy
|
|
13
|
+
* is the error-correcting code of the lossy optical read — telegraphic text
|
|
14
|
+
* gives the language prior less signal to repair under-resolved glyphs. This
|
|
15
|
+
* pass is therefore an opt-in EXPERIMENT (`PXPIPE_CAVEMAN=1`) until the A/B
|
|
16
|
+
* harness shows gist/verbatim recall does not regress.
|
|
17
|
+
*
|
|
18
|
+
* Hard requirements:
|
|
19
|
+
* - DETERMINISTIC and idempotent. The slab image bytes are the prompt-cache
|
|
20
|
+
* key; any nondeterminism here busts the cache every turn. No LLM, no
|
|
21
|
+
* randomness, no locale/config dependence.
|
|
22
|
+
* - VERBATIM-SAFE. Never touches: fenced/indented code, markdown table
|
|
23
|
+
* rows, inline `code` spans, double-quoted spans, or any token that is
|
|
24
|
+
* not purely alphabetic — ids, hashes, paths, URLs, env vars, numbers,
|
|
25
|
+
* ALL-CAPS and CamelCase tokens all pass through untouched. Only whole
|
|
26
|
+
* lowercase (or Capitalized) words from the curated EN/PT lists drop.
|
|
27
|
+
*/
|
|
28
|
+
// --- word lists -------------------------------------------------------------
|
|
29
|
+
/** Multi-letter articles, EN + PT.
|
|
30
|
+
* Deliberately EXCLUDES 'as': it is a PT plural article but also a
|
|
31
|
+
* load-bearing EN conjunction ("treat as data") — dropping it inverts
|
|
32
|
+
* meaning in the EN prose that dominates real slabs. */
|
|
33
|
+
const ARTICLES = new Set([
|
|
34
|
+
// EN
|
|
35
|
+
'the',
|
|
36
|
+
'an',
|
|
37
|
+
// PT
|
|
38
|
+
'um',
|
|
39
|
+
'uma',
|
|
40
|
+
'uns',
|
|
41
|
+
'umas',
|
|
42
|
+
'os',
|
|
43
|
+
]);
|
|
44
|
+
/** Single-letter articles ('a' EN/PT, 'o' PT): drop only the exact-lowercase
|
|
45
|
+
* form. An isolated capital "A"/"O" is more likely an option label, list
|
|
46
|
+
* marker, or grade than an article. */
|
|
47
|
+
const SINGLE_LETTER_ARTICLES = new Set(['a', 'o']);
|
|
48
|
+
/** Intensity/filler adverbs and politeness words whose removal cannot invert
|
|
49
|
+
* meaning. Deliberately EXCLUDES hedges that flip polarity when removed
|
|
50
|
+
* ('quite': "not quite right" ≠ "not right") and quantifiers that carry
|
|
51
|
+
* real information ('only', 'just', 'muito'/'muitos'). */
|
|
52
|
+
const FILLERS = new Set([
|
|
53
|
+
// EN
|
|
54
|
+
'really',
|
|
55
|
+
'actually',
|
|
56
|
+
'basically',
|
|
57
|
+
'simply',
|
|
58
|
+
'essentially',
|
|
59
|
+
'certainly',
|
|
60
|
+
'definitely',
|
|
61
|
+
'obviously',
|
|
62
|
+
'literally',
|
|
63
|
+
'very',
|
|
64
|
+
'please',
|
|
65
|
+
'kindly',
|
|
66
|
+
// PT
|
|
67
|
+
'realmente',
|
|
68
|
+
'basicamente',
|
|
69
|
+
'simplesmente',
|
|
70
|
+
'essencialmente',
|
|
71
|
+
'certamente',
|
|
72
|
+
'definitivamente',
|
|
73
|
+
'obviamente',
|
|
74
|
+
'literalmente',
|
|
75
|
+
]);
|
|
76
|
+
/** Politeness/verbosity phrases rewritten before word drops. Applied only to
|
|
77
|
+
* unprotected prose segments; each replacement is idempotent (its output
|
|
78
|
+
* never re-matches its pattern). */
|
|
79
|
+
const PHRASES = [
|
|
80
|
+
[/\bplease note that\b/gi, ''],
|
|
81
|
+
[/\bnote that\b/gi, ''],
|
|
82
|
+
[/\bkeep in mind that\b/gi, ''],
|
|
83
|
+
[/\bin order to\b/gi, 'to'],
|
|
84
|
+
[/\bas well as\b/gi, 'and'],
|
|
85
|
+
[/\bmake sure that\b/gi, 'ensure'],
|
|
86
|
+
// PT
|
|
87
|
+
[/\bpor favor\b/gi, ''],
|
|
88
|
+
[/\ba fim de\b/gi, 'para'],
|
|
89
|
+
[/\bnote que\b/gi, ''],
|
|
90
|
+
[/\btenha em mente que\b/gi, ''],
|
|
91
|
+
];
|
|
92
|
+
// --- protections ------------------------------------------------------------
|
|
93
|
+
/** Fence opener/closer (``` or ~~~), optionally indented. */
|
|
94
|
+
const FENCE_LINE = /^\s*(```|~~~)/;
|
|
95
|
+
/** Markdown code by indentation (4 spaces or a tab). */
|
|
96
|
+
const INDENTED_CODE = /^(?: {4}|\t)/;
|
|
97
|
+
/** Markdown table row. */
|
|
98
|
+
const TABLE_ROW = /^\s*\|/;
|
|
99
|
+
/** Spans that must survive byte-exact even inside prose: inline `code`
|
|
100
|
+
* and double-quoted strings (quoted error messages get grepped verbatim).
|
|
101
|
+
* Single quotes are NOT protected — apostrophes ("don't") would open
|
|
102
|
+
* phantom spans. Capture group so `split` keeps the spans. */
|
|
103
|
+
const PROTECTED_SPAN = /(`[^`\n]+`|"[^"\n]*")/g;
|
|
104
|
+
/** Letters-only token (Unicode-aware: PT diacritics included). Anything with
|
|
105
|
+
* a digit, slash, dot, underscore, etc. never matches — that single check
|
|
106
|
+
* protects ids, paths, URLs, hashes, numbers, and env vars. */
|
|
107
|
+
const LETTERS_ONLY = /^\p{L}+$/u;
|
|
108
|
+
// --- core -------------------------------------------------------------------
|
|
109
|
+
function isDroppableWord(token) {
|
|
110
|
+
if (!LETTERS_ONLY.test(token))
|
|
111
|
+
return false;
|
|
112
|
+
const lower = token.toLowerCase();
|
|
113
|
+
const isLower = token === lower;
|
|
114
|
+
// Allow exact-lowercase and Capitalized ("The"); reject ALL-CAPS and
|
|
115
|
+
// mixed-case (PATH, iOS) — those read as identifiers, not grammar.
|
|
116
|
+
const isCapitalized = !isLower &&
|
|
117
|
+
token[0] === token[0].toUpperCase() &&
|
|
118
|
+
token.slice(1) === token.slice(1).toLowerCase();
|
|
119
|
+
if (!isLower && !isCapitalized)
|
|
120
|
+
return false;
|
|
121
|
+
if (ARTICLES.has(lower) || FILLERS.has(lower))
|
|
122
|
+
return true;
|
|
123
|
+
return isLower && SINGLE_LETTER_ARTICLES.has(lower);
|
|
124
|
+
}
|
|
125
|
+
/** Compress one unprotected prose segment. Preserves the segment's exact
|
|
126
|
+
* leading/trailing whitespace (it may butt up against a protected span);
|
|
127
|
+
* interior whitespace collapses to single spaces — acceptable for prose,
|
|
128
|
+
* and the slab path runs `compactSlabWhitespace` afterwards anyway. */
|
|
129
|
+
function compressSegment(seg) {
|
|
130
|
+
let s = seg;
|
|
131
|
+
for (const [re, sub] of PHRASES)
|
|
132
|
+
s = s.replace(re, sub);
|
|
133
|
+
const lead = s.match(/^\s*/)[0];
|
|
134
|
+
if (lead.length === s.length)
|
|
135
|
+
return s; // empty or all-whitespace
|
|
136
|
+
const trail = s.match(/\s*$/)[0];
|
|
137
|
+
const body = s.slice(lead.length, s.length - trail.length);
|
|
138
|
+
const words = body.split(/\s+/).filter((w) => w.length > 0 && !isDroppableWord(w));
|
|
139
|
+
return lead + words.join(' ') + trail;
|
|
140
|
+
}
|
|
141
|
+
function compressLine(line) {
|
|
142
|
+
if (INDENTED_CODE.test(line) || TABLE_ROW.test(line))
|
|
143
|
+
return line;
|
|
144
|
+
if (!/\p{L}/u.test(line))
|
|
145
|
+
return line; // nothing droppable, skip regex work
|
|
146
|
+
const parts = line.split(PROTECTED_SPAN);
|
|
147
|
+
let out = '';
|
|
148
|
+
for (let i = 0; i < parts.length; i++) {
|
|
149
|
+
const part = parts[i];
|
|
150
|
+
// Odd indexes are the captured protected spans — byte-exact passthrough.
|
|
151
|
+
out += i % 2 === 1 ? part : compressSegment(part);
|
|
152
|
+
}
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
/**
|
|
156
|
+
* Deterministically strip low-information words from prose. Code (fenced,
|
|
157
|
+
* indented, inline), table rows, quoted spans, and every non-alphabetic
|
|
158
|
+
* token pass through byte-exact. Line structure is preserved (no line merges
|
|
159
|
+
* or splits) so renderer row accounting and paging stay comparable.
|
|
160
|
+
*
|
|
161
|
+
* Callers gate on `classifyContent(text) === 'other'` — running this over
|
|
162
|
+
* JSON/logs would be a no-op waste of CPU at best.
|
|
163
|
+
*/
|
|
164
|
+
export function cavemanize(text) {
|
|
165
|
+
if (text.length === 0)
|
|
166
|
+
return text;
|
|
167
|
+
const lines = text.split('\n');
|
|
168
|
+
let inFence = false;
|
|
169
|
+
for (let i = 0; i < lines.length; i++) {
|
|
170
|
+
const line = lines[i];
|
|
171
|
+
if (FENCE_LINE.test(line)) {
|
|
172
|
+
inFence = !inFence;
|
|
173
|
+
continue; // the fence marker itself stays verbatim
|
|
174
|
+
}
|
|
175
|
+
if (inFence)
|
|
176
|
+
continue;
|
|
177
|
+
const compressed = compressLine(line);
|
|
178
|
+
if (compressed !== line)
|
|
179
|
+
lines[i] = compressed;
|
|
180
|
+
}
|
|
181
|
+
return lines.join('\n');
|
|
182
|
+
}
|
|
183
|
+
//# sourceMappingURL=caveman.js.map
|
|
@@ -0,0 +1,128 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Core logic for `pxpipe export` — renders a source text to PNG pages, extracts
|
|
3
|
+
* a verbatim factsheet, builds a manifest and paste-ready prompt, and returns a
|
|
4
|
+
* token-cost report + list of artifacts to write.
|
|
5
|
+
*
|
|
6
|
+
* Pure-ish: no argv, no stdout, no process.exit, no fs calls — all I/O is
|
|
7
|
+
* delegated to the thin CLI runner in src/node.ts.
|
|
8
|
+
*/
|
|
9
|
+
export declare const DEFAULT_EXPORT_MODEL = "claude-sonnet-4-5";
|
|
10
|
+
/** Default column width — dense content mode (312 cols = 1568 px). */
|
|
11
|
+
export declare const DEFAULT_EXPORT_COLS: number;
|
|
12
|
+
/**
|
|
13
|
+
* Match a relative file path against a glob pattern.
|
|
14
|
+
* Supported wildcards: `*` (non-separator), `**` (any including `/`), `?` (single non-sep).
|
|
15
|
+
* When the pattern contains no `/`, matching is against the basename only
|
|
16
|
+
* (e.g. `*.ts` matches `src/foo.ts`).
|
|
17
|
+
*/
|
|
18
|
+
export declare function matchGlob(pattern: string, filePath: string): boolean;
|
|
19
|
+
/**
|
|
20
|
+
* Decide whether a relative file path should be included given include/exclude glob lists.
|
|
21
|
+
* - Exclude patterns are checked first; any match → exclude.
|
|
22
|
+
* - If include patterns are given, at least one must match.
|
|
23
|
+
* - If no include patterns are given, everything passes (unless excluded).
|
|
24
|
+
*/
|
|
25
|
+
export declare function shouldIncludeFile(filePath: string, include: string[], exclude: string[]): boolean;
|
|
26
|
+
export interface ExportParsed {
|
|
27
|
+
targets: string[];
|
|
28
|
+
include: string[];
|
|
29
|
+
exclude: string[];
|
|
30
|
+
git: boolean;
|
|
31
|
+
diff: string | undefined;
|
|
32
|
+
stdin: boolean;
|
|
33
|
+
cols: number;
|
|
34
|
+
out: string;
|
|
35
|
+
model: string;
|
|
36
|
+
json: boolean;
|
|
37
|
+
open: boolean;
|
|
38
|
+
}
|
|
39
|
+
export type ExportArgvResult = {
|
|
40
|
+
kind: 'opts';
|
|
41
|
+
parsed: ExportParsed;
|
|
42
|
+
} | {
|
|
43
|
+
kind: 'help';
|
|
44
|
+
} | {
|
|
45
|
+
kind: 'error';
|
|
46
|
+
message: string;
|
|
47
|
+
};
|
|
48
|
+
/**
|
|
49
|
+
* Parse the argv array for the `export` subcommand.
|
|
50
|
+
* Returns a discriminated union so the caller (node.ts) decides
|
|
51
|
+
* whether to exit, print help, or proceed.
|
|
52
|
+
*
|
|
53
|
+
* @param defaultOut Base output directory (default: $TMPDIR or /tmp).
|
|
54
|
+
*/
|
|
55
|
+
export declare function parseExportArgv(argv: string[], defaultOut?: string): ExportArgvResult;
|
|
56
|
+
/**
|
|
57
|
+
* Per-image vision-token cost for a rendered PNG at the given pixel dimensions.
|
|
58
|
+
*
|
|
59
|
+
* - **Claude / Anthropic models** (`model.startsWith('claude')` or
|
|
60
|
+
* `model.includes('anthropic')`): uses Anthropic's billing formula
|
|
61
|
+
* `ceil(width × height / 750 × 1.10)` (the same formula and constants as
|
|
62
|
+
* `imageTokensForRows` in transform.ts, reusing the exported
|
|
63
|
+
* `ANTHROPIC_PIXELS_PER_TOKEN` / `IMAGE_COST_SAFETY_MARGIN` consts).
|
|
64
|
+
* - **GPT / o-series models**: delegates to `openAIVisionTokens` which uses the
|
|
65
|
+
* GPT-4o tile-pricing formula (85 + 170 × tiles after scaling).
|
|
66
|
+
*/
|
|
67
|
+
export declare function exportImageTokens(model: string, width: number, height: number): number;
|
|
68
|
+
export interface ExportTokenReport {
|
|
69
|
+
textTokens: number;
|
|
70
|
+
imageTokens: number;
|
|
71
|
+
percentSaved: number;
|
|
72
|
+
/** Count of unique identifier strings extracted across all pages (paths, SHAs, ids, …).
|
|
73
|
+
* This is a count of items, not an LLM token count. */
|
|
74
|
+
factsheetItemCount: number;
|
|
75
|
+
/** Identifiers extracted across all pages that did not fit within the 64-item
|
|
76
|
+
* factsheet budget. Zero when all extracted items were kept. */
|
|
77
|
+
factsheetDropped: number;
|
|
78
|
+
}
|
|
79
|
+
/**
|
|
80
|
+
* Estimate text vs image token cost for `sourceText` without rendering.
|
|
81
|
+
* Uses the same formula as the internal gate:
|
|
82
|
+
* stripW = 2·PAD_X + cols·CELL_W
|
|
83
|
+
* imageTokens = estimateImageCount(text, cols) × exportImageTokens(model, stripW, MAX_HEIGHT_PX)
|
|
84
|
+
* textTokens = sourceText.length / REPORT_CHARS_PER_TOKEN
|
|
85
|
+
*
|
|
86
|
+
* `exportImageTokens` routes to the Anthropic billing formula (width×height/750×1.10)
|
|
87
|
+
* for Claude models, and to the GPT tile-pricing formula for GPT/o-series models.
|
|
88
|
+
*
|
|
89
|
+
* `factsheetItemCount` is the number of unique precision-critical identifier strings
|
|
90
|
+
* extracted across all pages (paths, SHAs, ids, …); it is NOT an LLM token count.
|
|
91
|
+
* `factsheetDropped` is the count of extracted identifiers that did not fit within
|
|
92
|
+
* the 64-item per-export budget.
|
|
93
|
+
*/
|
|
94
|
+
export declare function computeTokenReport(sourceText: string, cols: number, model: string): ExportTokenReport;
|
|
95
|
+
export interface ExportPageInfo {
|
|
96
|
+
filename: string;
|
|
97
|
+
bytes: number;
|
|
98
|
+
width: number;
|
|
99
|
+
height: number;
|
|
100
|
+
}
|
|
101
|
+
export interface ExportManifest {
|
|
102
|
+
sourceChars: number;
|
|
103
|
+
files: string[];
|
|
104
|
+
pages: ExportPageInfo[];
|
|
105
|
+
cols: number;
|
|
106
|
+
model: string;
|
|
107
|
+
generatedAt: string;
|
|
108
|
+
tokenReport: ExportTokenReport;
|
|
109
|
+
}
|
|
110
|
+
export interface ExportArtifact {
|
|
111
|
+
filename: string;
|
|
112
|
+
data: Uint8Array;
|
|
113
|
+
}
|
|
114
|
+
export interface ExportResult {
|
|
115
|
+
manifest: ExportManifest;
|
|
116
|
+
artifacts: ExportArtifact[];
|
|
117
|
+
}
|
|
118
|
+
export interface ExportCoreOptions {
|
|
119
|
+
/** Relative file paths listed in the manifest and prompt (display only). */
|
|
120
|
+
sourceFiles: string[];
|
|
121
|
+
cols: number;
|
|
122
|
+
model: string;
|
|
123
|
+
}
|
|
124
|
+
export declare function buildPromptText(pageCount: number, factsheet: string, files: string[], droppedItems?: number): string;
|
|
125
|
+
/** 8-char hex hash of the first 512 chars + length of `text`. */
|
|
126
|
+
export declare function sourceShortHash(text: string): string;
|
|
127
|
+
export declare function runExportCore(sourceText: string, opts: ExportCoreOptions): Promise<ExportResult>;
|
|
128
|
+
//# sourceMappingURL=export.d.ts.map
|