@xamukavila/pxpipe 0.8.0

package/dist/core/export.js

@@ -0,0 +1,390 @@
+/**
+ * Core logic for `pxpipe export` — renders a source text to PNG pages, extracts
+ * a verbatim factsheet, builds a manifest and paste-ready prompt, and returns a
+ * token-cost report + list of artifacts to write.
+ *
+ * Pure-ish: no argv, no stdout, no process.exit, no fs calls — all I/O is
+ * delegated to the thin CLI runner in src/node.ts.
+ */
+import { DENSE_CONTENT_CHARS_PER_IMAGE, DENSE_CONTENT_COLS, MAX_HEIGHT_PX, PAD_X, CELL_W, } from './render.js';
+// Dogfood the public SDK: render via the same `./transform` entry external
+// consumers import (pxpipe-proxy/transform → renderTextToImages), not the
+// internal leaf renderer.
+import { renderTextToImages } from './library.js';
+import { estimateImageCount, ANTHROPIC_PIXELS_PER_TOKEN, IMAGE_COST_SAFETY_MARGIN, REPORT_CHARS_PER_TOKEN } from './transform.js';
+import { openAIVisionTokens } from './openai.js';
+import { factSheetTextFromEntries, extractFactSheetTokensAllPages, extractFactSheetEntriesAllPages, } from './factsheet.js';
+// ---------------------------------------------------------------------------
+// Public constants
+// ---------------------------------------------------------------------------
+export const DEFAULT_EXPORT_MODEL = 'claude-sonnet-4-5';
+// Chars-per-token for the reporting estimate now lives in transform.ts as
+// REPORT_CHARS_PER_TOKEN (single source of truth for all token-estimate constants).
+/** Default column width — dense content mode (312 cols = 1568 px). */
+export const DEFAULT_EXPORT_COLS = DENSE_CONTENT_COLS;
+// ---------------------------------------------------------------------------
+// Glob matching (no external glob library — node:fs only per convention)
+// ---------------------------------------------------------------------------
+/**
+ * Match a relative file path against a glob pattern.
+ * Supported wildcards: `*` (non-separator), `**` (any including `/`), `?` (single non-sep).
+ * When the pattern contains no `/`, matching is against the basename only
+ * (e.g. `*.ts` matches `src/foo.ts`).
+ */
+export function matchGlob(pattern, filePath) {
+    const pat = pattern.replace(/\\/g, '/');
+    const fp = filePath.replace(/\\/g, '/').replace(/^\.\//, '');
+    if (!pat.includes('/')) {
+        // No separator → match basename only
+        const sep = fp.lastIndexOf('/');
+        const basename = sep >= 0 ? fp.slice(sep + 1) : fp;
+        return _globTest(pat, basename);
+    }
+    return _globTest(pat, fp);
+}
+function _globTest(pattern, str) {
+    const SPECIAL = new Set([
+        '.', '+', '^', '$', '{', '}', '(', ')', '|', '[', ']', '\\',
+    ]);
+    let re = '';
+    let i = 0;
+    while (i < pattern.length) {
+        const ch = pattern[i];
+        if (ch === undefined)
+            break;
+        const next = pattern[i + 1];
+        const nextNext = pattern[i + 2];
+        if (ch === '*' && next === '*') {
+            if (nextNext === '/') {
+                re += '(?:.*/)?';
+                i += 3;
+            }
+            else {
+                re += '.*';
+                i += 2;
+            }
+        }
+        else if (ch === '*') {
+            re += '[^/]*';
+            i++;
+        }
+        else if (ch === '?') {
+            re += '[^/]';
+            i++;
+        }
+        else if (SPECIAL.has(ch)) {
+            re += '\\' + ch;
+            i++;
+        }
+        else {
+            re += ch;
+            i++;
+        }
+    }
+    try {
+        return new RegExp(`^${re}$`).test(str);
+    }
+    catch {
+        return false;
+    }
+}
+/**
+ * Decide whether a relative file path should be included given include/exclude glob lists.
+ * - Exclude patterns are checked first; any match → exclude.
+ * - If include patterns are given, at least one must match.
+ * - If no include patterns are given, everything passes (unless excluded).
+ */
+export function shouldIncludeFile(filePath, include, exclude) {
+    if (exclude.some((pat) => matchGlob(pat, filePath)))
+        return false;
+    if (include.length > 0)
+        return include.some((pat) => matchGlob(pat, filePath));
+    return true;
+}
+/**
+ * Parse the argv array for the `export` subcommand.
+ * Returns a discriminated union so the caller (node.ts) decides
+ * whether to exit, print help, or proceed.
+ *
+ * @param defaultOut  Base output directory (default: $TMPDIR or /tmp).
+ */
+export function parseExportArgv(argv, defaultOut) {
+    const targets = [];
+    const include = [];
+    const exclude = [];
+    let git = false;
+    let diff;
+    let stdin = false;
+    // Locked to the proxy's density — NO CLI knob. Export must render exactly what the
+    // proxy ships to the model; the proxy has no width flag, so neither does export.
+    const cols = DENSE_CONTENT_COLS;
+    let out = defaultOut ??
+        (typeof process !== 'undefined'
+            ? (process.env['TMPDIR'] ?? process.env['TEMP'] ?? '/tmp')
+            : '/tmp');
+    let model = DEFAULT_EXPORT_MODEL;
+    let json = false;
+    let open = false;
+    for (let i = 0; i < argv.length; i++) {
+        const a = argv[i];
+        if (a === undefined)
+            break;
+        if (a === '-h' || a === '--help') {
+            return { kind: 'help' };
+        }
+        else if (a === '--include' || a === '--exclude') {
+            i++;
+            const val = argv[i];
+            if (val === undefined || val.startsWith('-')) {
+                return { kind: 'error', message: `${a} requires a value` };
+            }
+            if (a === '--include')
+                include.push(val);
+            else
+                exclude.push(val);
+        }
+        else if (a.startsWith('--include=')) {
+            const v = a.slice('--include='.length);
+            if (!v)
+                return { kind: 'error', message: '--include= requires a non-empty value' };
+            include.push(v);
+        }
+        else if (a.startsWith('--exclude=')) {
+            const v = a.slice('--exclude='.length);
+            if (!v)
+                return { kind: 'error', message: '--exclude= requires a non-empty value' };
+            exclude.push(v);
+        }
+        else if (a === '--git') {
+            git = true;
+        }
+        else if (a === '--diff') {
+            i++;
+            const val = argv[i];
+            if (val === undefined || val.startsWith('-')) {
+                return { kind: 'error', message: '--diff requires a value' };
+            }
+            diff = val;
+        }
+        else if (a.startsWith('--diff=')) {
+            const v = a.slice('--diff='.length);
+            if (!v)
+                return { kind: 'error', message: '--diff= requires a non-empty value' };
+            diff = v;
+        }
+        else if (a === '--stdin') {
+            stdin = true;
+        }
+        else if (a === '--out') {
+            i++;
+            const val = argv[i];
+            if (val === undefined)
+                return { kind: 'error', message: '--out requires a value' };
+            out = val;
+        }
+        else if (a.startsWith('--out=')) {
+            const v = a.slice('--out='.length);
+            if (!v)
+                return { kind: 'error', message: '--out= requires a non-empty value' };
+            out = v;
+        }
+        else if (a === '--model') {
+            i++;
+            const val = argv[i];
+            if (val === undefined)
+                return { kind: 'error', message: '--model requires a value' };
+            model = val;
+        }
+        else if (a.startsWith('--model=')) {
+            const v = a.slice('--model='.length);
+            if (!v)
+                return { kind: 'error', message: '--model= requires a non-empty value' };
+            model = v;
+        }
+        else if (a === '--json') {
+            json = true;
+        }
+        else if (a === '--open') {
+            open = true;
+        }
+        else if (a.startsWith('-')) {
+            return { kind: 'error', message: `unknown option: ${a}` };
+        }
+        else {
+            targets.push(a);
+        }
+    }
+    return {
+        kind: 'opts',
+        parsed: { targets, include, exclude, git, diff, stdin, cols, out, model, json, open },
+    };
+}
+// ---------------------------------------------------------------------------
+// Model-routing image-token helper
+// ---------------------------------------------------------------------------
+/**
+ * Per-image vision-token cost for a rendered PNG at the given pixel dimensions.
+ *
+ * - **Claude / Anthropic models** (`model.startsWith('claude')` or
+ *   `model.includes('anthropic')`): uses Anthropic's billing formula
+ *   `ceil(width × height / 750 × 1.10)` (the same formula and constants as
+ *   `imageTokensForRows` in transform.ts, reusing the exported
+ *   `ANTHROPIC_PIXELS_PER_TOKEN` / `IMAGE_COST_SAFETY_MARGIN` consts).
+ * - **GPT / o-series models**: delegates to `openAIVisionTokens` which uses the
+ *   GPT-4o tile-pricing formula (85 + 170 × tiles after scaling).
+ */
+export function exportImageTokens(model, width, height) {
+    if (model.startsWith('claude') || model.includes('anthropic')) {
+        return Math.ceil((width * height / ANTHROPIC_PIXELS_PER_TOKEN) * IMAGE_COST_SAFETY_MARGIN);
+    }
+    return openAIVisionTokens(model, width, height);
+}
+/**
+ * Estimate text vs image token cost for `sourceText` without rendering.
+ * Uses the same formula as the internal gate:
+ *   stripW = 2·PAD_X + cols·CELL_W
+ *   imageTokens = estimateImageCount(text, cols) × exportImageTokens(model, stripW, MAX_HEIGHT_PX)
+ *   textTokens = sourceText.length / REPORT_CHARS_PER_TOKEN
+ *
+ * `exportImageTokens` routes to the Anthropic billing formula (width×height/750×1.10)
+ * for Claude models, and to the GPT tile-pricing formula for GPT/o-series models.
+ *
+ * `factsheetItemCount` is the number of unique precision-critical identifier strings
+ * extracted across all pages (paths, SHAs, ids, …); it is NOT an LLM token count.
+ * `factsheetDropped` is the count of extracted identifiers that did not fit within
+ * the 64-item per-export budget.
+ */
+export function computeTokenReport(sourceText, cols, model) {
+    const stripW = 2 * PAD_X + cols * CELL_W;
+    const estImages = estimateImageCount(sourceText, cols, 1, DENSE_CONTENT_CHARS_PER_IMAGE);
+    const perStrip = exportImageTokens(model, stripW, MAX_HEIGHT_PX);
+    const imageTokens = Math.round(estImages * perStrip);
+    const textTokens = Math.round(sourceText.length / REPORT_CHARS_PER_TOKEN);
+    const percentSaved = textTokens > 0
+        ? Math.round(((textTokens - imageTokens) / textTokens) * 1000) / 10
+        : 0;
+    const { kept, dropped } = extractFactSheetTokensAllPages(sourceText, DENSE_CONTENT_CHARS_PER_IMAGE);
+    return {
+        textTokens,
+        imageTokens,
+        percentSaved,
+        factsheetItemCount: kept.length,
+        factsheetDropped: dropped,
+    };
+}
+// ---------------------------------------------------------------------------
+// Prompt template
+// ---------------------------------------------------------------------------
+export function buildPromptText(pageCount, factsheet, files, droppedItems = 0) {
+    const lastPageStr = String(pageCount).padStart(3, '0');
+    const fileSection = files.length > 0
+        ? `Source files (${files.length}):\n${files.map((f) => `  ${f}`).join('\n')}\n\n`
+        : '';
+    const factsheetNote = droppedItems > 0
+        ? `2. For any path, file name, identifier, hash, version number, or other precision-critical\n` +
+            `   value: check factsheet.txt first — identifiers listed there are verbatim exact.\n` +
+            `   Note: ${droppedItems} identifier(s) were extracted but not captured due to the 64-item\n` +
+            `   budget — OCR carefully for any exact values not listed in factsheet.txt.\n`
+        : `2. For any path, file name, identifier, hash, version number, or other precision-critical\n` +
+            `   value: use the verbatim text from factsheet.txt, NOT what you read off the image.\n` +
+            `   factsheet.txt is the authoritative source of truth for all exact strings.\n`;
+    return (`These ${pageCount} image${pageCount !== 1 ? 's' : ''} contain source code/text rendered as PNG pages by pxpipe.\n\n` +
+        fileSection +
+        `Instructions for the reading agent:\n` +
+        `1. Read the images in order: page-001.png through page-${lastPageStr}.png.\n` +
+        factsheetNote +
+        `3. Treat the content as if you had read the source files directly.\n\n` +
+        `Factsheet (verbatim identifiers — quote from here, not from image pixels):\n` +
+        (factsheet || '(none)') +
+        '\n');
+}
+// ---------------------------------------------------------------------------
+// Simple hash for output directory naming
+// ---------------------------------------------------------------------------
+/** FNV-1a 32-bit hash — deterministic, no crypto dep. */
+function fnv32a(str) {
+    let h = 0x811c9dc5;
+    for (let i = 0; i < str.length; i++) {
+        h ^= str.charCodeAt(i) & 0xff;
+        h = (h * 0x01000193) >>> 0;
+    }
+    return h;
+}
+/** 8-char hex hash of the first 512 chars + length of `text`. */
+export function sourceShortHash(text) {
+    const sample = text.slice(0, 512) + '\x00' + String(text.length);
+    return fnv32a(sample).toString(16).padStart(8, '0');
+}
+// ---------------------------------------------------------------------------
+// Core export runner (pure-ish: no fs, no argv, no stdout)
+// ---------------------------------------------------------------------------
+export async function runExportCore(sourceText, opts) {
+    const generatedAt = new Date().toISOString();
+    const enc = new TextEncoder();
+    // Render to PNG pages via the public SDK primitive. shrink=true sizes the canvas
+    // to the widest line so short-line code isn't padded to full width; multiCol='auto'
+    // packs as many columns side-by-side as fit. Same render surface shipped to external
+    // SDK consumers (pxpipe-proxy/transform).
+    const { pages: images } = await renderTextToImages(sourceText, {
+        cols: opts.cols,
+        shrink: true,
+        multiCol: 'auto',
+        reflow: true,
+    });
+    // Compute token costs using actual rendered image dimensions (more accurate than estimate).
+    // exportImageTokens routes to the Anthropic billing formula for claude-* models and to
+    // the GPT tile-pricing formula for GPT/o-series models.
+    const textTokens = Math.round(sourceText.length / REPORT_CHARS_PER_TOKEN);
+    let imageTokens = 0;
+    for (const img of images) {
+        imageTokens += exportImageTokens(opts.model, img.width, img.height);
+    }
+    imageTokens = Math.round(imageTokens);
+    const percentSaved = textTokens > 0
+        ? Math.round(((textTokens - imageTokens) / textTokens) * 1000) / 10
+        : 0;
+    // Extract factsheet tokens across ALL pages so identifiers from page 3+ are covered.
+    const { kept: fsKept, dropped: fsDropped } = extractFactSheetEntriesAllPages(sourceText, DENSE_CONTENT_CHARS_PER_IMAGE);
+    const fsText = factSheetTextFromEntries(fsKept);
+    const tokenReport = {
+        textTokens,
+        imageTokens,
+        percentSaved,
+        factsheetItemCount: fsKept.length,
+        factsheetDropped: fsDropped,
+    };
+    // Build artifacts
+    const artifacts = [];
+    const pages = [];
+    for (const [i, img] of images.entries()) {
+        const filename = `page-${String(i + 1).padStart(3, '0')}.png`;
+        artifacts.push({ filename, data: img.png });
+        pages.push({
+            filename,
+            bytes: img.png.byteLength,
+            width: img.width,
+            height: img.height,
+        });
+    }
+    // factsheet.txt
+    artifacts.push({ filename: 'factsheet.txt', data: enc.encode(fsText) });
+    // manifest.json
+    const manifest = {
+        sourceChars: sourceText.length,
+        files: opts.sourceFiles,
+        pages,
+        cols: opts.cols,
+        model: opts.model,
+        generatedAt,
+        tokenReport,
+    };
+    artifacts.push({
+        filename: 'manifest.json',
+        data: enc.encode(JSON.stringify(manifest, null, 2)),
+    });
+    // prompt.txt
+    const promptText = buildPromptText(images.length, fsText, opts.sourceFiles, fsDropped);
+    artifacts.push({ filename: 'prompt.txt', data: enc.encode(promptText) });
+    return { manifest, artifacts };
+}
+//# sourceMappingURL=export.js.map

package/dist/core/factsheet.d.ts

@@ -0,0 +1,78 @@
+/**
+ * Verbatim fact-sheet for imaged content.
+ *
+ * When pxpipe renders a block (system slab, history, tool_result, reminder) to a PNG,
+ * the precision-critical, hard-to-OCR strings inside it — file paths, URLs, SHAs/UUIDs,
+ * version numbers, CLI flags, large numbers, CONST_IDS — are exactly what a model is
+ * most likely to misread off the image yet most likely to need quoted verbatim. This
+ * module extracts those tokens so they ride next to the image as plain text: the model
+ * quotes them without re-reading the PNG, and they stay in the cached prefix.
+ *
+ * Deterministic by construction (fixed pattern order, length-desc/lexical sort, no
+ * Date/random) → the emitted text is byte-stable across turns and never busts the
+ * Anthropic prompt cache. Empirically ~5% of source chars on production history
+ * (median 4.9%, max 12.1%, N=10), which preserves the imaging token win.
+ */
+/** Budget cap: highest-priority tokens kept first. Exported so consumers can report drops. */
+export declare const MAX_TOKENS = 64;
+/**
+ * Extract deduped, precision-critical tokens from `text`. Substrings of a longer kept
+ * token are dropped (so `/github.com` inside the full URL, `lib/x.ts` inside
+ * `src/lib/x.ts`, etc. collapse to the most specific form); the 64-token budget is then
+ * filled by priority tier (see `priorityTier`) so short, high-consequence tokens are never
+ * evicted by long low-risk URLs.
+ *
+ * Every token class is whitespace-free, so we split on whitespace first and skip
+ * blob-length chunks. That bounds each regex to a short chunk and keeps extraction
+ * strictly O(n) — no quadratic backtracking on delimiter-heavy input like base64 or
+ * minified bundles (which embed `/` and would otherwise make the path patterns blow up).
+ */
+export declare function extractFactSheetTokens(text: string): string[];
+/** A kept fact-sheet token plus how many times it occurs in the scanned text.
+ *  Counts are advisory (occurrences, not lines) but deterministic → cache-stable. */
+export interface FactSheetEntry {
+    readonly token: string;
+    readonly count: number;
+}
+/**
+ * Like `extractFactSheetTokens`, but each kept token carries its occurrence count.
+ * Counts make the fact sheet a *quantitative* index: tally questions over imaged
+ * content ("how many lines mention CODE-X?") become answerable from text instead
+ * of from counting rows of 5×8 px glyphs — the one operation page images are worst
+ * at. The kept-token SET and its order are byte-identical to the pre-count
+ * behaviour; only counts are new. Same-token spans matched by two patterns are
+ * deduped by offset so a token is never double-counted.
+ */
+export declare function extractFactSheetEntries(text: string): FactSheetEntry[];
+/**
+ * Page-aware variant of `extractFactSheetTokens` for large source texts.
+ *
+ * Splits `text` into chunks of `charsPerPage` (use `DENSE_CONTENT_CHARS_PER_IMAGE`
+ * from render.ts for the export pipeline), calls `extractFactSheetTokens` on each
+ * chunk (each chunk is smaller than MAX_SCAN so no truncation occurs), merges the
+ * results across all chunks with first-seen deduplication, then applies a single
+ * global priority-budget pass to select the best MAX_TOKENS identifiers.
+ *
+ * Returns `{ kept, dropped }` where `dropped` is the count of identifiers that
+ * survived extraction across all pages but did not fit in the MAX_TOKENS budget.
+ *
+ * Does NOT mutate the behaviour of `extractFactSheetTokens` or `factSheetText`.
+ */
+export declare function extractFactSheetTokensAllPages(text: string, charsPerPage: number): {
+    kept: string[];
+    dropped: number;
+};
+/** Entry-carrying variant of `extractFactSheetTokensAllPages`: same kept set and
+ *  order, with per-token occurrence counts summed across all pages. */
+export declare function extractFactSheetEntriesAllPages(text: string, charsPerPage: number): {
+    kept: FactSheetEntry[];
+    dropped: number;
+};
+/** Build the one-line fact-sheet string from a pre-extracted token list. */
+export declare function factSheetTextFromTokens(tokens: string[]): string;
+/** Build the one-line fact-sheet string from token+count entries. Byte-identical to
+ *  `factSheetTextFromTokens` when no token repeats, so existing sheets stay cache-stable. */
+export declare function factSheetTextFromEntries(entries: readonly FactSheetEntry[]): string;
+/** One-line fact-sheet string for `text`, or `''` when nothing notable was found. */
+export declare function factSheetText(text: string): string;
+//# sourceMappingURL=factsheet.d.ts.map