@astudioplus/compressor 0.1.0

package/dist/engine/detect.d.ts

@@ -0,0 +1,2 @@
+import type { ContentKind } from './types.ts';
+export declare function detectKind(content: string, filePath?: string): ContentKind;

package/dist/engine/detect.js

@@ -0,0 +1,47 @@
+const CODE_EXTENSIONS = new Set([
+    'ts', 'tsx', 'js', 'jsx', 'mjs', 'cjs',
+    'rs', 'py', 'go', 'java', 'c', 'h', 'cpp', 'hpp',
+    'rb', 'php', 'swift', 'kt', 'scala',
+    'sh', 'zsh', 'css', 'scss', 'sql', 'toml', 'yaml', 'yml',
+]);
+// Checkmark bullets (/^\s*[✓✗√×✘]\s/) alone are NOT a test-log signal: docs,
+// checklists, and CLI summaries use them too. Require an unambiguous runner
+// header/summary before classifying (and thus lossily filtering) as test-log.
+const TEST_LOG_RES = [
+    /^(PASS|FAIL)\s/m,
+    /^Tests:\s/m,
+    /\d+\s+pass(?:ed|ing)\b/,
+    /\d+\s+fail(?:ed|ing)\b/,
+    /^test result:/m,
+    /--- FAIL/,
+    /^test\s+\S+\s+\.\.\.\s+(?:ok|FAILED)\s*$/m,
+    // node:test spec ('ℹ pass 118') and tap ('# pass 118') summary counters
+    /^(?:ℹ|#) (?:tests|suites|pass|fail|cancelled|skipped|todo) \d+\s*$/m,
+];
+const BUILD_LOG_RES = [
+    /error\[E\d+\]/,
+    /^\s*(error|warning)(\s+TS\d+)?:/m,
+    /\bCompiling\s/,
+    /npm ERR!/,
+];
+const STACK_FRAME_RE = /^\s+at .+:\d+:\d+/m;
+export function detectKind(content, filePath) {
+    if (filePath !== undefined) {
+        const ext = extensionOf(filePath);
+        if (ext !== undefined && CODE_EXTENSIONS.has(ext))
+            return 'code';
+    }
+    if (content.startsWith('#!'))
+        return 'code';
+    if (TEST_LOG_RES.some((re) => re.test(content)))
+        return 'test-log';
+    if (BUILD_LOG_RES.some((re) => re.test(content)))
+        return 'build-log';
+    if (STACK_FRAME_RE.test(content) && /\berror\b/i.test(content))
+        return 'build-log';
+    return 'generic';
+}
+function extensionOf(filePath) {
+    const match = /\.([A-Za-z0-9]+)$/.exec(filePath);
+    return match?.[1]?.toLowerCase();
+}

package/dist/engine/index.d.ts

@@ -0,0 +1,4 @@
+import type { CompressMeta, CompressResult, Estimator, Policy } from './types.ts';
+export * from './types.ts';
+export { policyFor } from './policy.ts';
+export declare function compress(content: string, meta: CompressMeta, policy: Policy, estimate: Estimator): CompressResult;

package/dist/engine/index.js

@@ -0,0 +1,90 @@
+import { OMISSION_MARKER } from "./types.js";
+import { detectKind } from "./detect.js";
+import { collapseBlankRuns, dedupeLines, stripAnsi, truncateHeadTail } from "./tiers/structural.js";
+import { langFromPath, skeleton, stripComments } from "./tiers/code.js";
+import { filterBuildLog, filterTestLog } from "./tiers/logs.js";
+export * from "./types.js";
+export { policyFor } from "./policy.js";
+export function compress(content, meta, policy, estimate) {
+    const estTokensIn = estimate(content);
+    const bytesIn = utf8Bytes(content);
+    const passthrough = meta.mode === 'full' ||
+        meta.targeted === true ||
+        content.includes(OMISSION_MARKER) ||
+        estTokensIn < policy.touch;
+    if (passthrough) {
+        return {
+            content,
+            stats: {
+                bytesIn,
+                bytesOut: bytesIn,
+                estTokensIn,
+                estTokensOut: estTokensIn,
+                kind: detectKind(content, meta.filePath),
+                transforms: [],
+            },
+        };
+    }
+    let current = content;
+    const transforms = [];
+    const apply = (result) => {
+        current = result.content;
+        if (result.transform !== undefined)
+            transforms.push(result.transform);
+    };
+    // Decision estimator: threshold checks and truncation boundaries must not
+    // depend on marker TEXT, only on content — otherwise the marker-style
+    // experiment arms (plain/deterrent/informative phrasings of different
+    // lengths) would diverge in WHICH lines they keep, not just in marker
+    // wording. Any line containing OMISSION_MARKER mid-pipeline was inserted by
+    // an earlier tier in this run (pre-marked input passes through above), and
+    // every style inserts the same NUMBER of marker lines, so excluding them
+    // keeps decisions identical across styles.
+    const decide = (text) => estimate(text.includes(OMISSION_MARKER)
+        ? text
+            .split('\n')
+            .filter((line) => !line.includes(OMISSION_MARKER))
+            .join('\n')
+        : text);
+    if (policy.structural) {
+        apply(stripAnsi(current));
+        apply(collapseBlankRuns(current));
+        apply(dedupeLines(current));
+    }
+    const kind = detectKind(current, meta.filePath);
+    if (policy.codeAware && kind === 'code') {
+        const lang = langFromPath(meta.filePath);
+        if (decide(current) > policy.skeleton) {
+            apply(skeleton(current, lang, meta, estimate, policy.markerStyle));
+        }
+        else if (decide(current) > policy.commentStrip) {
+            apply(stripComments(current, lang, policy.markerStyle));
+        }
+    }
+    if (policy.logAware && decide(current) > policy.logFilter) {
+        if (kind === 'test-log')
+            apply(filterTestLog(current, policy.markerStyle));
+        else if (kind === 'build-log')
+            apply(filterBuildLog(current, policy.markerStyle));
+    }
+    if (decide(current) > policy.truncateBudget) {
+        // Earlier tiers (except strip-ansi) delete lines, so array positions no
+        // longer correspond to file line numbers.
+        const positionsAreFileLines = transforms.every((t) => t.id === 'strip-ansi');
+        apply(truncateHeadTail(current, meta, policy, decide, positionsAreFileLines));
+    }
+    return {
+        content: current,
+        stats: {
+            bytesIn,
+            bytesOut: utf8Bytes(current),
+            estTokensIn,
+            estTokensOut: estimate(current),
+            kind,
+            transforms,
+        },
+    };
+}
+function utf8Bytes(text) {
+    return new TextEncoder().encode(text).length;
+}

package/dist/engine/policy.d.ts

@@ -0,0 +1,2 @@
+import type { Mode, Policy } from './types.ts';
+export declare function policyFor(mode: Mode): Policy;

package/dist/engine/policy.js

@@ -0,0 +1,48 @@
+export function policyFor(mode) {
+    switch (mode) {
+        case 'full':
+            return {
+                structural: false,
+                codeAware: false,
+                logAware: false,
+                // 'plain' everywhere for now: the marker-style experiment (bench
+                // hookArgs --marker-style) varies this per arm and picks the winner.
+                markerStyle: 'plain',
+                touch: Infinity,
+                truncateBudget: Infinity,
+                commentStrip: Infinity,
+                skeleton: Infinity,
+                logFilter: Infinity,
+            };
+        // PLAN.md: optimized = tier 1 + comment-strip; lossy tier-3 log filtering
+        // is reserved for slim.
+        case 'optimized':
+            return {
+                structural: true,
+                codeAware: true,
+                logAware: false,
+                markerStyle: 'plain',
+                touch: 600,
+                truncateBudget: 5000,
+                commentStrip: 2000,
+                skeleton: Infinity,
+                logFilter: Infinity,
+            };
+        case 'slim':
+            return {
+                structural: true,
+                codeAware: true,
+                logAware: true,
+                markerStyle: 'plain',
+                touch: 300,
+                // measured (bench-20260610-114234/-123102): a 2,500 budget pushed the
+                // model into offset/limit pagination — targeted reads pass through, so
+                // recovery re-reads nullified all savings (worst cell exceeded the
+                // uncompressed baseline). 5,000 stays under the recovery trigger.
+                truncateBudget: 5000,
+                commentStrip: 1000,
+                skeleton: 6000,
+                logFilter: 800,
+            };
+    }
+}

package/dist/engine/tiers/code.d.ts

@@ -0,0 +1,7 @@
+import type { CompressMeta, Estimator, MarkerStyle } from '../types.ts';
+import type { TierResult } from './structural.ts';
+export type CodeLang = 'ts-js' | 'rust' | 'python' | 'go' | 'c-like' | 'ruby' | 'shell' | 'css' | 'sql' | 'config';
+export declare function langFromPath(filePath?: string): CodeLang | undefined;
+export declare function hasLineNumbers(content: string): boolean;
+export declare function stripComments(content: string, lang: CodeLang | undefined, style?: MarkerStyle): TierResult;
+export declare function skeleton(content: string, lang: CodeLang | undefined, meta: CompressMeta, estimate: Estimator, style?: MarkerStyle): TierResult;

package/dist/engine/tiers/code.js

@@ -0,0 +1,206 @@
+import { formatMatchLines, lineNumberOf, omissionMarker, scanFailureLines, tierResult, } from "./structural.js";
+const LANG_BY_EXT = {
+    ts: 'ts-js', tsx: 'ts-js', js: 'ts-js', jsx: 'ts-js', mjs: 'ts-js', cjs: 'ts-js',
+    rs: 'rust',
+    py: 'python',
+    go: 'go',
+    java: 'c-like', c: 'c-like', h: 'c-like', cpp: 'c-like', hpp: 'c-like',
+    swift: 'c-like', kt: 'c-like', scala: 'c-like', php: 'c-like',
+    rb: 'ruby',
+    sh: 'shell', zsh: 'shell',
+    css: 'css', scss: 'css',
+    sql: 'sql',
+    toml: 'config', yaml: 'config', yml: 'config',
+};
+export function langFromPath(filePath) {
+    if (filePath === undefined)
+        return undefined;
+    const match = /\.([A-Za-z0-9]+)$/.exec(filePath);
+    const ext = match?.[1]?.toLowerCase();
+    return ext === undefined ? undefined : LANG_BY_EXT[ext];
+}
+// Claude Code Read prefixes: "   123→content" (U+2192) or tab-separated variants.
+const LINE_NUM_RE = /^ *\d+(?:→|\t)/;
+export function hasLineNumbers(content) {
+    const lines = content.split('\n').filter((l) => l.length > 0);
+    if (lines.length < 2)
+        return false;
+    return lines.every((l) => LINE_NUM_RE.test(l));
+}
+function lineText(line) {
+    const match = LINE_NUM_RE.exec(line);
+    return match === null ? line : line.slice(match[0].length);
+}
+const COMMENT_SYNTAX = {
+    'ts-js': { line: ['//'], block: { open: '/*', close: '*/' } },
+    rust: { line: ['//'], block: { open: '/*', close: '*/' } },
+    python: { line: ['#'] },
+    go: { line: ['//'], block: { open: '/*', close: '*/' } },
+    'c-like': { line: ['//'], block: { open: '/*', close: '*/' } },
+    ruby: { line: ['#'] },
+    shell: { line: ['#'] },
+    css: { line: ['//'], block: { open: '/*', close: '*/' } },
+    sql: { line: ['--'], block: { open: '/*', close: '*/' } },
+    config: { line: ['#'] },
+};
+/** Advance python triple-quote string state across one line of code. */
+function scanPythonTriples(text, open) {
+    let state = open;
+    let i = 0;
+    while (i < text.length) {
+        if (state === null) {
+            const dq = text.indexOf('"""', i);
+            const sq = text.indexOf("'''", i);
+            const next = dq === -1 ? sq : sq === -1 ? dq : Math.min(dq, sq);
+            if (next === -1)
+                break;
+            state = text.startsWith('"""', next) ? '"""' : "'''";
+            i = next + 3;
+        }
+        else {
+            const close = text.indexOf(state, i);
+            if (close === -1)
+                break;
+            i = close + 3;
+            state = null;
+        }
+    }
+    return state;
+}
+function commentStripMarker(stripped, strippedLines, style) {
+    const head = `[compressor: ${stripped} comment/blank lines stripped — line numbers preserved`;
+    if (style === 'deterrent') {
+        return `${head}; comments are likely irrelevant to the problem you are chasing]`;
+    }
+    if (style === 'informative') {
+        const scan = scanFailureLines(strippedLines);
+        return scan.count === 0
+            ? `${head}; no error/failure/warning text among them; safe to skip]`
+            : `${head}; ${scan.count} stripped lines matching error/fail/warn at lines ${formatMatchLines(scan)}]`;
+    }
+    return `${head}]`;
+}
+export function stripComments(content, lang, style = 'plain') {
+    if (lang === undefined || !hasLineNumbers(content))
+        return { content };
+    // yaml/toml '#' lines inside block scalars are data, not comments — never strip config.
+    if (lang === 'config')
+        return { content };
+    const syntax = COMMENT_SYNTAX[lang];
+    const block = syntax.block;
+    const lines = content.split('\n');
+    const kept = [];
+    const strippedLines = [];
+    let stripped = 0;
+    let inBlock = false;
+    let tripleOpen = null;
+    for (const line of lines) {
+        if (line.length === 0) {
+            kept.push(line);
+            continue;
+        }
+        const text = lineText(line);
+        if (lang === 'python' && tripleOpen !== null) {
+            // inside a triple-quoted string: '#' and blank lines are literal data
+            tripleOpen = scanPythonTriples(text, tripleOpen);
+            kept.push(line);
+            continue;
+        }
+        const trimmed = text.trim();
+        let drop = false;
+        if (inBlock && block !== undefined) {
+            const closeIdx = trimmed.indexOf(block.close);
+            if (closeIdx >= 0) {
+                inBlock = false;
+                drop = trimmed.slice(closeIdx + block.close.length).trim() === '';
+            }
+            else {
+                drop = true;
+            }
+        }
+        else if (trimmed === '') {
+            drop = true;
+        }
+        else if (trimmed.startsWith('#!')) {
+            drop = false;
+        }
+        else if (syntax.line.some((p) => trimmed.startsWith(p))) {
+            drop = true;
+        }
+        else if (block !== undefined && trimmed.startsWith(block.open)) {
+            const rest = trimmed.slice(block.open.length);
+            const closeIdx = rest.indexOf(block.close);
+            if (closeIdx >= 0) {
+                drop = rest.slice(closeIdx + block.close.length).trim() === '';
+            }
+            else {
+                drop = true;
+                inBlock = true;
+            }
+        }
+        if (drop) {
+            stripped += 1;
+            strippedLines.push(line);
+        }
+        else {
+            if (lang === 'python')
+                tripleOpen = scanPythonTriples(text, null);
+            kept.push(line);
+        }
+    }
+    if (stripped === 0)
+        return { content };
+    const marker = commentStripMarker(stripped, strippedLines, style);
+    const trailing = kept.length > 0 && kept[kept.length - 1] === '' ? kept.pop() : undefined;
+    kept.push(marker);
+    if (trailing !== undefined)
+        kept.push(trailing);
+    return tierResult(content, kept.join('\n'), 'comment-strip');
+}
+const SIGNATURE_TESTS = {
+    'ts-js': (t) => /^(import|export|function|async function|class|interface|type)\b/.test(t),
+    rust: (t) => /^\s*(use|pub|fn|struct|enum|trait|impl|mod)\b/.test(t),
+    python: (t) => /^\s*(import|from|def|class|async def)\b/.test(t),
+    go: (t) => /^(package|import|func|type)\b/.test(t),
+};
+export function skeleton(content, lang, meta, estimate, style = 'plain') {
+    if (lang === undefined || !hasLineNumbers(content))
+        return { content };
+    const isSignature = SIGNATURE_TESTS[lang];
+    if (isSignature === undefined)
+        return { content };
+    const lines = content.split('\n');
+    const out = [];
+    let gap = [];
+    const flushGap = () => {
+        if (gap.length === 0)
+            return;
+        const first = gap[0];
+        const last = gap[gap.length - 1];
+        const a = first === undefined ? undefined : lineNumberOf(first);
+        const b = last === undefined ? undefined : lineNumberOf(last);
+        if (gap.length < 2 || a === undefined || b === undefined) {
+            out.push(...gap);
+        }
+        else {
+            out.push(omissionMarker(a, b, estimate(gap.join('\n')), meta, style, gap));
+        }
+        gap = [];
+    };
+    for (const line of lines) {
+        if (line.length === 0) {
+            flushGap();
+            out.push(line);
+            continue;
+        }
+        if (isSignature(lineText(line))) {
+            flushGap();
+            out.push(line);
+        }
+        else {
+            gap.push(line);
+        }
+    }
+    flushGap();
+    return tierResult(content, out.join('\n'), 'skeleton');
+}

package/dist/engine/tiers/logs.d.ts

@@ -0,0 +1,4 @@
+import type { MarkerStyle } from '../types.ts';
+import type { TierResult } from './structural.ts';
+export declare function filterTestLog(content: string, style?: MarkerStyle): TierResult;
+export declare function filterBuildLog(content: string, style?: MarkerStyle): TierResult;

package/dist/engine/tiers/logs.js

@@ -0,0 +1,139 @@
+import { scanFailureLines, tierResult } from "./structural.js";
+/**
+ * Style-aware omission marker shared by the test/build log filters. Logs have
+ * no file coordinates, so the informative variant reports match counts only.
+ */
+function logMarker(noun, omittedLines, style) {
+    const head = `[compressor: ${omittedLines.length} ${noun} lines omitted`;
+    if (style === 'deterrent') {
+        return `${head} — likely irrelevant; re-run with a narrower filter ONLY if the problem you are chasing points into the omitted output]`;
+    }
+    if (style === 'informative') {
+        const matches = scanFailureLines(omittedLines).count;
+        return matches === 0
+            ? `${head} — no error/failure/warning lines in the omitted output; safe to skip. Re-run with a narrower filter only if needed]`
+            : `${head} — ${matches} omitted lines matching error/fail/warn — re-run with a narrower filter to retrieve]`;
+    }
+    return `${head}]`;
+}
+const TEST_PASS_RES = [
+    /^\s*[✓✔√]\s/,
+    /^PASS\s/,
+    /^\s*--- PASS:/,
+    /^test\s+\S+\s+\.\.\.\s+ok\s*$/,
+    /\bPASSED\s*$/,
+    /^ok\s+\d+\s/,
+];
+const TEST_FAIL_RES = [
+    /^\s*[✗✘✖×]\s/,
+    /^FAIL\b/,
+    /--- FAIL/,
+    /\bFAILED\b/,
+    /^not ok\s/,
+    /\bpanicked\b/,
+    /^\s*●/,
+    // error reports only ('Error: boom', 'error: x'), not prose containing 'error'
+    /^\s*(?:[A-Za-z]*Error|error)\s*[:(]/,
+];
+const TEST_SUMMARY_RES = [
+    /^(Tests|Test Suites|Snapshots|Time|Duration):/,
+    /\d+\s+pass(?:ed|ing)\b/,
+    /\d+\s+fail(?:ed|ing)\b/,
+    /^test result:/,
+    /^Ran all test suites/,
+    /^=+ .* =+$/,
+    // node:test spec ('ℹ pass 118') and tap ('# pass 118') summary counters
+    /^(?:ℹ|#) (?:tests|suites|pass|fail|cancelled|skipped|todo|duration_ms) \d+\s*$/,
+];
+export function filterTestLog(content, style = 'plain') {
+    const lines = content.split('\n');
+    const isFail = (l) => TEST_FAIL_RES.some((re) => re.test(l));
+    const isSummary = (l) => TEST_SUMMARY_RES.some((re) => re.test(l));
+    const isPass = (l) => TEST_PASS_RES.some((re) => re.test(l));
+    // unrecognized format: never drop anything
+    if (!lines.some(isFail) && !lines.some(isSummary))
+        return { content };
+    const out = [];
+    const omittedLines = [];
+    let markerIdx = -1;
+    for (const line of lines) {
+        if (isPass(line) && !isFail(line) && !isSummary(line)) {
+            if (markerIdx < 0)
+                markerIdx = out.length;
+            omittedLines.push(line);
+        }
+        else {
+            out.push(line);
+        }
+    }
+    if (omittedLines.length === 0)
+        return { content };
+    out.splice(markerIdx, 0, logMarker('passing-test', omittedLines, style));
+    return tierResult(content, out.join('\n'), 'log-filter');
+}
+const BUILD_ERROR_RES = [
+    /error\[E\d+\]/,
+    /^\s*(error|warning)(\s+[A-Z]+\d+)?:/i,
+    /\b(error|warning)\s+TS\d+:/,
+    /:\d+(?::\d+)?:\s*(?:fatal\s+)?(?:error|warning)\b/,
+    /npm ERR!/,
+    /^\s*(Error|TypeError|SyntaxError|ReferenceError|RangeError)\b/,
+];
+const BUILD_STATUS_RES = [
+    /^\s*error: aborting/,
+    /^\s*error: could not compile/,
+    /Found \d+ errors?/,
+    /\d+ errors? generated/,
+    /^make: \*\*\*/,
+    /^Build (?:FAILED|failed|succeeded)/,
+    /^\s*Finished\b/,
+    /[✖✗] \d+ problems?/,
+    /exited with code \d+/,
+];
+export function filterBuildLog(content, style = 'plain') {
+    const lines = content.split('\n');
+    const isError = (l) => BUILD_ERROR_RES.some((re) => re.test(l));
+    const isStatus = (l) => BUILD_STATUS_RES.some((re) => re.test(l));
+    // unrecognized format: never drop anything
+    if (!lines.some(isError) && !lines.some(isStatus))
+        return { content };
+    const out = [];
+    const omittedLines = [];
+    let markerIdx = -1;
+    let inErrorBlock = false;
+    for (const line of lines) {
+        let keep;
+        if (isError(line)) {
+            inErrorBlock = true;
+            keep = true;
+        }
+        else if (isStatus(line)) {
+            inErrorBlock = false;
+            keep = true;
+        }
+        else if (inErrorBlock) {
+            if (line.trim() === '') {
+                inErrorBlock = false;
+                keep = false;
+            }
+            else {
+                keep = true;
+            }
+        }
+        else {
+            keep = false;
+        }
+        if (keep) {
+            out.push(line);
+        }
+        else {
+            if (markerIdx < 0)
+                markerIdx = out.length;
+            omittedLines.push(line);
+        }
+    }
+    if (omittedLines.length === 0)
+        return { content };
+    out.splice(markerIdx, 0, logMarker('build-log', omittedLines, style));
+    return tierResult(content, out.join('\n'), 'log-filter');
+}

package/dist/engine/tiers/structural.d.ts

@@ -0,0 +1,28 @@
+import type { AppliedTransform, CompressMeta, Estimator, MarkerStyle, Policy } from '../types.ts';
+export interface TierResult {
+    content: string;
+    transform?: AppliedTransform;
+}
+export declare function tierResult(before: string, after: string, id: string): TierResult;
+export declare function stripAnsi(content: string): TierResult;
+export declare function collapseBlankRuns(content: string): TierResult;
+export declare function dedupeLines(content: string): TierResult;
+export declare function lineNumberOf(line: string): number | undefined;
+export interface FailureScan {
+    /** matching lines in the scanned region (compressor markers excluded) */
+    count: number;
+    /** original-coordinate line numbers of the first 3 matches, when resolvable */
+    lines: number[];
+}
+/**
+ * Scan omitted lines for error/failure/warning content. Original coordinates
+ * follow the same rules as the marker math: embedded Read line numbers are
+ * authoritative; `firstFileLine + index` is the fallback when the caller
+ * vouches that positions are file lines. Marker lines inserted by earlier
+ * tiers in this run are never counted as content.
+ */
+export declare function scanFailureLines(lines: readonly string[], firstFileLine?: number): FailureScan;
+/** "L1, L2, L3 (first 3)" — the qualifier only when matches were truncated. */
+export declare function formatMatchLines(scan: FailureScan): string;
+export declare function omissionMarker(a: number, b: number, estTokens: number, meta: CompressMeta, style: MarkerStyle, omittedLines: readonly string[]): string;
+export declare function truncateHeadTail(content: string, meta: CompressMeta, policy: Policy, estimate: Estimator, positionsAreFileLines?: boolean): TierResult;