@astudioplus/compressor 0.1.0

package/dist/bench/runner.js

@@ -0,0 +1,121 @@
+import { runCell } from "./cell.js";
+import { appendResult, newRunId, runFilePath, writeRunMeta } from "./results.js";
+/**
+ * Cells that report no cost (timeouts, errors, subscription/Bedrock auth)
+ * still bill real API spend, so the dollar ceiling cannot see them. After
+ * this many consecutive no-cost cells the ceiling is unenforceable and the
+ * runner stops scheduling instead of burning the whole grid.
+ */
+export const MAX_CONSECUTIVE_NO_COST_CELLS = 3;
+function zeroedResult(runId, cell, error) {
+    return {
+        runId,
+        taskId: cell.task.id,
+        variantId: cell.variant.id,
+        trial: cell.trial,
+        model: cell.model,
+        servedModels: [],
+        baselineCheckPassed: null,
+        success: null,
+        usage: { input: 0, output: 0, cacheCreation: 0, cacheRead: 0 },
+        costUsd: null,
+        durationMs: 0,
+        numTurns: 0,
+        permissionDenials: 0,
+        toolCalls: {},
+        sessionId: null,
+        error,
+        timestamp: new Date().toISOString(),
+    };
+}
+function statusOf(row) {
+    if (row.error !== undefined)
+        return row.error;
+    if (row.success === null)
+        return 'unchecked';
+    return row.success ? 'pass' : 'fail';
+}
+export async function runBenchmark(opts) {
+    const runId = newRunId();
+    const resultsFile = runFilePath(opts.outDir, runId);
+    await writeRunMeta(opts.outDir, {
+        runId,
+        suite: opts.suite.name,
+        variantIds: opts.variants.map((v) => v.id),
+        model: opts.model,
+        trials: opts.trials,
+        startedAt: new Date().toISOString(),
+        maxBudgetUsd: opts.maxBudgetUsd,
+    });
+    // variants innermost: an early budget stop still covers every variant on
+    // the tasks that did run. Enforced group-atomically below: consecutive
+    // cells form one task×trial group (group size = variant count), the stop
+    // decision is made ONCE at a group's first cell and shared by the rest of
+    // the group, so a mid-group ceiling trip can never leave some arms of a
+    // task×trial measured and others skipped — cross-arm comparisons need
+    // complete groups.
+    const cells = [];
+    for (let trial = 1; trial <= opts.trials; trial += 1) {
+        for (const task of opts.suite.tasks) {
+            for (const variant of opts.variants) {
+                cells.push({ task, variant, trial, model: opts.model });
+            }
+        }
+    }
+    const groupSize = Math.max(1, opts.variants.length);
+    const results = new Array(cells.length);
+    let spentUsd = 0;
+    let noCostStreak = 0;
+    let next = 0;
+    // task×trial group index → skip reason decided at the group's first cell
+    // (null = the whole group runs)
+    const groupSkip = new Map();
+    const progress = (line) => opts.onProgress?.(line);
+    const worker = async () => {
+        while (next < cells.length) {
+            const index = next;
+            next += 1;
+            const cell = cells[index];
+            if (cell === undefined)
+                return;
+            const label = `${cell.task.id} × ${cell.variant.id} trial ${cell.trial}`;
+            const group = Math.floor(index / groupSize);
+            let skipReason = groupSkip.get(group);
+            if (skipReason === undefined) {
+                skipReason =
+                    spentUsd >= opts.maxBudgetUsd
+                        ? `skipped: budget ceiling ${opts.maxBudgetUsd} USD reached`
+                        : noCostStreak >= MAX_CONSECUTIVE_NO_COST_CELLS
+                            ? `skipped: ${MAX_CONSECUTIVE_NO_COST_CELLS} consecutive cells reported no cost — budget ceiling ${opts.maxBudgetUsd} USD is unenforceable`
+                            : null;
+                groupSkip.set(group, skipReason);
+            }
+            let row;
+            if (skipReason !== null) {
+                row = zeroedResult(runId, cell, skipReason);
+            }
+            else {
+                try {
+                    row = await runCell(cell, { runId, fixturesDir: opts.fixturesDir });
+                }
+                catch (error) {
+                    // runCell catches internally; this guards the pool regardless
+                    row = zeroedResult(runId, cell, error instanceof Error ? error.message : String(error));
+                }
+                if (typeof row.costUsd === 'number') {
+                    spentUsd += row.costUsd;
+                    noCostStreak = 0;
+                }
+                else {
+                    noCostStreak += 1;
+                }
+            }
+            results[index] = row;
+            await appendResult(opts.outDir, runId, row);
+            progress(`[${index + 1}/${cells.length}] ${label}: ${statusOf(row)} (spent $${spentUsd.toFixed(4)})`);
+        }
+    };
+    const poolSize = Math.min(Math.max(1, Math.floor(opts.concurrency) || 2), Math.max(1, cells.length));
+    await Promise.all(Array.from({ length: poolSize }, () => worker()));
+    return { runId, results, resultsFile };
+}

package/dist/bench/tasks.d.ts

@@ -0,0 +1,4 @@
+import type { SuiteSpec } from './types.ts';
+export declare function loadSuite(suitePath: string): Promise<SuiteSpec>;
+/** Fixture root shipped alongside a suite file: <suiteDir>/../fixtures. */
+export declare function suiteFixturesDir(suitePath: string): string;

package/dist/bench/tasks.js

@@ -0,0 +1,147 @@
+import { readFile } from 'node:fs/promises';
+import path from 'node:path';
+function isRecord(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function nonEmptyString(value) {
+    return typeof value === 'string' && value.length > 0;
+}
+function isStringArray(value) {
+    return Array.isArray(value) && value.every((item) => typeof item === 'string');
+}
+function message(error) {
+    return error instanceof Error ? error.message : String(error);
+}
+function parseCheck(raw, taskId) {
+    if (!isRecord(raw)) {
+        throw new Error(`task ${taskId}: check must be an object`);
+    }
+    const kind = raw['kind'];
+    if (kind === 'command') {
+        const command = raw['command'];
+        if (!nonEmptyString(command)) {
+            throw new Error(`task ${taskId}: command check requires a non-empty command string`);
+        }
+        return { kind: 'command', command };
+    }
+    if (kind === 'answer-regex') {
+        const pattern = raw['pattern'];
+        if (!nonEmptyString(pattern)) {
+            throw new Error(`task ${taskId}: answer-regex check requires a non-empty pattern string`);
+        }
+        const flags = raw['flags'];
+        if (flags !== undefined && typeof flags !== 'string') {
+            throw new Error(`task ${taskId}: answer-regex flags must be a string`);
+        }
+        try {
+            new RegExp(pattern, flags);
+        }
+        catch (error) {
+            throw new Error(`task ${taskId}: answer-regex pattern does not compile — ${message(error)}`);
+        }
+        return flags === undefined
+            ? { kind: 'answer-regex', pattern }
+            : { kind: 'answer-regex', pattern, flags };
+    }
+    throw new Error(`task ${taskId}: unknown check kind ${JSON.stringify(kind)} (expected 'command' or 'answer-regex')`);
+}
+/**
+ * Multi-turn check semantics (tasks with `turns`):
+ * - command checks run once, in the workspace, after the FINAL turn.
+ * - answer-regex checks treat the whole conversation as the answer: the cell
+ *   passes when the pattern matches ANY single turn's result text. Each turn
+ *   is judged separately (so `^` anchors apply per turn), never only the
+ *   final turn — follow-ups like "draft a PR comment" would otherwise erase
+ *   evidence the first turn already produced.
+ */
+function parseTurns(raw, taskId) {
+    if (!Array.isArray(raw) || raw.length === 0) {
+        throw new Error(`task ${taskId}: turns must be a non-empty array of follow-up prompt strings`);
+    }
+    return raw.map((turn, index) => {
+        if (!nonEmptyString(turn)) {
+            throw new Error(`task ${taskId}: turns[${index}] must be a non-empty string`);
+        }
+        return turn;
+    });
+}
+function parseFixture(raw, taskId) {
+    if (!nonEmptyString(raw)) {
+        throw new Error(`task ${taskId}: fixture must be a non-empty string`);
+    }
+    if (raw.includes('/') || raw.includes('\\') || raw === '.' || raw === '..') {
+        throw new Error(`task ${taskId}: fixture must be a bare directory name (no path separators), got ${JSON.stringify(raw)}`);
+    }
+    return raw;
+}
+function parseTask(raw, index, seenIds) {
+    if (!isRecord(raw)) {
+        throw new Error(`task at index ${index}: must be an object`);
+    }
+    const id = raw['id'];
+    if (!nonEmptyString(id)) {
+        throw new Error(`task at index ${index}: id must be a non-empty string`);
+    }
+    if (seenIds.has(id)) {
+        throw new Error(`task ${id}: duplicate id`);
+    }
+    seenIds.add(id);
+    const prompt = raw['prompt'];
+    if (!nonEmptyString(prompt)) {
+        throw new Error(`task ${id}: prompt must be a non-empty string`);
+    }
+    const fixture = parseFixture(raw['fixture'], id);
+    const check = parseCheck(raw['check'], id);
+    const tags = raw['tags'];
+    if (tags !== undefined && !isStringArray(tags)) {
+        throw new Error(`task ${id}: tags must be an array of strings`);
+    }
+    const task = { id, prompt, fixture, check };
+    const turns = raw['turns'];
+    if (turns !== undefined) {
+        task.turns = parseTurns(turns, id);
+    }
+    if (tags !== undefined) {
+        task.tags = tags;
+    }
+    return task;
+}
+export async function loadSuite(suitePath) {
+    let raw;
+    try {
+        raw = await readFile(suitePath, 'utf8');
+    }
+    catch (error) {
+        throw new Error(`suite ${suitePath}: unreadable — ${message(error)}`);
+    }
+    let parsed;
+    try {
+        parsed = JSON.parse(raw);
+    }
+    catch (error) {
+        throw new Error(`suite ${suitePath}: invalid JSON — ${message(error)}`);
+    }
+    if (!isRecord(parsed)) {
+        throw new Error(`suite ${suitePath}: root must be an object`);
+    }
+    const name = parsed['name'];
+    if (!nonEmptyString(name)) {
+        throw new Error(`suite ${suitePath}: name must be a non-empty string`);
+    }
+    const tasksRaw = parsed['tasks'];
+    if (!Array.isArray(tasksRaw) || tasksRaw.length === 0) {
+        throw new Error(`suite ${suitePath}: tasks must be a non-empty array`);
+    }
+    const seenIds = new Set();
+    try {
+        const tasks = tasksRaw.map((task, index) => parseTask(task, index, seenIds));
+        return { name, tasks };
+    }
+    catch (error) {
+        throw new Error(`suite ${suitePath}: ${message(error)}`);
+    }
+}
+/** Fixture root shipped alongside a suite file: <suiteDir>/../fixtures. */
+export function suiteFixturesDir(suitePath) {
+    return path.resolve(path.dirname(suitePath), '..', 'fixtures');
+}

package/dist/bench/types.d.ts

@@ -0,0 +1,109 @@
+import type { Mode } from '../engine/types.ts';
+import type { UsageTotals } from '../claude/transcripts.ts';
+/** How a task's outcome is judged. Binary, no vibes. */
+export type TaskCheck = 
+/** run in the cell workspace; exit 0 = pass */
+{
+    kind: 'command';
+    command: string;
+}
+/** matched against the model's final answer text (output-heavy tasks) */
+ | {
+    kind: 'answer-regex';
+    pattern: string;
+    flags?: string;
+};
+export interface TaskSpec {
+    id: string;
+    prompt: string;
+    /**
+     * Follow-up prompts forming a scripted multi-turn conversation: the runner
+     * sends `prompt`, then each entry via `claude -p --resume <session-id>`.
+     * Absent = single-shot (existing behavior).
+     */
+    turns?: string[];
+    /** directory name under bench/fixtures/ copied into the cell workspace */
+    fixture: string;
+    check: TaskCheck;
+    /** e.g. ['log-heavy', 'output-heavy', 'large-file'] — reporting only */
+    tags?: string[];
+}
+export interface SuiteSpec {
+    name: string;
+    tasks: TaskSpec[];
+}
+/**
+ * One experimental condition. Modes map to variants 1:1; ablations add
+ * variants like 'optimized-minus-out.no-preamble' (that atom removed) and
+ * 'optimized-plus-tokens.drop-articles' (a rejected atom added back).
+ */
+export interface Variant {
+    id: string;
+    baseMode: Mode;
+    /** rendered output-style body to install; null = no artifacts (full baseline) */
+    styleBody: string | null;
+    /** style name for settings.outputStyle; null for full */
+    styleName: string | null;
+    /** install the compression hook in this cell */
+    hook: boolean;
+    /**
+     * Extra args appended to the hook command (e.g. '--marker-style informative')
+     * so experiments can vary engine behavior per variant.
+     */
+    hookArgs?: string;
+}
+export interface CellSpec {
+    task: TaskSpec;
+    variant: Variant;
+    trial: number;
+    /** requested model */
+    model: string;
+}
+export interface CellResult {
+    runId: string;
+    taskId: string;
+    variantId: string;
+    trial: number;
+    model: string;
+    /** modelUsage keys from the result JSON — fallback can silently substitute */
+    servedModels: string[];
+    /**
+     * For command checks: did the check pass BEFORE the agent ran? A bugfix
+     * fixture whose check already passes is vacuous — report flags these.
+     */
+    baselineCheckPassed: boolean | null;
+    /** null = the check itself errored (infra problem, not task failure) */
+    success: boolean | null;
+    usage: UsageTotals;
+    costUsd: number | null;
+    durationMs: number;
+    numTurns: number;
+    /**
+     * Count of permission_denials in the result JSON. Non-zero denials corrupt
+     * the measurement (the model burns turns retrying instead of working) —
+     * the report flags them as a data-quality problem.
+     */
+    permissionDenials: number;
+    /**
+     * Per-turn usage for multi-turn cells (one entry per scripted turn, from
+     * each turn's result JSON). Absent for single-shot cells. Cell-level
+     * `usage` stays authoritative (summed from the final transcript).
+     */
+    turnUsage?: UsageTotals[];
+    /** tool_use counts by tool name, from the session transcript */
+    toolCalls: Record<string, number>;
+    sessionId: string | null;
+    /** cell-level infrastructure failure (claude crashed, timeout, parse error) */
+    error?: string;
+    timestamp: string;
+}
+export interface RunMeta {
+    runId: string;
+    suite: string;
+    variantIds: string[];
+    model: string;
+    trials: number;
+    startedAt: string;
+    /** hard cost ceiling; the runner stops scheduling cells when exceeded */
+    maxBudgetUsd: number;
+}

package/dist/bench/types.js

@@ -0,0 +1 @@
+export {};

package/dist/claude/transcripts.d.ts

@@ -0,0 +1,30 @@
+export interface UsageTotals {
+    input: number;
+    output: number;
+    cacheCreation: number;
+    cacheRead: number;
+}
+export interface SessionUsage {
+    sessionId: string;
+    file: string;
+    turns: number;
+    totals: UsageTotals;
+    byModel: Record<string, UsageTotals>;
+    sidechain: UsageTotals;
+    firstTimestamp?: string;
+    lastTimestamp?: string;
+}
+/**
+ * Claude Code encodes the project cwd as replace(/[^a-zA-Z0-9]/g, '-'); when
+ * the result exceeds 200 chars it is truncated and suffixed with a base36
+ * hash of the ORIGINAL path (verified against the 2.1.170 binary).
+ */
+export declare function encodeProjectDir(absPath: string): string;
+export declare function addUsage(a: UsageTotals, b: UsageTotals): UsageTotals;
+export declare function aggregateUsage(sessions: SessionUsage[]): UsageTotals;
+export declare function findTranscripts(opts: {
+    projectDir: string;
+    configDir?: string;
+    since?: Date;
+}): Promise<string[]>;
+export declare function readSessionUsage(file: string): Promise<SessionUsage>;

package/dist/claude/transcripts.js

@@ -0,0 +1,154 @@
+import { createReadStream } from 'node:fs';
+import { readdir, stat } from 'node:fs/promises';
+import { homedir } from 'node:os';
+import { basename, join } from 'node:path';
+import { createInterface } from 'node:readline';
+const ENCODED_DIR_MAX = 200;
+/** 32-bit java-style string hash, matching Claude Code's long-path suffix. */
+function hash32(text) {
+    let h = 0;
+    for (let i = 0; i < text.length; i += 1) {
+        h = ((h << 5) - h + text.charCodeAt(i)) | 0;
+    }
+    return h;
+}
+/**
+ * Claude Code encodes the project cwd as replace(/[^a-zA-Z0-9]/g, '-'); when
+ * the result exceeds 200 chars it is truncated and suffixed with a base36
+ * hash of the ORIGINAL path (verified against the 2.1.170 binary).
+ */
+export function encodeProjectDir(absPath) {
+    const dashed = absPath.replace(/[^a-zA-Z0-9]/g, '-');
+    if (dashed.length <= ENCODED_DIR_MAX) {
+        return dashed;
+    }
+    return `${dashed.slice(0, ENCODED_DIR_MAX)}-${Math.abs(hash32(absPath)).toString(36)}`;
+}
+const emptyTotals = () => ({
+    input: 0,
+    output: 0,
+    cacheCreation: 0,
+    cacheRead: 0,
+});
+export function addUsage(a, b) {
+    return {
+        input: a.input + b.input,
+        output: a.output + b.output,
+        cacheCreation: a.cacheCreation + b.cacheCreation,
+        cacheRead: a.cacheRead + b.cacheRead,
+    };
+}
+export function aggregateUsage(sessions) {
+    return sessions.reduce((acc, s) => addUsage(acc, s.totals), emptyTotals());
+}
+export async function findTranscripts(opts) {
+    const configDir = opts.configDir ?? process.env.CLAUDE_CONFIG_DIR ?? join(homedir(), '.claude');
+    const dir = join(configDir, 'projects', encodeProjectDir(opts.projectDir));
+    let names;
+    try {
+        names = await readdir(dir);
+    }
+    catch {
+        return [];
+    }
+    const files = [];
+    for (const name of names.filter((n) => n.endsWith('.jsonl')).sort()) {
+        const full = join(dir, name);
+        if (opts.since) {
+            try {
+                const info = await stat(full);
+                if (info.mtimeMs < opts.since.getTime())
+                    continue;
+            }
+            catch {
+                continue;
+            }
+        }
+        files.push(full);
+    }
+    return files;
+}
+function isRecord(value) {
+    return typeof value === 'object' && value !== null;
+}
+function num(value) {
+    return typeof value === 'number' && Number.isFinite(value) ? value : 0;
+}
+function usageFrom(usage) {
+    return {
+        input: num(usage['input_tokens']),
+        output: num(usage['output_tokens']),
+        cacheCreation: num(usage['cache_creation_input_tokens']),
+        cacheRead: num(usage['cache_read_input_tokens']),
+    };
+}
+export async function readSessionUsage(file) {
+    // The same requestId appears on multiple lines for streamed updates; the
+    // last occurrence carries final usage, so Map.set overwrites earlier ones.
+    const turns = new Map();
+    let sessionId = '';
+    let firstTimestamp;
+    let lastTimestamp;
+    let anonCounter = 0;
+    const rl = createInterface({
+        input: createReadStream(file, 'utf8'),
+        crlfDelay: Infinity,
+    });
+    for await (const line of rl) {
+        const trimmed = line.trim();
+        if (trimmed === '')
+            continue;
+        let parsed;
+        try {
+            parsed = JSON.parse(trimmed);
+        }
+        catch {
+            continue;
+        }
+        if (!isRecord(parsed) || parsed['type'] !== 'assistant')
+            continue;
+        const message = parsed['message'];
+        if (!isRecord(message))
+            continue;
+        const usage = message['usage'];
+        if (!isRecord(usage))
+            continue;
+        if (sessionId === '' && typeof parsed['sessionId'] === 'string') {
+            sessionId = parsed['sessionId'];
+        }
+        const ts = parsed['timestamp'];
+        if (typeof ts === 'string') {
+            firstTimestamp ??= ts;
+            lastTimestamp = ts;
+        }
+        const key = typeof parsed['requestId'] === 'string'
+            ? parsed['requestId']
+            : typeof message['id'] === 'string'
+                ? message['id']
+                : `anon-${anonCounter++}`;
+        turns.set(key, {
+            usage: usageFrom(usage),
+            model: typeof message['model'] === 'string' ? message['model'] : 'unknown',
+            sidechain: parsed['isSidechain'] === true,
+        });
+    }
+    let totals = emptyTotals();
+    let sidechain = emptyTotals();
+    const byModel = {};
+    for (const turn of turns.values()) {
+        totals = addUsage(totals, turn.usage);
+        byModel[turn.model] = addUsage(byModel[turn.model] ?? emptyTotals(), turn.usage);
+        if (turn.sidechain)
+            sidechain = addUsage(sidechain, turn.usage);
+    }
+    return {
+        sessionId: sessionId === '' ? basename(file, '.jsonl') : sessionId,
+        file,
+        turns: turns.size,
+        totals,
+        byModel,
+        sidechain,
+        firstTimestamp,
+        lastTimestamp,
+    };
+}

package/dist/cli/commands/benchmark.d.ts

@@ -0,0 +1,33 @@
+export interface BenchmarkCliOptions {
+    suite: string;
+    modes: string;
+    trials: string;
+    model: string;
+    ablate?: string;
+    ablateAdd?: string;
+    /** comma-separated atom categories (output|behavior) for group ablation */
+    ablateGroup?: string;
+    /** commander --no-hook: defaults true */
+    hook: boolean;
+    /** extra args appended to the hook command in every hook-bearing variant */
+    hookArgs?: string;
+    /**
+     * comma-separated marker styles: fans each hook-bearing variant out into
+     * one arm per style WITHIN this run (shared budget ceiling, balanced groups)
+     */
+    markerStyles?: string;
+    concurrency: string;
+    maxBudgetUsd: string;
+    out: string;
+}
+/**
+ * Preflight for marker-style experiments: the hook entry parses argv
+ * fail-open, so a STALE dist/hook.js that predates --marker-style ignores
+ * the flag silently and every arm measures identical 'plain' behavior — a
+ * three-arm run of pure noise with zero errors anywhere. Verify the exact
+ * installed hook command by piping the same over-budget payload through it
+ * with two different styles and requiring the outputs to differ.
+ * COMPRESSOR_NO_LEDGER keeps the probe out of the live savings ledger.
+ */
+export declare function assertHookHandlesMarkerStyle(baseHookCommand: string): Promise<void>;
+export declare function runBenchmarkCommand(opts: BenchmarkCliOptions): Promise<void>;