@inbrowser/agent 0.1.0 → 0.2.0

package/dist/dispatch-memoization.js

@@ -0,0 +1,197 @@
+/**
+ * Content-addressed memoization layer over `createDispatch`.
+ *
+ * `createMemoizedDispatch(registry, options?)` returns a `ToolDispatch`-shaped
+ * object that caches `ToolResult`s keyed on `(toolName, argsHash,
+ * workspaceHash, runtimeHash)`. The cache is consulted only for handlers
+ * tagged `pure` (see `isPure` in `./tools.ts`). Non-pure handlers bypass
+ * the cache entirely and always execute. Errors from the underlying
+ * dispatch propagate; they are NOT cached, since they may be transient.
+ *
+ * The returned object is structurally a `ToolDispatch` — strategies and
+ * downstream code that already accept `ToolDispatch` use it transparently.
+ * The one addition is `stats()`, which returns the running counters for
+ * hits / misses / bypassed calls. The cache lives for the lifetime of
+ * one `MemoizedDispatch` instance; there is no global state.
+ *
+ * Design notes:
+ *
+ * - Hashing uses FNV-1a 32-bit over a stable-stringified JSON
+ *   representation. The cache is for short-running test loops; a
+ *   cryptographic hash is overkill. Collisions are tolerable at our
+ *   cache sizes, and the cost of a missed hit is at worst a recomputation.
+ * - Argument keys are sorted at every level via `stableStringify` so two
+ *   structurally-equal arg objects produce the same key regardless of
+ *   property insertion order.
+ * - Workspace hash covers `presetId`, `rules`, `code`, and `appSource`.
+ *   `stitch` is excluded per the brief — pure tools don't read from it.
+ * - Runtime hash is included only when `'runtime' \in keyComponents`.
+ *   Defaults to `['workspace']`; opting into runtime opt-in keeps the
+ *   default key small for the dominant pure-tool population.
+ * - No eviction in v1. Eval runs are bounded; one instance per harness
+ *   trial keeps cache growth bounded too.
+ */
+import { createDispatch, isPure } from './tools.js';
+/**
+ * Wrap a registry in a memoizing dispatcher. The wrapper holds its own
+ * cache; the underlying dispatch is the standard `createDispatch(registry)`.
+ *
+ * Non-pure handlers (including unknown-tool errors) bypass the cache and
+ * are dispatched directly; `bypassed` is incremented for those calls.
+ */
+export function createMemoizedDispatch(registry, options) {
+    const keyComponents = options?.keyComponents ?? ['workspace'];
+    const includeRuntime = keyComponents.includes('runtime');
+    const includeWorkspace = keyComponents.includes('workspace');
+    const underlying = createDispatch(registry);
+    const cache = new Map();
+    const counters = { hits: 0, misses: 0, bypassed: 0 };
+    return {
+        async execute(call, ctx) {
+            const handler = findHandler(registry, call.name);
+            // Non-pure handlers (and unknown tools) skip the cache. Unknown
+            // tools surface their error message through the underlying
+            // dispatch unchanged — the cache layer is invisible on the
+            // non-pure path.
+            if (!handler || !isPure(handler)) {
+                counters.bypassed += 1;
+                return underlying.execute(call, ctx);
+            }
+            const key = buildCacheKey(call, ctx, includeWorkspace, includeRuntime);
+            const cached = cache.get(key);
+            if (cached !== undefined) {
+                counters.hits += 1;
+                return cached;
+            }
+            // Cache miss. Underlying dispatch runs; the result is cached on
+            // success. Note: `createDispatch` already converts thrown
+            // handlers into `{ ok: false, summary: '... threw: ...' }`
+            // results, so we never observe a thrown error here. The brief
+            // says "errors propagate, are not cached" — that maps to
+            // `ok === false` here. We cache only successful results.
+            const result = await underlying.execute(call, ctx);
+            counters.misses += 1;
+            if (result.ok) {
+                cache.set(key, result);
+            }
+            return result;
+        },
+        stats() {
+            return { ...counters };
+        },
+    };
+}
+/**
+ * Build the deterministic cache key for `(toolName, argsHash,
+ * workspaceHash, runtimeHash)`. Components are joined with `|` so a
+ * single string is hashable in one pass on lookup.
+ */
+function buildCacheKey(call, ctx, includeWorkspace, includeRuntime) {
+    const argsKey = hashFnv1a32(stableStringify(call.args));
+    const wsKey = includeWorkspace ? hashFnv1a32(stableStringify(workspaceShape(ctx.workspace))) : '';
+    const rtKey = includeRuntime ? hashFnv1a32(stableStringify(runtimeShape(ctx.runtime))) : '';
+    return `${call.name}|${argsKey}|${wsKey}|${rtKey}`;
+}
+/**
+ * Project the workspace into the subset of fields a pure tool can
+ * legitimately read. `stitch` is excluded — design context is
+ * orthogonal to the documented pure-tool population (rules-stdlib-list,
+ * path-discovery, etc.). Two workspaces that differ only in `stitch`
+ * are treated as equivalent for cache purposes.
+ */
+function workspaceShape(ws) {
+    if (!ws)
+        return { _present: false };
+    return {
+        presetId: ws.presetId,
+        rules: ws.rules,
+        code: ws.code,
+        appSource: ws.appSource,
+    };
+}
+/**
+ * Project the runtime into a stable shape. Only included in the cache
+ * key when the caller opts into `'runtime'` in `keyComponents`.
+ */
+function runtimeShape(rt) {
+    if (!rt)
+        return { _present: false };
+    return {
+        terminal: rt.terminal,
+        runSummary: rt.runSummary,
+        deploy: rt.deploy,
+        parseError: rt.parseError,
+        uiErrors: rt.uiErrors,
+        sandboxVersion: rt.sandboxVersion,
+    };
+}
+/**
+ * Stable JSON serialization: object keys are sorted alphabetically at
+ * every nesting level. Arrays preserve order (they are positional).
+ * Functions, symbols, `undefined` properties are omitted (standard
+ * JSON behaviour). `null` is preserved.
+ *
+ * This is intentionally not `JSON.stringify(value)` — that emits keys
+ * in insertion order, so two structurally-equal arg objects assembled
+ * differently would produce different cache keys.
+ */
+export function stableStringify(value) {
+    return serialize(value);
+}
+function serialize(value) {
+    if (value === null)
+        return 'null';
+    if (value === undefined)
+        return 'undefined';
+    if (typeof value === 'number') {
+        return Number.isFinite(value) ? JSON.stringify(value) : 'null';
+    }
+    if (typeof value === 'string' || typeof value === 'boolean') {
+        return JSON.stringify(value);
+    }
+    if (Array.isArray(value)) {
+        return `[${value.map(serialize).join(',')}]`;
+    }
+    if (typeof value === 'object') {
+        const obj = value;
+        const keys = Object.keys(obj).sort();
+        const parts = [];
+        for (const k of keys) {
+            const v = obj[k];
+            if (v === undefined)
+                continue;
+            parts.push(`${JSON.stringify(k)}:${serialize(v)}`);
+        }
+        return `{${parts.join(',')}}`;
+    }
+    // Functions, symbols, bigints — fall back to a stable string form.
+    // Pure tool args should never contain these; the fallback is defensive.
+    return JSON.stringify(String(value));
+}
+/**
+ * FNV-1a 32-bit hash. Returns the lowercase hex string. Fast,
+ * dependency-free, and collision-tolerable at our cache sizes.
+ * Iterates the UTF-16 code units of the input; sufficient for our
+ * stably-stringified JSON payloads, which only contain ASCII control
+ * characters and JSON syntax tokens plus user-supplied string data.
+ */
+export function hashFnv1a32(input) {
+    let hash = 0x811c9dc5; // FNV offset basis (32-bit)
+    for (let i = 0; i < input.length; i += 1) {
+        hash ^= input.charCodeAt(i);
+        // 32-bit FNV prime multiplication via shifts (avoids precision loss).
+        hash = (hash + ((hash << 1) + (hash << 4) + (hash << 7) + (hash << 8) + (hash << 24))) >>> 0;
+    }
+    return hash.toString(16).padStart(8, '0');
+}
+/**
+ * Lookup a handler by name without exposing a `get` method on the
+ * registry interface. Mirrors the helper in `tools.ts`; duplicated here
+ * to keep the memoization module independent of internal helpers.
+ */
+function findHandler(registry, name) {
+    if (!registry.has(name))
+        return undefined;
+    return registry.list().find((h) => h.name === name);
+}
+//# sourceMappingURL=dispatch-memoization.js.map

package/dist/dispatch-memoization.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"dispatch-memoization.js","sourceRoot":"","sources":["../src/dispatch-memoization.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAgCG;AAEH,OAAO,EAAE,cAAc,EAAE,MAAM,EAAE,MAAM,YAAY,CAAC;AAsCpD;;;;;;GAMG;AACH,MAAM,UAAU,sBAAsB,CACpC,QAAsB,EACtB,OAAqB;IAErB,MAAM,aAAa,GAAuB,OAAO,EAAE,aAAa,IAAI,CAAC,WAAW,CAAC,CAAC;IAClF,MAAM,cAAc,GAAG,aAAa,CAAC,QAAQ,CAAC,SAAS,CAAC,CAAC;IACzD,MAAM,gBAAgB,GAAG,aAAa,CAAC,QAAQ,CAAC,WAAW,CAAC,CAAC;IAE7D,MAAM,UAAU,GAAG,cAAc,CAAC,QAAQ,CAAC,CAAC;IAC5C,MAAM,KAAK,GAAG,IAAI,GAAG,EAAsB,CAAC;IAC5C,MAAM,QAAQ,GAAc,EAAE,IAAI,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,QAAQ,EAAE,CAAC,EAAE,CAAC;IAEhE,OAAO;QACL,KAAK,CAAC,OAAO,CAAC,IAAc,EAAE,GAAgB;YAC5C,MAAM,OAAO,GAAG,WAAW,CAAC,QAAQ,EAAE,IAAI,CAAC,IAAI,CAAC,CAAC;YACjD,gEAAgE;YAChE,2DAA2D;YAC3D,2DAA2D;YAC3D,iBAAiB;YACjB,IAAI,CAAC,OAAO,IAAI,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,CAAC;gBACjC,QAAQ,CAAC,QAAQ,IAAI,CAAC,CAAC;gBACvB,OAAO,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACvC,CAAC;YAED,MAAM,GAAG,GAAG,aAAa,CAAC,IAAI,EAAE,GAAG,EAAE,gBAAgB,EAAE,cAAc,CAAC,CAAC;YACvE,MAAM,MAAM,GAAG,KAAK,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;YAC9B,IAAI,MAAM,KAAK,SAAS,EAAE,CAAC;gBACzB,QAAQ,CAAC,IAAI,IAAI,CAAC,CAAC;gBACnB,OAAO,MAAM,CAAC;YAChB,CAAC;YAED,gEAAgE;YAChE,0DAA0D;YAC1D,2DAA2D;YAC3D,8DAA8D;YAC9D,yDAAyD;YACzD,yDAAyD;YACzD,MAAM,MAAM,GAAG,MAAM,UAAU,CAAC,OAAO,CAAC,IAAI,EAAE,GAAG,CAAC,CAAC;YACnD,QAAQ,CAAC,MAAM,IAAI,CAAC,CAAC;YACrB,IAAI,MAAM,CAAC,EAAE,EAAE,CAAC;gBACd,KAAK,CAAC,GAAG,CAAC,GAAG,EAAE,MAAM,CAAC,CAAC;YACzB,CAAC;YACD,OAAO,MAAM,CAAC;QAChB,CAAC;QACD,KAAK;YACH,OAAO,EAAE,GAAG,QAAQ,EAAE,CAAC;QACzB,CAAC;KACF,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,SAAS,aAAa,CACpB,IAAc,EACd,GAAgB,EAChB,gBAAyB,EACzB,cAAuB;IAEvB,MAAM,OAAO,GAAG,WAAW,CAAC,eAAe,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,CAAC;IACxD,MAAM,KAAK,GAAG,gBAAgB,CAAC,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,cAAc,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAClG,MAAM,KAAK,GAAG,cAAc,CAAC,CAAC,CAAC,WAAW,CAAC,eAAe,CAAC,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC5F,OAAO,GAAG,IAAI,CAAC,IAAI,IAAI,OAAO,IAAI,KAAK,IAAI,KAAK,EAAE,CAAC;AACrD,CAAC;AAED;;;;;;GAMG;AACH,SAAS,cAAc,CAAC,EAAc;IACpC,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACpC,OAAO;QACL,QAAQ,EAAE,EAAE,CAAC,QAAQ;QACrB,KAAK,EAAE,EAAE,CAAC,KAAK;QACf,IAAI,EAAE,EAAE,CAAC,IAAI;QACb,SAAS,EAAE,EAAE,CAAC,SAAS;KACxB,CAAC;AACJ,CAAC;AAED;;;GAGG;AACH,SAAS,YAAY,CAAC,EAAiB;IACrC,IAAI,CAAC,EAAE;QAAE,OAAO,EAAE,QAAQ,EAAE,KAAK,EAAE,CAAC;IACpC,OAAO;QACL,QAAQ,EAAE,EAAE,CAAC,QAAQ;QACrB,UAAU,EAAE,EAAE,CAAC,UAAU;QACzB,MAAM,EAAE,EAAE,CAAC,MAAM;QACjB,UAAU,EAAE,EAAE,CAAC,UAAU;QACzB,QAAQ,EAAE,EAAE,CAAC,QAAQ;QACrB,cAAc,EAAE,EAAE,CAAC,cAAc;KAClC,CAAC;AACJ,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,OAAO,SAAS,CAAC,KAAK,CAAC,CAAC;AAC1B,CAAC;AAED,SAAS,SAAS,CAAC,KAAc;IAC/B,IAAI,KAAK,KAAK,IAAI;QAAE,OAAO,MAAM,CAAC;IAClC,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,WAAW,CAAC;IAC5C,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,OAAO,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC;IACjE,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,IAAI,OAAO,KAAK,KAAK,SAAS,EAAE,CAAC;QAC5D,OAAO,IAAI,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC;IAC/B,CAAC;IACD,IAAI,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,EAAE,CAAC;QACzB,OAAO,IAAI,KAAK,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAC/C,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,GAAG,GAAG,KAAgC,CAAC;QAC7C,MAAM,IAAI,GAAG,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,IAAI,EAAE,CAAC;QACrC,MAAM,KAAK,GAAa,EAAE,CAAC;QAC3B,KAAK,MAAM,CAAC,IAAI,IAAI,EAAE,CAAC;YACrB,MAAM,CAAC,GAAG,GAAG,CAAC,CAAC,CAAC,CAAC;YACjB,IAAI,CAAC,KAAK,SAAS;gBAAE,SAAS;YAC9B,KAAK,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC,SAAS,CAAC,CAAC,CAAC,IAAI,SAAS,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC;QACrD,CAAC;QACD,OAAO,IAAI,KAAK,CAAC,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC;IAChC,CAAC;IACD,mEAAmE;IACnE,wEAAwE;IACxE,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC;AACvC,CAAC;AAED;;;;;;GAMG;AACH,MAAM,UAAU,WAAW,CAAC,KAAa;IACvC,IAAI,IAAI,GAAG,UAAU,CAAC,CAAC,4BAA4B;IACnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,KAAK,CAAC,MAAM,EAAE,CAAC,IAAI,CAAC,EAAE,CAAC;QACzC,IAAI,IAAI,KAAK,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC;QAC5B,sEAAsE;QACtE,IAAI,GAAG,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,IAAI,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC;IAC/F,CAAC;IACD,OAAO,IAAI,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC,QAAQ,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC;AAC5C,CAAC;AAED;;;;GAIG;AACH,SAAS,WAAW,CAAC,QAAsB,EAAE,IAAY;IACvD,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,CAAC;QAAE,OAAO,SAAS,CAAC;IAC1C,OAAO,QAAQ,CAAC,IAAI,EAAE,CAAC,IAAI,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,KAAK,IAAI,CAAC,CAAC;AACtD,CAAC"}

package/dist/eval/comparison-report.d.ts

@@ -0,0 +1,164 @@
+/**
+ * `compareMetrics` — the eval harness's A/B comparison report.
+ *
+ * Consumes two named `MetricsTable[]`s (typically a "baseline" and a
+ * "variant"), pairs them by `fixtureId`, and for each per-metric
+ * column emits a row containing the inputs (mean +/- spread for both
+ * sides), the delta (`variantMean - baselineMean`), the threshold
+ * (`max(baselineSpread, variantSpread)`), and a label.
+ *
+ * The label uses the implementation plan's no-effect rule:
+ *
+ *   - If `Math.abs(delta) < threshold`, label `no-effect`. This is the
+ *     central discipline: noisy trial-to-trial variance must not be
+ *     reported as a winner.
+ *   - Otherwise consult the metric's polarity (see `POLARITY` below):
+ *       higher-is-better -> sign of `delta` picks the winner;
+ *       lower-is-better  -> sign of `delta` picks the loser;
+ *       neutral          -> emit `changed` (no winner, just a flag).
+ *
+ * Polarity is a static map declared in this module. Adding a new
+ * metric to the collector forces an entry here because the map is
+ * keyed by a string-literal union derived from `MetricsTable` fields.
+ *
+ * Two renderers ship with the comparator:
+ *
+ *   - `renderMarkdown(report)` — a human-readable markdown table for
+ *     piping to stdout from a CLI consumer.
+ *   - `renderJson(report)` — `JSON.stringify(report, null, 2)`. Plain
+ *     data; round-trips through `JSON.parse(JSON.stringify(report))`
+ *     without loss.
+ *
+ * No statistical sophistication beyond the no-effect rule. Confidence
+ * intervals, cross-fixture aggregation, and multi-variant comparisons
+ * are deliberately deferred to follow-up branches.
+ *
+ * Browser-safe — no Node imports.
+ *
+ * Note on naming: metric names match the collector exactly. Do not
+ * rename. The CLI, the report, and any later UI all key on these
+ * identifiers, and renaming in one layer without the others silently
+ * breaks consumers.
+ */
+import type { AggregateStat, MetricsTable } from './metric-collector.js';
+/**
+ * The set of metric names a comparison row can target. Each name is
+ * the dotted path from `AggregatedMetrics` down to an `AggregateStat`.
+ * `toolCallCount` is a nested object on the aggregate; the three
+ * children each get their own polarity entry.
+ *
+ * Keep this in lockstep with `AggregatedMetrics` in
+ * `./metric-collector.ts`. The compiler enforces exhaustiveness on
+ * `POLARITY` below.
+ */
+export type ComparisonMetricName = 'taskSuccessRate' | 'wallClockMs' | 'promptTokens' | 'completionTokens' | 'toolCallCount.total' | 'toolCallCount.reads' | 'toolCallCount.mutations' | 'turnCount' | 'peakContextWindowBytes' | 'truthfulnessViolationRate' | 'dispatchVsLlmRatio';
+/** Direction in which "more" is better, worse, or neither. */
+export type Polarity = 'higher-is-better' | 'lower-is-better' | 'neutral';
+/**
+ * Static polarity table. Choices are documented per row; the
+ * neutral defaults are the metrics the implementation plan
+ * explicitly says are context-dependent.
+ *
+ * Adding a new metric to the collector requires adding an entry
+ * here. The `Record<ComparisonMetricName, Polarity>` shape keeps the
+ * compiler honest if `ComparisonMetricName` is extended.
+ */
+export declare const POLARITY: Record<ComparisonMetricName, Polarity>;
+/** Label assigned to a per-metric row after applying the no-effect rule. */
+export type ComparisonLabel = 'no-effect' | 'winner-baseline' | 'winner-variant' | 'changed';
+/**
+ * A single per-fixture, per-metric row.
+ *
+ * `delta` is `variantMean - baselineMean`. Both means are passed
+ * through verbatim from the input aggregates and may be `undefined`
+ * (the collector returns `undefined` for metrics it could not
+ * compute). When either mean is `undefined`, the row's `label` is
+ * `no-effect` and `delta` / `threshold` are `undefined` — there is
+ * nothing to compare.
+ */
+export interface ComparisonRow {
+    fixtureId: string;
+    metric: ComparisonMetricName;
+    polarity: Polarity;
+    baseline: AggregateStat;
+    variant: AggregateStat;
+    /** `variantMean - baselineMean`. `undefined` when either mean is `undefined`. */
+    delta: number | undefined;
+    /** `max(baselineSpread, variantSpread)`. `undefined` when either spread is `undefined`. */
+    threshold: number | undefined;
+    label: ComparisonLabel;
+}
+/** Per-fixture grouping. Carries the comparison rows plus a coverage status. */
+export interface ComparisonFixture {
+    fixtureId: string;
+    /**
+     * `both` when the fixture appears in both inputs;
+     * `baseline-only` / `variant-only` when one side is missing the
+     * fixture. The missing-side cases carry no per-metric rows.
+     */
+    status: 'both' | 'baseline-only' | 'variant-only';
+    rows: ComparisonRow[];
+}
+/** The full report. Suitable for both renderers. */
+export interface ComparisonReport {
+    /** Label for the left-hand side. Defaults to `'baseline'`. */
+    baselineName: string;
+    /** Label for the right-hand side. Defaults to `'variant'`. */
+    variantName: string;
+    /** One entry per fixture across both inputs, in baseline-first union order. */
+    fixtures: ComparisonFixture[];
+}
+/** Input to `compareMetrics`. */
+export interface CompareMetricsInput {
+    baseline: readonly MetricsTable[];
+    variant: readonly MetricsTable[];
+    /** Optional label for the baseline column. Defaults to `'baseline'`. */
+    baselineName?: string;
+    /** Optional label for the variant column. Defaults to `'variant'`. */
+    variantName?: string;
+    /**
+     * Override entries in the static `POLARITY` map for this report.
+     * Useful when a specific experiment has a directional read on a
+     * normally-neutral metric (e.g. parallel dispatch expects
+     * `toolCallCount.reads` to stay flat — which is still neutral —
+     * but a memoization experiment might prefer lower `wallClockMs`
+     * exclusively without changing other directions; in practice the
+     * defaults are correct and this is rarely needed).
+     */
+    polarityOverrides?: Partial<Record<ComparisonMetricName, Polarity>>;
+}
+/**
+ * Build a `ComparisonReport` from two named metric tables.
+ *
+ * Fixtures present in both sides are paired; fixtures unique to one
+ * side surface with an explicit `status` and no rows. Within each
+ * paired fixture, every entry in `POLARITY` produces one row.
+ *
+ * Never throws on missing data — `undefined` means / spreads carry
+ * through to `undefined` deltas / thresholds and a `no-effect` label.
+ */
+export declare function compareMetrics(input: CompareMetricsInput): ComparisonReport;
+/**
+ * Render a `ComparisonReport` as plain JSON.
+ *
+ * Implementation is `JSON.stringify(report, null, 2)`. The report is
+ * pure data: no functions, no `undefined` in places where the
+ * renderer cares (numeric fields surface as `null` after a round-trip
+ * but the consumer treats `null` and missing the same way).
+ */
+export declare function renderJson(report: ComparisonReport): string;
+/**
+ * Render a `ComparisonReport` as a markdown document.
+ *
+ * One section per fixture. Each section has a header with the
+ * fixture id and its coverage status, then a table with seven
+ * columns: metric, baseline (mean +/- spread), variant (mean +/-
+ * spread), delta, threshold, polarity, label.
+ *
+ * Numeric formatting:
+ *   - mean / spread use up to four significant digits;
+ *   - `undefined` renders as `-`;
+ *   - delta and threshold use the same formatter as mean.
+ */
+export declare function renderMarkdown(report: ComparisonReport): string;
+//# sourceMappingURL=comparison-report.d.ts.map

package/dist/eval/comparison-report.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-report.d.ts","sourceRoot":"","sources":["../../src/eval/comparison-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,YAAY,EAAE,MAAM,uBAAuB,CAAC;AAIzE;;;;;;;;;GASG;AACH,MAAM,MAAM,oBAAoB,GAC5B,iBAAiB,GACjB,aAAa,GACb,cAAc,GACd,kBAAkB,GAClB,qBAAqB,GACrB,qBAAqB,GACrB,yBAAyB,GACzB,WAAW,GACX,wBAAwB,GACxB,2BAA2B,GAC3B,oBAAoB,CAAC;AAEzB,8DAA8D;AAC9D,MAAM,MAAM,QAAQ,GAAG,kBAAkB,GAAG,iBAAiB,GAAG,SAAS,CAAC;AAE1E;;;;;;;;GAQG;AACH,eAAO,MAAM,QAAQ,EAAE,MAAM,CAAC,oBAAoB,EAAE,QAAQ,CAqB3D,CAAC;AAIF,4EAA4E;AAC5E,MAAM,MAAM,eAAe,GAAG,WAAW,GAAG,iBAAiB,GAAG,gBAAgB,GAAG,SAAS,CAAC;AAE7F;;;;;;;;;GASG;AACH,MAAM,WAAW,aAAa;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,MAAM,EAAE,oBAAoB,CAAC;IAC7B,QAAQ,EAAE,QAAQ,CAAC;IACnB,QAAQ,EAAE,aAAa,CAAC;IACxB,OAAO,EAAE,aAAa,CAAC;IACvB,iFAAiF;IACjF,KAAK,EAAE,MAAM,GAAG,SAAS,CAAC;IAC1B,2FAA2F;IAC3F,SAAS,EAAE,MAAM,GAAG,SAAS,CAAC;IAC9B,KAAK,EAAE,eAAe,CAAC;CACxB;AAED,gFAAgF;AAChF,MAAM,WAAW,iBAAiB;IAChC,SAAS,EAAE,MAAM,CAAC;IAClB;;;;OAIG;IACH,MAAM,EAAE,MAAM,GAAG,eAAe,GAAG,cAAc,CAAC;IAClD,IAAI,EAAE,aAAa,EAAE,CAAC;CACvB;AAED,oDAAoD;AACpD,MAAM,WAAW,gBAAgB;IAC/B,8DAA8D;IAC9D,YAAY,EAAE,MAAM,CAAC;IACrB,8DAA8D;IAC9D,WAAW,EAAE,MAAM,CAAC;IACpB,+EAA+E;IAC/E,QAAQ,EAAE,iBAAiB,EAAE,CAAC;CAC/B;AAID,iCAAiC;AACjC,MAAM,WAAW,mBAAmB;IAClC,QAAQ,EAAE,SAAS,YAAY,EAAE,CAAC;IAClC,OAAO,EAAE,SAAS,YAAY,EAAE,CAAC;IACjC,wEAAwE;IACxE,YAAY,CAAC,EAAE,MAAM,CAAC;IACtB,sEAAsE;IACtE,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB;;;;;;;;OAQG;IACH,iBAAiB,CAAC,EAAE,OAAO,CAAC,MAAM,CAAC,oBAAoB,EAAE,QAAQ,CAAC,CAAC,CAAC;CACrE;AAED;;;;;;;;;GASG;AACH,wBAAgB,cAAc,CAAC,KAAK,EAAE,mBAAmB,GAAG,gBAAgB,CA8C3E;AAgHD;;;;;;;GAOG;AACH,wBAAgB,UAAU,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CAE3D;AAED;;;;;;;;;;;;GAYG;AACH,wBAAgB,cAAc,CAAC,MAAM,EAAE,gBAAgB,GAAG,MAAM,CA6C/D"}