@inbrowser/agent 0.1.0 → 0.2.0

package/dist/eval/comparison-report.js

@@ -0,0 +1,316 @@
+/**
+ * `compareMetrics` — the eval harness's A/B comparison report.
+ *
+ * Consumes two named `MetricsTable[]`s (typically a "baseline" and a
+ * "variant"), pairs them by `fixtureId`, and for each per-metric
+ * column emits a row containing the inputs (mean +/- spread for both
+ * sides), the delta (`variantMean - baselineMean`), the threshold
+ * (`max(baselineSpread, variantSpread)`), and a label.
+ *
+ * The label uses the implementation plan's no-effect rule:
+ *
+ *   - If `Math.abs(delta) < threshold`, label `no-effect`. This is the
+ *     central discipline: noisy trial-to-trial variance must not be
+ *     reported as a winner.
+ *   - Otherwise consult the metric's polarity (see `POLARITY` below):
+ *       higher-is-better -> sign of `delta` picks the winner;
+ *       lower-is-better  -> sign of `delta` picks the loser;
+ *       neutral          -> emit `changed` (no winner, just a flag).
+ *
+ * Polarity is a static map declared in this module. Adding a new
+ * metric to the collector forces an entry here because the map is
+ * keyed by a string-literal union derived from `MetricsTable` fields.
+ *
+ * Two renderers ship with the comparator:
+ *
+ *   - `renderMarkdown(report)` — a human-readable markdown table for
+ *     piping to stdout from a CLI consumer.
+ *   - `renderJson(report)` — `JSON.stringify(report, null, 2)`. Plain
+ *     data; round-trips through `JSON.parse(JSON.stringify(report))`
+ *     without loss.
+ *
+ * No statistical sophistication beyond the no-effect rule. Confidence
+ * intervals, cross-fixture aggregation, and multi-variant comparisons
+ * are deliberately deferred to follow-up branches.
+ *
+ * Browser-safe — no Node imports.
+ *
+ * Note on naming: metric names match the collector exactly. Do not
+ * rename. The CLI, the report, and any later UI all key on these
+ * identifiers, and renaming in one layer without the others silently
+ * breaks consumers.
+ */
+/**
+ * Static polarity table. Choices are documented per row; the
+ * neutral defaults are the metrics the implementation plan
+ * explicitly says are context-dependent.
+ *
+ * Adding a new metric to the collector requires adding an entry
+ * here. The `Record<ComparisonMetricName, Polarity>` shape keeps the
+ * compiler honest if `ComparisonMetricName` is extended.
+ */
+export const POLARITY = {
+    // Higher success rate is strictly better.
+    taskSuccessRate: 'higher-is-better',
+    // Lower latency / token spend / context pressure is better.
+    wallClockMs: 'lower-is-better',
+    promptTokens: 'lower-is-better',
+    completionTokens: 'lower-is-better',
+    peakContextWindowBytes: 'lower-is-better',
+    // Lower violation rate is strictly better (zero is the goal).
+    truthfulnessViolationRate: 'lower-is-better',
+    // Tool-call counts are context-dependent: more reads can mean
+    // better grounding (good) or wasted effort (bad). Neutral by
+    // default. Specific experiments can re-interpret these via the
+    // `polarityOverrides` input.
+    'toolCallCount.total': 'neutral',
+    'toolCallCount.reads': 'neutral',
+    'toolCallCount.mutations': 'neutral',
+    // Turn count is context-dependent for the same reason.
+    turnCount: 'neutral',
+    // Dispatch-vs-LLM ratio is a diagnostic, not a quality metric.
+    dispatchVsLlmRatio: 'neutral',
+};
+/**
+ * Build a `ComparisonReport` from two named metric tables.
+ *
+ * Fixtures present in both sides are paired; fixtures unique to one
+ * side surface with an explicit `status` and no rows. Within each
+ * paired fixture, every entry in `POLARITY` produces one row.
+ *
+ * Never throws on missing data — `undefined` means / spreads carry
+ * through to `undefined` deltas / thresholds and a `no-effect` label.
+ */
+export function compareMetrics(input) {
+    const baselineName = input.baselineName ?? 'baseline';
+    const variantName = input.variantName ?? 'variant';
+    const overrides = input.polarityOverrides ?? {};
+    const baselineMap = byFixtureId(input.baseline);
+    const variantMap = byFixtureId(input.variant);
+    // Union order: baseline fixtures in their input order, then variant
+    // fixtures that did not appear in baseline (also in input order).
+    const ids = [];
+    const seen = new Set();
+    for (const t of input.baseline) {
+        if (seen.has(t.fixtureId))
+            continue;
+        seen.add(t.fixtureId);
+        ids.push(t.fixtureId);
+    }
+    for (const t of input.variant) {
+        if (seen.has(t.fixtureId))
+            continue;
+        seen.add(t.fixtureId);
+        ids.push(t.fixtureId);
+    }
+    const fixtures = [];
+    for (const fixtureId of ids) {
+        const b = baselineMap.get(fixtureId);
+        const v = variantMap.get(fixtureId);
+        if (b && !v) {
+            fixtures.push({ fixtureId, status: 'baseline-only', rows: [] });
+            continue;
+        }
+        if (!b && v) {
+            fixtures.push({ fixtureId, status: 'variant-only', rows: [] });
+            continue;
+        }
+        if (!b || !v)
+            continue; // Defensive: should not happen given the union build.
+        const rows = [];
+        for (const metric of METRIC_NAMES) {
+            const polarity = overrides[metric] ?? POLARITY[metric];
+            const baselineStat = readStat(b, metric);
+            const variantStat = readStat(v, metric);
+            rows.push(buildRow(fixtureId, metric, polarity, baselineStat, variantStat));
+        }
+        fixtures.push({ fixtureId, status: 'both', rows });
+    }
+    return { baselineName, variantName, fixtures };
+}
+/** Iteration order for metric rows in a fixture. Stable and matches `POLARITY`. */
+const METRIC_NAMES = [
+    'taskSuccessRate',
+    'wallClockMs',
+    'promptTokens',
+    'completionTokens',
+    'toolCallCount.total',
+    'toolCallCount.reads',
+    'toolCallCount.mutations',
+    'turnCount',
+    'peakContextWindowBytes',
+    'truthfulnessViolationRate',
+    'dispatchVsLlmRatio',
+];
+function buildRow(fixtureId, metric, polarity, baseline, variant) {
+    if (baseline.mean === undefined ||
+        variant.mean === undefined ||
+        baseline.stdDev === undefined ||
+        variant.stdDev === undefined) {
+        return {
+            fixtureId,
+            metric,
+            polarity,
+            baseline,
+            variant,
+            delta: undefined,
+            threshold: undefined,
+            label: 'no-effect',
+        };
+    }
+    const delta = variant.mean - baseline.mean;
+    const threshold = Math.max(baseline.stdDev, variant.stdDev);
+    const label = classify(delta, threshold, polarity);
+    return { fixtureId, metric, polarity, baseline, variant, delta, threshold, label };
+}
+/**
+ * Apply the no-effect rule, then the polarity rule.
+ *
+ * No-effect rule: `Math.abs(delta) < threshold`. Strict `<` per the
+ * implementation plan — when `|delta|` exactly equals the spread, the
+ * row still rises to a winner. We add one degenerate carve-out: a
+ * delta of exactly zero is always no-effect regardless of threshold,
+ * because there is no direction to pick a winner from. This matters
+ * for single-trial inputs where both spreads are zero.
+ *
+ * Polarity rule:
+ *   - higher-is-better: positive delta -> variant wins; negative -> baseline wins.
+ *   - lower-is-better:  positive delta -> baseline wins; negative -> variant wins.
+ *   - neutral:          movement gets `changed`; no winner is implied.
+ */
+function classify(delta, threshold, polarity) {
+    if (delta === 0)
+        return 'no-effect';
+    if (Math.abs(delta) < threshold)
+        return 'no-effect';
+    if (polarity === 'neutral')
+        return 'changed';
+    if (polarity === 'higher-is-better') {
+        return delta > 0 ? 'winner-variant' : 'winner-baseline';
+    }
+    // lower-is-better
+    return delta > 0 ? 'winner-baseline' : 'winner-variant';
+}
+function byFixtureId(tables) {
+    const out = new Map();
+    for (const t of tables) {
+        // First-seen wins. Duplicate fixture ids in one input are unusual
+        // but not the comparator's problem to surface.
+        if (!out.has(t.fixtureId))
+            out.set(t.fixtureId, t);
+    }
+    return out;
+}
+function readStat(table, metric) {
+    switch (metric) {
+        case 'taskSuccessRate':
+            return table.aggregate.taskSuccessRate;
+        case 'wallClockMs':
+            return table.aggregate.wallClockMs;
+        case 'promptTokens':
+            return table.aggregate.promptTokens;
+        case 'completionTokens':
+            return table.aggregate.completionTokens;
+        case 'toolCallCount.total':
+            return table.aggregate.toolCallCount.total;
+        case 'toolCallCount.reads':
+            return table.aggregate.toolCallCount.reads;
+        case 'toolCallCount.mutations':
+            return table.aggregate.toolCallCount.mutations;
+        case 'turnCount':
+            return table.aggregate.turnCount;
+        case 'peakContextWindowBytes':
+            return table.aggregate.peakContextWindowBytes;
+        case 'truthfulnessViolationRate':
+            return table.aggregate.truthfulnessViolationRate;
+        case 'dispatchVsLlmRatio':
+            return table.aggregate.dispatchVsLlmRatio;
+    }
+}
+// ---------- renderers ----------
+/**
+ * Render a `ComparisonReport` as plain JSON.
+ *
+ * Implementation is `JSON.stringify(report, null, 2)`. The report is
+ * pure data: no functions, no `undefined` in places where the
+ * renderer cares (numeric fields surface as `null` after a round-trip
+ * but the consumer treats `null` and missing the same way).
+ */
+export function renderJson(report) {
+    return JSON.stringify(report, null, 2);
+}
+/**
+ * Render a `ComparisonReport` as a markdown document.
+ *
+ * One section per fixture. Each section has a header with the
+ * fixture id and its coverage status, then a table with seven
+ * columns: metric, baseline (mean +/- spread), variant (mean +/-
+ * spread), delta, threshold, polarity, label.
+ *
+ * Numeric formatting:
+ *   - mean / spread use up to four significant digits;
+ *   - `undefined` renders as `-`;
+ *   - delta and threshold use the same formatter as mean.
+ */
+export function renderMarkdown(report) {
+    const lines = [];
+    lines.push(`# Comparison: ${report.baselineName} vs ${report.variantName}`);
+    lines.push('');
+    if (report.fixtures.length === 0) {
+        lines.push('_No fixtures in either input._');
+        return lines.join('\n');
+    }
+    for (const fixture of report.fixtures) {
+        lines.push(`## ${fixture.fixtureId}`);
+        if (fixture.status === 'baseline-only') {
+            lines.push('');
+            lines.push(`_Present in ${report.baselineName} only — no comparison rows._`);
+            lines.push('');
+            continue;
+        }
+        if (fixture.status === 'variant-only') {
+            lines.push('');
+            lines.push(`_Present in ${report.variantName} only — no comparison rows._`);
+            lines.push('');
+            continue;
+        }
+        lines.push('');
+        lines.push(`| metric | ${report.baselineName} | ${report.variantName} | delta | threshold | polarity | label |`);
+        lines.push('|---|---|---|---|---|---|---|');
+        for (const row of fixture.rows) {
+            const cells = [
+                row.metric,
+                formatStat(row.baseline),
+                formatStat(row.variant),
+                formatNumber(row.delta),
+                formatNumber(row.threshold),
+                row.polarity,
+                row.label,
+            ];
+            lines.push(`| ${cells.join(' | ')} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n').trimEnd();
+}
+function formatStat(stat) {
+    if (stat.mean === undefined || stat.stdDev === undefined)
+        return '-';
+    return `${formatNumber(stat.mean)} ± ${formatNumber(stat.stdDev)} (n=${stat.count})`;
+}
+function formatNumber(value) {
+    if (value === undefined)
+        return '-';
+    if (!Number.isFinite(value))
+        return String(value);
+    if (value === 0)
+        return '0';
+    const abs = Math.abs(value);
+    if (abs >= 1000)
+        return value.toFixed(0);
+    if (abs >= 1)
+        return value.toFixed(2);
+    // Small numbers: four significant digits keeps spreads readable.
+    return value.toPrecision(4);
+}
+//# sourceMappingURL=comparison-report.js.map

package/dist/eval/comparison-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"comparison-report.js","sourceRoot":"","sources":["../../src/eval/comparison-report.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AAgCH;;;;;;;;GAQG;AACH,MAAM,CAAC,MAAM,QAAQ,GAA2C;IAC9D,0CAA0C;IAC1C,eAAe,EAAE,kBAAkB;IACnC,4DAA4D;IAC5D,WAAW,EAAE,iBAAiB;IAC9B,YAAY,EAAE,iBAAiB;IAC/B,gBAAgB,EAAE,iBAAiB;IACnC,sBAAsB,EAAE,iBAAiB;IACzC,8DAA8D;IAC9D,yBAAyB,EAAE,iBAAiB;IAC5C,8DAA8D;IAC9D,6DAA6D;IAC7D,+DAA+D;IAC/D,6BAA6B;IAC7B,qBAAqB,EAAE,SAAS;IAChC,qBAAqB,EAAE,SAAS;IAChC,yBAAyB,EAAE,SAAS;IACpC,uDAAuD;IACvD,SAAS,EAAE,SAAS;IACpB,+DAA+D;IAC/D,kBAAkB,EAAE,SAAS;CAC9B,CAAC;AA0EF;;;;;;;;;GASG;AACH,MAAM,UAAU,cAAc,CAAC,KAA0B;IACvD,MAAM,YAAY,GAAG,KAAK,CAAC,YAAY,IAAI,UAAU,CAAC;IACtD,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,SAAS,CAAC;IACnD,MAAM,SAAS,GAAG,KAAK,CAAC,iBAAiB,IAAI,EAAE,CAAC;IAChD,MAAM,WAAW,GAAG,WAAW,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IAChD,MAAM,UAAU,GAAG,WAAW,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;IAE9C,oEAAoE;IACpE,kEAAkE;IAClE,MAAM,GAAG,GAAa,EAAE,CAAC;IACzB,MAAM,IAAI,GAAG,IAAI,GAAG,EAAU,CAAC;IAC/B,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,QAAQ,EAAE,CAAC;QAC/B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,SAAS;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IACD,KAAK,MAAM,CAAC,IAAI,KAAK,CAAC,OAAO,EAAE,CAAC;QAC9B,IAAI,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,SAAS;QACpC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;QACtB,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC;IACxB,CAAC;IAED,MAAM,QAAQ,GAAwB,EAAE,CAAC;IACzC,KAAK,MAAM,SAAS,IAAI,GAAG,EAAE,CAAC;QAC5B,MAAM,CAAC,GAAG,WAAW,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACrC,MAAM,CAAC,GAAG,UAAU,CAAC,GAAG,CAAC,SAAS,CAAC,CAAC;QACpC,IAAI,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,eAAe,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAChE,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,CAAC,EAAE,CAAC;YACZ,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,cAAc,EAAE,IAAI,EAAE,EAAE,EAAE,CAAC,CAAC;YAC/D,SAAS;QACX,CAAC;QACD,IAAI,CAAC,CAAC,IAAI,CAAC,CAAC;YAAE,SAAS,CAAC,sDAAsD;QAC9E,MAAM,IAAI,GAAoB,EAAE,CAAC;QACjC,KAAK,MAAM,MAAM,IAAI,YAAY,EAAE,CAAC;YAClC,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,IAAI,QAAQ,CAAC,MAAM,CAAC,CAAC;YACvD,MAAM,YAAY,GAAG,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACzC,MAAM,WAAW,GAAG,QAAQ,CAAC,CAAC,EAAE,MAAM,CAAC,CAAC;YACxC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,YAAY,EAAE,WAAW,CAAC,CAAC,CAAC;QAC9E,CAAC;QACD,QAAQ,CAAC,IAAI,CAAC,EAAE,SAAS,EAAE,MAAM,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IAED,OAAO,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,CAAC;AACjD,CAAC;AAED,mFAAmF;AACnF,MAAM,YAAY,GAAoC;IACpD,iBAAiB;IACjB,aAAa;IACb,cAAc;IACd,kBAAkB;IAClB,qBAAqB;IACrB,qBAAqB;IACrB,yBAAyB;IACzB,WAAW;IACX,wBAAwB;IACxB,2BAA2B;IAC3B,oBAAoB;CACrB,CAAC;AAEF,SAAS,QAAQ,CACf,SAAiB,EACjB,MAA4B,EAC5B,QAAkB,EAClB,QAAuB,EACvB,OAAsB;IAEtB,IACE,QAAQ,CAAC,IAAI,KAAK,SAAS;QAC3B,OAAO,CAAC,IAAI,KAAK,SAAS;QAC1B,QAAQ,CAAC,MAAM,KAAK,SAAS;QAC7B,OAAO,CAAC,MAAM,KAAK,SAAS,EAC5B,CAAC;QACD,OAAO;YACL,SAAS;YACT,MAAM;YACN,QAAQ;YACR,QAAQ;YACR,OAAO;YACP,KAAK,EAAE,SAAS;YAChB,SAAS,EAAE,SAAS;YACpB,KAAK,EAAE,WAAW;SACnB,CAAC;IACJ,CAAC;IACD,MAAM,KAAK,GAAG,OAAO,CAAC,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC;IAC3C,MAAM,SAAS,GAAG,IAAI,CAAC,GAAG,CAAC,QAAQ,CAAC,MAAM,EAAE,OAAO,CAAC,MAAM,CAAC,CAAC;IAC5D,MAAM,KAAK,GAAG,QAAQ,CAAC,KAAK,EAAE,SAAS,EAAE,QAAQ,CAAC,CAAC;IACnD,OAAO,EAAE,SAAS,EAAE,MAAM,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,EAAE,KAAK,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC;AACrF,CAAC;AAED;;;;;;;;;;;;;;GAcG;AACH,SAAS,QAAQ,CAAC,KAAa,EAAE,SAAiB,EAAE,QAAkB;IACpE,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,WAAW,CAAC;IACpC,IAAI,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,GAAG,SAAS;QAAE,OAAO,WAAW,CAAC;IACpD,IAAI,QAAQ,KAAK,SAAS;QAAE,OAAO,SAAS,CAAC;IAC7C,IAAI,QAAQ,KAAK,kBAAkB,EAAE,CAAC;QACpC,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,gBAAgB,CAAC,CAAC,CAAC,iBAAiB,CAAC;IAC1D,CAAC;IACD,kBAAkB;IAClB,OAAO,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC,CAAC,gBAAgB,CAAC;AAC1D,CAAC;AAED,SAAS,WAAW,CAAC,MAA+B;IAClD,MAAM,GAAG,GAAG,IAAI,GAAG,EAAwB,CAAC;IAC5C,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,kEAAkE;QAClE,+CAA+C;QAC/C,IAAI,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,CAAC;YAAE,GAAG,CAAC,GAAG,CAAC,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,CAAC;IACrD,CAAC;IACD,OAAO,GAAG,CAAC;AACb,CAAC;AAED,SAAS,QAAQ,CAAC,KAAmB,EAAE,MAA4B;IACjE,QAAQ,MAAM,EAAE,CAAC;QACf,KAAK,iBAAiB;YACpB,OAAO,KAAK,CAAC,SAAS,CAAC,eAAe,CAAC;QACzC,KAAK,aAAa;YAChB,OAAO,KAAK,CAAC,SAAS,CAAC,WAAW,CAAC;QACrC,KAAK,cAAc;YACjB,OAAO,KAAK,CAAC,SAAS,CAAC,YAAY,CAAC;QACtC,KAAK,kBAAkB;YACrB,OAAO,KAAK,CAAC,SAAS,CAAC,gBAAgB,CAAC;QAC1C,KAAK,qBAAqB;YACxB,OAAO,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC;QAC7C,KAAK,qBAAqB;YACxB,OAAO,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,KAAK,CAAC;QAC7C,KAAK,yBAAyB;YAC5B,OAAO,KAAK,CAAC,SAAS,CAAC,aAAa,CAAC,SAAS,CAAC;QACjD,KAAK,WAAW;YACd,OAAO,KAAK,CAAC,SAAS,CAAC,SAAS,CAAC;QACnC,KAAK,wBAAwB;YAC3B,OAAO,KAAK,CAAC,SAAS,CAAC,sBAAsB,CAAC;QAChD,KAAK,2BAA2B;YAC9B,OAAO,KAAK,CAAC,SAAS,CAAC,yBAAyB,CAAC;QACnD,KAAK,oBAAoB;YACvB,OAAO,KAAK,CAAC,SAAS,CAAC,kBAAkB,CAAC;IAC9C,CAAC;AACH,CAAC;AAED,kCAAkC;AAElC;;;;;;;GAOG;AACH,MAAM,UAAU,UAAU,CAAC,MAAwB;IACjD,OAAO,IAAI,CAAC,SAAS,CAAC,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC;AACzC,CAAC;AAED;;;;;;;;;;;;GAYG;AACH,MAAM,UAAU,cAAc,CAAC,MAAwB;IACrD,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,KAAK,CAAC,IAAI,CAAC,iBAAiB,MAAM,CAAC,YAAY,OAAO,MAAM,CAAC,WAAW,EAAE,CAAC,CAAC;IAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,IAAI,MAAM,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACjC,KAAK,CAAC,IAAI,CAAC,gCAAgC,CAAC,CAAC;QAC7C,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;IAC1B,CAAC;IAED,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;QACtC,KAAK,CAAC,IAAI,CAAC,MAAM,OAAO,CAAC,SAAS,EAAE,CAAC,CAAC;QACtC,IAAI,OAAO,CAAC,MAAM,KAAK,eAAe,EAAE,CAAC;YACvC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,YAAY,8BAA8B,CAAC,CAAC;YAC7E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,IAAI,OAAO,CAAC,MAAM,KAAK,cAAc,EAAE,CAAC;YACtC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,WAAW,8BAA8B,CAAC,CAAC;YAC5E,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;YACf,SAAS;QACX,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CACR,cAAc,MAAM,CAAC,YAAY,MAAM,MAAM,CAAC,WAAW,2CAA2C,CACrG,CAAC;QACF,KAAK,CAAC,IAAI,CAAC,+BAA+B,CAAC,CAAC;QAC5C,KAAK,MAAM,GAAG,IAAI,OAAO,CAAC,IAAI,EAAE,CAAC;YAC/B,MAAM,KAAK,GAAG;gBACZ,GAAG,CAAC,MAAM;gBACV,UAAU,CAAC,GAAG,CAAC,QAAQ,CAAC;gBACxB,UAAU,CAAC,GAAG,CAAC,OAAO,CAAC;gBACvB,YAAY,CAAC,GAAG,CAAC,KAAK,CAAC;gBACvB,YAAY,CAAC,GAAG,CAAC,SAAS,CAAC;gBAC3B,GAAG,CAAC,QAAQ;gBACZ,GAAG,CAAC,KAAK;aACV,CAAC;YACF,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;QACzC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC,OAAO,EAAE,CAAC;AACpC,CAAC;AAED,SAAS,UAAU,CAAC,IAAmB;IACrC,IAAI,IAAI,CAAC,IAAI,KAAK,SAAS,IAAI,IAAI,CAAC,MAAM,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IACrE,OAAO,GAAG,YAAY,CAAC,IAAI,CAAC,IAAI,CAAC,MAAM,YAAY,CAAC,IAAI,CAAC,MAAM,CAAC,OAAO,IAAI,CAAC,KAAK,GAAG,CAAC;AACvF,CAAC;AAED,SAAS,YAAY,CAAC,KAAyB;IAC7C,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO,GAAG,CAAC;IACpC,IAAI,CAAC,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC;QAAE,OAAO,MAAM,CAAC,KAAK,CAAC,CAAC;IAClD,IAAI,KAAK,KAAK,CAAC;QAAE,OAAO,GAAG,CAAC;IAC5B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC;IAC5B,IAAI,GAAG,IAAI,IAAI;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACzC,IAAI,GAAG,IAAI,CAAC;QAAE,OAAO,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,CAAC;IACtC,iEAAiE;IACjE,OAAO,KAAK,CAAC,WAAW,CAAC,CAAC,CAAC,CAAC;AAC9B,CAAC"}

package/dist/eval/fixture.d.ts

@@ -0,0 +1,74 @@
+/**
+ * Task fixture schema for the eval harness.
+ *
+ * A `TaskFixture` describes one reproducible evaluation case: a user
+ * prompt, an initial workspace state, and a reference to the success
+ * specification that decides whether a run passed.
+ *
+ * This file is browser-safe — types, `validateFixture`, `parseFixture`,
+ * `applyWorkspaceOverrides` only. File loading lives in
+ * `eval/load-node.ts` and is re-exported via `@inbrowser/agent/node`.
+ *
+ * Example fixture (JSON):
+ *
+ *     {
+ *       "id": "firestore-rules-audit/seed-open-write-01",
+ *       "skill": "firestore-rules-audit",
+ *       "description": "Detects open-write vulnerability on /users",
+ *       "prompt": "Audit my Firestore rules for security issues.",
+ *       "initialState": {
+ *         "rules": "rules_version='2'; ..."
+ *       },
+ *       "successSpec": {
+ *         "name": "firestore-rules-audit/names-planted-vulnerability",
+ *         "args": { "vulnerability": "open-write-users" }
+ *       }
+ *     }
+ */
+import type { StitchContext, Workspace } from '../types/workspace.js';
+export type SkillName = 'firestore-rules-audit' | 'firebase-project-audit' | 'rtdb-data-modeling' | 'firebase-security-rules' | 'firebase-client-sdk' | 'pyric-agents' | 'playground-prompts' | 'rtdb-game-rules' | 'firestore-game-rules';
+export declare const SKILL_NAMES: readonly SkillName[];
+export interface PartialWorkspace {
+    presetId?: string;
+    rules?: string;
+    code?: string;
+    appSource?: string;
+    stitch?: Partial<StitchContext>;
+}
+export interface SuccessSpecReference {
+    /** Stable spec identifier in `family/spec-name` kebab-case form. */
+    name: string;
+    /** Optional structured arguments passed when the spec runs. */
+    args?: Record<string, unknown>;
+}
+export interface TaskFixture {
+    /** Stable identifier in `skill-prefix/case-name` kebab-case form. */
+    id: string;
+    /** The skill family this fixture exercises. */
+    skill: SkillName;
+    /** One-line human summary; shown in comparison reports. */
+    description: string;
+    /** Optional free-form prose; ignored by tooling. */
+    notes?: string;
+    /** The user prompt, verbatim. */
+    prompt: string;
+    /** Optional initial workspace state, applied on top of EMPTY_WORKSPACE. */
+    initialState?: PartialWorkspace;
+    /** Reference to the success spec that decides this fixture's pass/fail. */
+    successSpec: SuccessSpecReference;
+}
+export interface ValidationError {
+    path: string;
+    message: string;
+}
+export type ValidationResult = {
+    ok: true;
+    fixture: TaskFixture;
+} | {
+    ok: false;
+    errors: ValidationError[];
+};
+export declare function validateFixture(input: unknown): ValidationResult;
+export declare function parseFixture(json: string): ValidationResult;
+export declare function applyWorkspaceOverrides(base: Workspace, overrides: PartialWorkspace | undefined): Workspace;
+//# sourceMappingURL=fixture.d.ts.map

package/dist/eval/fixture.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixture.d.ts","sourceRoot":"","sources":["../../src/eval/fixture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAEH,OAAO,KAAK,EAAE,aAAa,EAAE,SAAS,EAAE,MAAM,uBAAuB,CAAC;AAEtE,MAAM,MAAM,SAAS,GACjB,uBAAuB,GACvB,wBAAwB,GACxB,oBAAoB,GACpB,yBAAyB,GACzB,qBAAqB,GACrB,cAAc,GACd,oBAAoB,GACpB,iBAAiB,GACjB,sBAAsB,CAAC;AAE3B,eAAO,MAAM,WAAW,EAAE,SAAS,SAAS,EAU3C,CAAC;AAEF,MAAM,WAAW,gBAAgB;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC;IAClB,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,IAAI,CAAC,EAAE,MAAM,CAAC;IACd,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,OAAO,CAAC,aAAa,CAAC,CAAC;CACjC;AAED,MAAM,WAAW,oBAAoB;IACnC,oEAAoE;IACpE,IAAI,EAAE,MAAM,CAAC;IACb,+DAA+D;IAC/D,IAAI,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CAChC;AAED,MAAM,WAAW,WAAW;IAC1B,qEAAqE;IACrE,EAAE,EAAE,MAAM,CAAC;IACX,+CAA+C;IAC/C,KAAK,EAAE,SAAS,CAAC;IACjB,2DAA2D;IAC3D,WAAW,EAAE,MAAM,CAAC;IACpB,oDAAoD;IACpD,KAAK,CAAC,EAAE,MAAM,CAAC;IACf,iCAAiC;IACjC,MAAM,EAAE,MAAM,CAAC;IACf,2EAA2E;IAC3E,YAAY,CAAC,EAAE,gBAAgB,CAAC;IAChC,2EAA2E;IAC3E,WAAW,EAAE,oBAAoB,CAAC;CACnC;AAED,MAAM,WAAW,eAAe;IAC9B,IAAI,EAAE,MAAM,CAAC;IACb,OAAO,EAAE,MAAM,CAAC;CACjB;AAED,MAAM,MAAM,gBAAgB,GACxB;IAAE,EAAE,EAAE,IAAI,CAAC;IAAC,OAAO,EAAE,WAAW,CAAA;CAAE,GAClC;IAAE,EAAE,EAAE,KAAK,CAAC;IAAC,MAAM,EAAE,eAAe,EAAE,CAAA;CAAE,CAAC;AAmB7C,wBAAgB,eAAe,CAAC,KAAK,EAAE,OAAO,GAAG,gBAAgB,CAgDhE;AAED,wBAAgB,YAAY,CAAC,IAAI,EAAE,MAAM,GAAG,gBAAgB,CAS3D;AAED,wBAAgB,uBAAuB,CACrC,IAAI,EAAE,SAAS,EACf,SAAS,EAAE,gBAAgB,GAAG,SAAS,GACtC,SAAS,CAaX"}

package/dist/eval/fixture.js

@@ -0,0 +1,217 @@
+/**
+ * Task fixture schema for the eval harness.
+ *
+ * A `TaskFixture` describes one reproducible evaluation case: a user
+ * prompt, an initial workspace state, and a reference to the success
+ * specification that decides whether a run passed.
+ *
+ * This file is browser-safe — types, `validateFixture`, `parseFixture`,
+ * `applyWorkspaceOverrides` only. File loading lives in
+ * `eval/load-node.ts` and is re-exported via `@inbrowser/agent/node`.
+ *
+ * Example fixture (JSON):
+ *
+ *     {
+ *       "id": "firestore-rules-audit/seed-open-write-01",
+ *       "skill": "firestore-rules-audit",
+ *       "description": "Detects open-write vulnerability on /users",
+ *       "prompt": "Audit my Firestore rules for security issues.",
+ *       "initialState": {
+ *         "rules": "rules_version='2'; ..."
+ *       },
+ *       "successSpec": {
+ *         "name": "firestore-rules-audit/names-planted-vulnerability",
+ *         "args": { "vulnerability": "open-write-users" }
+ *       }
+ *     }
+ */
+export const SKILL_NAMES = [
+    'firestore-rules-audit',
+    'firebase-project-audit',
+    'rtdb-data-modeling',
+    'firebase-security-rules',
+    'firebase-client-sdk',
+    'pyric-agents',
+    'playground-prompts',
+    'rtdb-game-rules',
+    'firestore-game-rules',
+];
+const ID_PATTERN = /^[a-z][a-z0-9-]*\/[a-z][a-z0-9-]*$/;
+const SPEC_NAME_PATTERN = /^[a-z][a-z0-9-]*\/[a-z][a-z0-9-]*$/;
+const SKILL_NAME_SET = new Set(SKILL_NAMES);
+const FIXTURE_KEYS = new Set([
+    'id',
+    'skill',
+    'description',
+    'notes',
+    'prompt',
+    'initialState',
+    'successSpec',
+]);
+const INITIAL_STATE_KEYS = new Set(['presetId', 'rules', 'code', 'appSource', 'stitch']);
+const STITCH_KEYS = new Set(['projectId', 'latestScreenUrl', 'brief']);
+const SPEC_KEYS = new Set(['name', 'args']);
+export function validateFixture(input) {
+    if (!isPlainObject(input)) {
+        return { ok: false, errors: [{ path: '', message: 'fixture must be a plain object' }] };
+    }
+    const obj = input;
+    const errors = [];
+    requireString(obj, 'id', errors, (value) => {
+        if (!ID_PATTERN.test(value)) {
+            errors.push({
+                path: 'id',
+                message: 'id must match `skill-prefix/case-name` kebab-case (e.g., "firestore-rules-audit/seed-01")',
+            });
+        }
+    });
+    requireString(obj, 'skill', errors, (value) => {
+        if (!SKILL_NAME_SET.has(value)) {
+            errors.push({
+                path: 'skill',
+                message: `skill must be one of: ${SKILL_NAMES.join(', ')}`,
+            });
+        }
+    });
+    requireString(obj, 'description', errors);
+    optionalString(obj, 'notes', errors);
+    requireString(obj, 'prompt', errors);
+    if (obj.initialState !== undefined) {
+        validateInitialState(obj.initialState, 'initialState', errors);
+    }
+    if (obj.successSpec === undefined) {
+        errors.push({ path: 'successSpec', message: 'successSpec is required' });
+    }
+    else {
+        validateSpecRef(obj.successSpec, 'successSpec', errors);
+    }
+    for (const key of Object.keys(obj)) {
+        if (!FIXTURE_KEYS.has(key)) {
+            errors.push({ path: key, message: `unknown field "${key}"` });
+        }
+    }
+    if (errors.length > 0)
+        return { ok: false, errors };
+    return { ok: true, fixture: obj };
+}
+export function parseFixture(json) {
+    let parsed;
+    try {
+        parsed = JSON.parse(json);
+    }
+    catch (err) {
+        const message = err instanceof Error ? err.message : 'invalid JSON';
+        return { ok: false, errors: [{ path: '', message: `failed to parse JSON: ${message}` }] };
+    }
+    return validateFixture(parsed);
+}
+export function applyWorkspaceOverrides(base, overrides) {
+    if (!overrides)
+        return base;
+    return {
+        presetId: overrides.presetId ?? base.presetId,
+        rules: overrides.rules ?? base.rules,
+        code: overrides.code ?? base.code,
+        appSource: overrides.appSource ?? base.appSource,
+        stitch: {
+            projectId: overrides.stitch?.projectId ?? base.stitch.projectId,
+            latestScreenUrl: overrides.stitch?.latestScreenUrl ?? base.stitch.latestScreenUrl,
+            brief: overrides.stitch?.brief ?? base.stitch.brief,
+        },
+    };
+}
+function isPlainObject(value) {
+    return typeof value === 'object' && value !== null && !Array.isArray(value);
+}
+function requireString(obj, key, errors, onValid) {
+    const value = obj[key];
+    if (value === undefined) {
+        errors.push({ path: key, message: `${key} is required` });
+        return;
+    }
+    if (typeof value !== 'string') {
+        errors.push({ path: key, message: `${key} must be a string` });
+        return;
+    }
+    if (value.length === 0) {
+        errors.push({ path: key, message: `${key} must not be empty` });
+        return;
+    }
+    if (onValid)
+        onValid(value);
+}
+function optionalString(obj, key, errors) {
+    const value = obj[key];
+    if (value === undefined)
+        return;
+    if (typeof value !== 'string') {
+        errors.push({ path: key, message: `${key} must be a string when present` });
+    }
+}
+function validateInitialState(value, path, errors) {
+    if (!isPlainObject(value)) {
+        errors.push({ path, message: `${path} must be an object` });
+        return;
+    }
+    for (const key of Object.keys(value)) {
+        if (!INITIAL_STATE_KEYS.has(key)) {
+            errors.push({ path: `${path}.${key}`, message: `unknown field "${key}"` });
+        }
+    }
+    for (const key of ['presetId', 'rules', 'code', 'appSource']) {
+        const v = value[key];
+        if (v !== undefined && typeof v !== 'string') {
+            errors.push({
+                path: `${path}.${key}`,
+                message: `${path}.${key} must be a string when present`,
+            });
+        }
+    }
+    if (value.stitch !== undefined) {
+        validateStitchOverrides(value.stitch, `${path}.stitch`, errors);
+    }
+}
+function validateStitchOverrides(value, path, errors) {
+    if (!isPlainObject(value)) {
+        errors.push({ path, message: `${path} must be an object when present` });
+        return;
+    }
+    for (const key of Object.keys(value)) {
+        if (!STITCH_KEYS.has(key)) {
+            errors.push({ path: `${path}.${key}`, message: `unknown field "${key}"` });
+        }
+    }
+    for (const key of ['projectId', 'latestScreenUrl', 'brief']) {
+        const v = value[key];
+        if (v !== undefined && v !== null && typeof v !== 'string') {
+            errors.push({
+                path: `${path}.${key}`,
+                message: `${path}.${key} must be a string or null when present`,
+            });
+        }
+    }
+}
+function validateSpecRef(value, path, errors) {
+    if (!isPlainObject(value)) {
+        errors.push({ path, message: `${path} must be an object` });
+        return;
+    }
+    for (const key of Object.keys(value)) {
+        if (!SPEC_KEYS.has(key)) {
+            errors.push({ path: `${path}.${key}`, message: `unknown field "${key}"` });
+        }
+    }
+    if (typeof value.name !== 'string') {
+        errors.push({ path: `${path}.name`, message: `${path}.name must be a string` });
+    }
+    else if (!SPEC_NAME_PATTERN.test(value.name)) {
+        errors.push({
+            path: `${path}.name`,
+            message: `${path}.name must match \`family/spec-name\` kebab-case`,
+        });
+    }
+    if (value.args !== undefined && !isPlainObject(value.args)) {
+        errors.push({ path: `${path}.args`, message: `${path}.args must be an object when present` });
+    }
+}
+//# sourceMappingURL=fixture.js.map

package/dist/eval/fixture.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"fixture.js","sourceRoot":"","sources":["../../src/eval/fixture.ts"],"names":[],"mappings":"AAAA;;;;;;;;;;;;;;;;;;;;;;;;;;GA0BG;AAeH,MAAM,CAAC,MAAM,WAAW,GAAyB;IAC/C,uBAAuB;IACvB,wBAAwB;IACxB,oBAAoB;IACpB,yBAAyB;IACzB,qBAAqB;IACrB,cAAc;IACd,oBAAoB;IACpB,iBAAiB;IACjB,sBAAsB;CACvB,CAAC;AA2CF,MAAM,UAAU,GAAG,oCAAoC,CAAC;AACxD,MAAM,iBAAiB,GAAG,oCAAoC,CAAC;AAC/D,MAAM,cAAc,GAAwB,IAAI,GAAG,CAAC,WAAW,CAAC,CAAC;AAEjE,MAAM,YAAY,GAAG,IAAI,GAAG,CAAC;IAC3B,IAAI;IACJ,OAAO;IACP,aAAa;IACb,OAAO;IACP,QAAQ;IACR,cAAc;IACd,aAAa;CACd,CAAC,CAAC;AACH,MAAM,kBAAkB,GAAG,IAAI,GAAG,CAAC,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,EAAE,QAAQ,CAAC,CAAC,CAAC;AACzF,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,CAAC,WAAW,EAAE,iBAAiB,EAAE,OAAO,CAAC,CAAC,CAAC;AACvE,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,CAAC,MAAM,EAAE,MAAM,CAAC,CAAC,CAAC;AAE5C,MAAM,UAAU,eAAe,CAAC,KAAc;IAC5C,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,gCAAgC,EAAE,CAAC,EAAE,CAAC;IAC1F,CAAC;IACD,MAAM,GAAG,GAAG,KAAK,CAAC;IAClB,MAAM,MAAM,GAAsB,EAAE,CAAC;IAErC,aAAa,CAAC,GAAG,EAAE,IAAI,EAAE,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QACzC,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;YAC5B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,IAAI;gBACV,OAAO,EACL,2FAA2F;aAC9F,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,aAAa,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,EAAE,CAAC,KAAK,EAAE,EAAE;QAC5C,IAAI,CAAC,cAAc,CAAC,GAAG,CAAC,KAAK,CAAC,EAAE,CAAC;YAC/B,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,OAAO;gBACb,OAAO,EAAE,yBAAyB,WAAW,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE;aAC3D,CAAC,CAAC;QACL,CAAC;IACH,CAAC,CAAC,CAAC;IAEH,aAAa,CAAC,GAAG,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IAC1C,cAAc,CAAC,GAAG,EAAE,OAAO,EAAE,MAAM,CAAC,CAAC;IACrC,aAAa,CAAC,GAAG,EAAE,QAAQ,EAAE,MAAM,CAAC,CAAC;IAErC,IAAI,GAAG,CAAC,YAAY,KAAK,SAAS,EAAE,CAAC;QACnC,oBAAoB,CAAC,GAAG,CAAC,YAAY,EAAE,cAAc,EAAE,MAAM,CAAC,CAAC;IACjE,CAAC;IAED,IAAI,GAAG,CAAC,WAAW,KAAK,SAAS,EAAE,CAAC;QAClC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,EAAE,yBAAyB,EAAE,CAAC,CAAC;IAC3E,CAAC;SAAM,CAAC;QACN,eAAe,CAAC,GAAG,CAAC,WAAW,EAAE,aAAa,EAAE,MAAM,CAAC,CAAC;IAC1D,CAAC;IAED,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC;QACnC,IAAI,CAAC,YAAY,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC3B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,kBAAkB,GAAG,GAAG,EAAE,CAAC,CAAC;QAChE,CAAC;IACH,CAAC;IAED,IAAI,MAAM,CAAC,MAAM,GAAG,CAAC;QAAE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC;IACpD,OAAO,EAAE,EAAE,EAAE,IAAI,EAAE,OAAO,EAAE,GAA6B,EAAE,CAAC;AAC9D,CAAC;AAED,MAAM,UAAU,YAAY,CAAC,IAAY;IACvC,IAAI,MAAe,CAAC;IACpB,IAAI,CAAC;QACH,MAAM,GAAG,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IAC5B,CAAC;IAAC,OAAO,GAAG,EAAE,CAAC;QACb,MAAM,OAAO,GAAG,GAAG,YAAY,KAAK,CAAC,CAAC,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC,CAAC,cAAc,CAAC;QACpE,OAAO,EAAE,EAAE,EAAE,KAAK,EAAE,MAAM,EAAE,CAAC,EAAE,IAAI,EAAE,EAAE,EAAE,OAAO,EAAE,yBAAyB,OAAO,EAAE,EAAE,CAAC,EAAE,CAAC;IAC5F,CAAC;IACD,OAAO,eAAe,CAAC,MAAM,CAAC,CAAC;AACjC,CAAC;AAED,MAAM,UAAU,uBAAuB,CACrC,IAAe,EACf,SAAuC;IAEvC,IAAI,CAAC,SAAS;QAAE,OAAO,IAAI,CAAC;IAC5B,OAAO;QACL,QAAQ,EAAE,SAAS,CAAC,QAAQ,IAAI,IAAI,CAAC,QAAQ;QAC7C,KAAK,EAAE,SAAS,CAAC,KAAK,IAAI,IAAI,CAAC,KAAK;QACpC,IAAI,EAAE,SAAS,CAAC,IAAI,IAAI,IAAI,CAAC,IAAI;QACjC,SAAS,EAAE,SAAS,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS;QAChD,MAAM,EAAE;YACN,SAAS,EAAE,SAAS,CAAC,MAAM,EAAE,SAAS,IAAI,IAAI,CAAC,MAAM,CAAC,SAAS;YAC/D,eAAe,EAAE,SAAS,CAAC,MAAM,EAAE,eAAe,IAAI,IAAI,CAAC,MAAM,CAAC,eAAe;YACjF,KAAK,EAAE,SAAS,CAAC,MAAM,EAAE,KAAK,IAAI,IAAI,CAAC,MAAM,CAAC,KAAK;SACpD;KACF,CAAC;AACJ,CAAC;AAED,SAAS,aAAa,CAAC,KAAc;IACnC,OAAO,OAAO,KAAK,KAAK,QAAQ,IAAI,KAAK,KAAK,IAAI,IAAI,CAAC,KAAK,CAAC,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9E,CAAC;AAED,SAAS,aAAa,CACpB,GAA4B,EAC5B,GAAW,EACX,MAAyB,EACzB,OAAiC;IAEjC,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,SAAS,EAAE,CAAC;QACxB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,GAAG,cAAc,EAAE,CAAC,CAAC;QAC1D,OAAO;IACT,CAAC;IACD,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,GAAG,mBAAmB,EAAE,CAAC,CAAC;QAC/D,OAAO;IACT,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QACvB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,GAAG,oBAAoB,EAAE,CAAC,CAAC;QAChE,OAAO;IACT,CAAC;IACD,IAAI,OAAO;QAAE,OAAO,CAAC,KAAK,CAAC,CAAC;AAC9B,CAAC;AAED,SAAS,cAAc,CACrB,GAA4B,EAC5B,GAAW,EACX,MAAyB;IAEzB,MAAM,KAAK,GAAG,GAAG,CAAC,GAAG,CAAC,CAAC;IACvB,IAAI,KAAK,KAAK,SAAS;QAAE,OAAO;IAChC,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;QAC9B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,EAAE,OAAO,EAAE,GAAG,GAAG,gCAAgC,EAAE,CAAC,CAAC;IAC9E,CAAC;AACH,CAAC;AAED,SAAS,oBAAoB,CAAC,KAAc,EAAE,IAAY,EAAE,MAAyB;IACnF,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAC5D,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,kBAAkB,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACjC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,kBAAkB,GAAG,GAAG,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,UAAU,EAAE,OAAO,EAAE,MAAM,EAAE,WAAW,CAAU,EAAE,CAAC;QACtE,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,SAAS,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC7C,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE;gBACtB,OAAO,EAAE,GAAG,IAAI,IAAI,GAAG,gCAAgC;aACxD,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,IAAI,KAAK,CAAC,MAAM,KAAK,SAAS,EAAE,CAAC;QAC/B,uBAAuB,CAAC,KAAK,CAAC,MAAM,EAAE,GAAG,IAAI,SAAS,EAAE,MAAM,CAAC,CAAC;IAClE,CAAC;AACH,CAAC;AAED,SAAS,uBAAuB,CAAC,KAAc,EAAE,IAAY,EAAE,MAAyB;IACtF,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,iCAAiC,EAAE,CAAC,CAAC;QACzE,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,WAAW,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,kBAAkB,GAAG,GAAG,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,CAAC,WAAW,EAAE,iBAAiB,EAAE,OAAO,CAAU,EAAE,CAAC;QACrE,MAAM,CAAC,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC;QACrB,IAAI,CAAC,KAAK,SAAS,IAAI,CAAC,KAAK,IAAI,IAAI,OAAO,CAAC,KAAK,QAAQ,EAAE,CAAC;YAC3D,MAAM,CAAC,IAAI,CAAC;gBACV,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE;gBACtB,OAAO,EAAE,GAAG,IAAI,IAAI,GAAG,wCAAwC;aAChE,CAAC,CAAC;QACL,CAAC;IACH,CAAC;AACH,CAAC;AAED,SAAS,eAAe,CAAC,KAAc,EAAE,IAAY,EAAE,MAAyB;IAC9E,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,EAAE,CAAC;QAC1B,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,GAAG,IAAI,oBAAoB,EAAE,CAAC,CAAC;QAC5D,OAAO;IACT,CAAC;IACD,KAAK,MAAM,GAAG,IAAI,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QACrC,IAAI,CAAC,SAAS,CAAC,GAAG,CAAC,GAAG,CAAC,EAAE,CAAC;YACxB,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,IAAI,GAAG,EAAE,EAAE,OAAO,EAAE,kBAAkB,GAAG,GAAG,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC;IACD,IAAI,OAAO,KAAK,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QACnC,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,wBAAwB,EAAE,CAAC,CAAC;IAClF,CAAC;SAAM,IAAI,CAAC,iBAAiB,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC/C,MAAM,CAAC,IAAI,CAAC;YACV,IAAI,EAAE,GAAG,IAAI,OAAO;YACpB,OAAO,EAAE,GAAG,IAAI,kDAAkD;SACnE,CAAC,CAAC;IACL,CAAC;IACD,IAAI,KAAK,CAAC,IAAI,KAAK,SAAS,IAAI,CAAC,aAAa,CAAC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC;QAC3D,MAAM,CAAC,IAAI,CAAC,EAAE,IAAI,EAAE,GAAG,IAAI,OAAO,EAAE,OAAO,EAAE,GAAG,IAAI,sCAAsC,EAAE,CAAC,CAAC;IAChG,CAAC;AACH,CAAC"}

package/dist/eval/index.d.ts

@@ -0,0 +1,13 @@
+export type { PartialWorkspace, SkillName, SuccessSpecReference, TaskFixture, ValidationError, ValidationResult, } from './fixture.js';
+export { SKILL_NAMES, applyWorkspaceOverrides, parseFixture, validateFixture } from './fixture.js';
+export type { RunRecord } from './run-record.js';
+export type { RunFixtureInput, RunFixturesDeps, RunFixturesOptions, } from './runner.js';
+export { defaultSystemPromptBuilder, runFixture, runFixtures } from './runner.js';
+export type { RunSnapshot, SpecFn, SpecRegistry, SpecResult } from './spec-framework.js';
+export { createSpecRegistry, evaluateSpec } from './spec-framework.js';
+export type { AggregateStat, AggregatedMetrics, CollectMetricsInput, MetricsTable, TrialMetrics, } from './metric-collector.js';
+export { aggregateTrials, collectMetrics, extractTrialMetrics } from './metric-collector.js';
+export type { CompareMetricsInput, ComparisonFixture, ComparisonLabel, ComparisonMetricName, ComparisonReport, ComparisonRow, Polarity, } from './comparison-report.js';
+export { POLARITY, compareMetrics, renderJson, renderMarkdown } from './comparison-report.js';
+export { CUSTOM_SPEC_NAMES, SPEC_FINAL_RULES_EXCLUDES_LITERAL, SPEC_FINAL_RULES_INCLUDES_LITERAL, SPEC_FINAL_RUNTIME_RUN_SUMMARY_OK, SPEC_GAME_RULES_SIMULATOR_ACCEPTS_POSITIVE_AND_REJECTS_CHEAT, SPEC_PYRIC_AGENTS_LINT_CLEAN_AND_RULE_REJECTS_CHEAT, SPEC_REPORT_MENTIONS_ALL_OF, SPEC_REPORT_MENTIONS_AT_LEAST_ONE_OF, SPEC_TRACE_CONTAINS_TOOL_CALL_BY_NAME, STARTER_SPEC_NAMES, finalRulesExcludesLiteral, finalRulesIncludesLiteral, finalRuntimeRunSummaryOk, gameRulesSimulatorAcceptsPositiveAndRejectsCheat, pyricAgentsLintCleanAndRuleRejectsCheat, registerAllSpecs, registerCustomSpecs, registerStarterSpecs, reportMentionsAllOf, reportMentionsAtLeastOneOf, traceContainsToolCallByName, } from './spec-helpers.js';
+//# sourceMappingURL=index.d.ts.map

package/dist/eval/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/eval/index.ts"],"names":[],"mappings":"AAAA,YAAY,EACV,gBAAgB,EAChB,SAAS,EACT,oBAAoB,EACpB,WAAW,EACX,eAAe,EACf,gBAAgB,GACjB,MAAM,cAAc,CAAC;AACtB,OAAO,EAAE,WAAW,EAAE,uBAAuB,EAAE,YAAY,EAAE,eAAe,EAAE,MAAM,cAAc,CAAC;AAEnG,YAAY,EAAE,SAAS,EAAE,MAAM,iBAAiB,CAAC;AACjD,YAAY,EACV,eAAe,EACf,eAAe,EACf,kBAAkB,GACnB,MAAM,aAAa,CAAC;AACrB,OAAO,EAAE,0BAA0B,EAAE,UAAU,EAAE,WAAW,EAAE,MAAM,aAAa,CAAC;AAElF,YAAY,EAAE,WAAW,EAAE,MAAM,EAAE,YAAY,EAAE,UAAU,EAAE,MAAM,qBAAqB,CAAC;AACzF,OAAO,EAAE,kBAAkB,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAEvE,YAAY,EACV,aAAa,EACb,iBAAiB,EACjB,mBAAmB,EACnB,YAAY,EACZ,YAAY,GACb,MAAM,uBAAuB,CAAC;AAC/B,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,mBAAmB,EAAE,MAAM,uBAAuB,CAAC;AAE7F,YAAY,EACV,mBAAmB,EACnB,iBAAiB,EACjB,eAAe,EACf,oBAAoB,EACpB,gBAAgB,EAChB,aAAa,EACb,QAAQ,GACT,MAAM,wBAAwB,CAAC;AAChC,OAAO,EAAE,QAAQ,EAAE,cAAc,EAAE,UAAU,EAAE,cAAc,EAAE,MAAM,wBAAwB,CAAC;AAE9F,OAAO,EACL,iBAAiB,EACjB,iCAAiC,EACjC,iCAAiC,EACjC,iCAAiC,EACjC,4DAA4D,EAC5D,mDAAmD,EACnD,2BAA2B,EAC3B,oCAAoC,EACpC,qCAAqC,EACrC,kBAAkB,EAClB,yBAAyB,EACzB,yBAAyB,EACzB,wBAAwB,EACxB,gDAAgD,EAChD,uCAAuC,EACvC,gBAAgB,EAChB,mBAAmB,EACnB,oBAAoB,EACpB,mBAAmB,EACnB,0BAA0B,EAC1B,2BAA2B,GAC5B,MAAM,mBAAmB,CAAC"}