@alisio/core 0.6.0 → 0.7.0

package/dist/eval/plan.d.ts

@@ -0,0 +1,26 @@
+/** Deterministic run planning of the reliability lab (specs/010 D6, D15). Pure. */
+/** Deterministic PRNG in [0, 1) (same algorithm as `scripts/bench/lib.ts`). */
+export declare function mulberry32(seed: number): () => number;
+export interface PlannedRun {
+    taskId: string;
+    repetition: number;
+}
+/**
+ * Every (task, repetition) pair exactly once, in a seeded order so provider drift over time is not
+ * correlated with one task. The same task ids, k and seed always give the same plan, whatever order
+ * the ids were listed in: the variants of a comparison run the same plan.
+ */
+export declare function planRuns(taskIds: string[], k: number, seed: number): PlannedRun[];
+export interface VariantStep extends PlannedRun {
+    variantId: string;
+    /** Index of the plan step this run belongs to (every variant runs each step). */
+    step: number;
+}
+/**
+ * Expands a plan so every variant runs EVERY plan step, one after another, instead of all of A and
+ * then all of B. The starting variant rotates with the step (A,B then B,A for two variants) so no
+ * variant systematically goes first. This spreads provider drift over time across the variants
+ * and keeps the same tasks, repetitions and seeds for each; it does not remove the drift (runs
+ * still happen at different moments) and nothing is paired.
+ */
+export declare function interleaveVariants(plan: PlannedRun[], variantIds: string[]): VariantStep[];

package/dist/eval/plan.js

@@ -0,0 +1,44 @@
+/** Deterministic run planning of the reliability lab (specs/010 D6, D15). Pure. */
+/** Deterministic PRNG in [0, 1) (same algorithm as `scripts/bench/lib.ts`). */
+export function mulberry32(seed) {
+    let a = seed >>> 0;
+    return () => {
+        a = (a + 0x6d2b79f5) >>> 0;
+        let t = a;
+        t = Math.imul(t ^ (t >>> 15), t | 1);
+        t ^= t + Math.imul(t ^ (t >>> 7), t | 61);
+        return ((t ^ (t >>> 14)) >>> 0) / 4294967296;
+    };
+}
+/**
+ * Every (task, repetition) pair exactly once, in a seeded order so provider drift over time is not
+ * correlated with one task. The same task ids, k and seed always give the same plan, whatever order
+ * the ids were listed in: the variants of a comparison run the same plan.
+ */
+export function planRuns(taskIds, k, seed) {
+    const plan = [...taskIds]
+        .sort()
+        .flatMap((taskId) => Array.from({ length: k }, (_, repetition) => ({ taskId, repetition })));
+    const random = mulberry32(seed);
+    for (let i = plan.length - 1; i > 0; i--) {
+        const j = Math.floor(random() * (i + 1));
+        [plan[i], plan[j]] = [plan[j], plan[i]];
+    }
+    return plan;
+}
+/**
+ * Expands a plan so every variant runs EVERY plan step, one after another, instead of all of A and
+ * then all of B. The starting variant rotates with the step (A,B then B,A for two variants) so no
+ * variant systematically goes first. This spreads provider drift over time across the variants
+ * and keeps the same tasks, repetitions and seeds for each; it does not remove the drift (runs
+ * still happen at different moments) and nothing is paired.
+ */
+export function interleaveVariants(plan, variantIds) {
+    const n = variantIds.length;
+    return plan.flatMap((run, step) => variantIds.map((_, offset) => ({
+        variantId: variantIds[(step + offset) % n],
+        taskId: run.taskId,
+        repetition: run.repetition,
+        step,
+    })));
+}

package/dist/eval/report.d.ts

@@ -0,0 +1,53 @@
+/**
+ * Aggregation of run records into a reliability report (specs/010 5.2, D6, D7) and its Markdown
+ * rendering. Pure: no I/O.
+ *
+ * What the numbers mean (the review of Phase 3 starts here):
+ * - A run that ended `error` is infrastructure, not a repetition of the task. It is excluded from
+ *   every rate and spread and counted in `errored`. `k` of a task is its number of JUDGED runs
+ *   (`pass`, `fail`, `budget_exceeded`); `budget_exceeded` is a non-pass that stays in the rate.
+ * - `successRate` is passes / judged runs with a 95 % Wilson interval. `passAtK` is "at least one
+ *   judged run passed", `passPowK` "every judged run passed" (false with no judged run).
+ * - Spreads (median, variance) use the judged runs; the sample variance has n - 1 in the divisor.
+ * - `tokensPerSuccess` / `wallMsPerSuccess` / `costUsdPerSuccess` are the TOTAL spent over all
+ *   judged runs (failures included) divided by the number of successes. Token and cost totals are
+ *   computed only when every judged run reported usage: a partial sum would understate them.
+ * - `measured` (provider usage, wall time) and `estimated` (money from caller-supplied rates)
+ *   never share a field. Without rates no cost is computed and no `estimated` key exists.
+ */
+import type { EvalRates, EvalReport, EvalRunRecord, EvalTaskMetrics } from "@alisio/sdk";
+/** Fewest judged runs per task below which no improvement is claimed (specs/010 O3). */
+export declare const MIN_POWER_K = 5;
+/** Runs cannot be aggregated into one report (mixed conditions, duplicates, bad rates...). */
+export declare class EvalReportError extends Error {
+    constructor(message: string);
+}
+export interface BuildReportOptions {
+    /** USD per million tokens. Without them no cost is computed (nothing is guessed). */
+    rates?: EvalRates;
+    /** Seed the runs were planned from (default: the lowest `seed` among the records). */
+    seedBase?: number;
+    /** Repetitions requested per task (default: the most records any one task has). */
+    k?: number;
+    /** The variant these runs are (`eval variants`): kept in the report with its overrides. */
+    variant?: {
+        id: string;
+        overrides: Record<string, unknown>;
+    };
+}
+/** What the token figures are and are not (they feed `tokensPerSuccess`, spreads and cost). */
+export declare const REPORT_NOTES: readonly ["Token figures are main-loop tokens (excludes compaction calls): the summarizer calls of context compaction are not counted.", "Estimated cost is derived from those tokens at flat input/output rates; it ignores cached-input and reasoning-token pricing."];
+/** Estimated cost of one run from caller-supplied rates; undefined without reported usage. */
+export declare function estimateRunCost(run: EvalRunRecord, rates: EvalRates): number | undefined;
+/**
+ * Report over `records`, which must come from one suite version, one model, one mode and one
+ * provider/harness version. `configHash` also covers each task's own policy and budget, so it may
+ * differ between tasks but never between repetitions of one task; the report's
+ * `conditions.configHash` is then the hash of the sorted `taskId -> configHash` pairs (equal to the
+ * shared value when every task has the same one).
+ */
+export declare function buildReport(records: EvalRunRecord[], options?: BuildReportOptions): EvalReport;
+/** Tasks whose judged runs fall below the power threshold. */
+export declare function lowPowerTasks(report: EvalReport): EvalTaskMetrics[];
+/** Markdown of a report: conditions, warnings, then Measured and Estimated as separate sections. */
+export declare function renderReportMarkdown(report: EvalReport): string;

package/dist/eval/report.js

@@ -0,0 +1,258 @@
+/**
+ * Aggregation of run records into a reliability report (specs/010 5.2, D6, D7) and its Markdown
+ * rendering. Pure: no I/O.
+ *
+ * What the numbers mean (the review of Phase 3 starts here):
+ * - A run that ended `error` is infrastructure, not a repetition of the task. It is excluded from
+ *   every rate and spread and counted in `errored`. `k` of a task is its number of JUDGED runs
+ *   (`pass`, `fail`, `budget_exceeded`); `budget_exceeded` is a non-pass that stays in the rate.
+ * - `successRate` is passes / judged runs with a 95 % Wilson interval. `passAtK` is "at least one
+ *   judged run passed", `passPowK` "every judged run passed" (false with no judged run).
+ * - Spreads (median, variance) use the judged runs; the sample variance has n - 1 in the divisor.
+ * - `tokensPerSuccess` / `wallMsPerSuccess` / `costUsdPerSuccess` are the TOTAL spent over all
+ *   judged runs (failures included) divided by the number of successes. Token and cost totals are
+ *   computed only when every judged run reported usage: a partial sum would understate them.
+ * - `measured` (provider usage, wall time) and `estimated` (money from caller-supplied rates)
+ *   never share a field. Without rates no cost is computed and no `estimated` key exists.
+ */
+import { createHash } from "node:crypto";
+import { median, passAtK, passPowK, sampleVariance, wilsonInterval } from "./stats.js";
+/** Fewest judged runs per task below which no improvement is claimed (specs/010 O3). */
+export const MIN_POWER_K = 5;
+/** Runs cannot be aggregated into one report (mixed conditions, duplicates, bad rates...). */
+export class EvalReportError extends Error {
+    constructor(message) {
+        super(message);
+        this.name = "EvalReportError";
+    }
+}
+/** What the token figures are and are not (they feed `tokensPerSuccess`, spreads and cost). */
+export const REPORT_NOTES = [
+    "Token figures are main-loop tokens (excludes compaction calls): the summarizer calls of context compaction are not counted.",
+    "Estimated cost is derived from those tokens at flat input/output rates; it ignores cached-input and reasoning-token pricing.",
+];
+/** Main-loop tokens of a run; undefined when usage is missing or only some turns reported it. */
+const tokensOf = (run) => run.measured.inputTokens !== undefined &&
+    run.measured.outputTokens !== undefined &&
+    !run.measured.usagePartial
+    ? run.measured.inputTokens + run.measured.outputTokens
+    : undefined;
+/** Estimated cost of one run from caller-supplied rates; undefined without reported usage. */
+export function estimateRunCost(run, rates) {
+    const { inputTokens, outputTokens, usagePartial } = run.measured;
+    if (inputTokens === undefined || outputTokens === undefined || usagePartial)
+        return undefined;
+    return (inputTokens * rates.inputPerMTokUsd + outputTokens * rates.outputPerMTokUsd) / 1_000_000;
+}
+const sum = (values) => values.reduce((a, b) => a + b, 0);
+function assertRates(rates) {
+    for (const [name, value] of Object.entries(rates))
+        if (typeof value !== "number" || !Number.isFinite(value) || value < 0)
+            throw new EvalReportError(`Rate ${name} must be a finite number of USD per million tokens, not negative (got ${value}).`);
+}
+/** Fails when `field` takes more than one value across `records`. */
+function assertUniform(records, field, hint) {
+    const values = [...new Set(records.map((r) => String(r[field])))];
+    if (values.length > 1)
+        throw new EvalReportError(`Cannot aggregate runs measured under different ${field}: ${values.map((v) => `"${v}"`).join(", ")}. ${hint}`);
+}
+function taskMetrics(taskId, runs, rates) {
+    const judged = runs.filter((r) => r.status !== "error");
+    const errored = runs.length - judged.length;
+    const successes = judged.filter((r) => r.status === "pass").length;
+    const wilson = wilsonInterval(successes, judged.length);
+    const wallTimes = judged.map((r) => r.measured.wallMs);
+    const tokenSamples = judged.flatMap((r) => {
+        const tokens = tokensOf(r);
+        return tokens === undefined ? [] : [tokens];
+    });
+    const allReportedUsage = judged.length > 0 && tokenSamples.length === judged.length;
+    const medianWallMs = median(wallTimes);
+    const varianceWallMs = sampleVariance(wallTimes);
+    // Same rule as tokensPerSuccess: a spread over the runs that happened to report would mislead.
+    const varianceTokens = allReportedUsage ? sampleVariance(tokenSamples) : null;
+    const measured = {
+        ...(medianWallMs === null ? {} : { medianWallMs }),
+        ...(varianceWallMs === null ? {} : { varianceWallMs }),
+        ...(varianceTokens === null ? {} : { varianceTokens }),
+        ...(successes > 0 && allReportedUsage
+            ? { tokensPerSuccess: sum(tokenSamples) / successes }
+            : {}),
+        ...(successes > 0 ? { wallMsPerSuccess: sum(wallTimes) / successes } : {}),
+    };
+    const costs = rates ? judged.map((r) => estimateRunCost(r, rates)) : [];
+    const costKnown = rates !== undefined && successes > 0 && costs.every((c) => c !== undefined);
+    const hashes = [...new Set(runs.map((r) => r.configHash))];
+    return {
+        taskId,
+        k: judged.length,
+        successRate: {
+            value: judged.length === 0 ? 0 : successes / judged.length,
+            wilsonLow: wilson.low,
+            wilsonHigh: wilson.high,
+        },
+        passAtK: passAtK(judged.map((r) => r.status === "pass")),
+        passPowK: passPowK(judged.map((r) => r.status === "pass")),
+        ...(errored > 0 ? { errored } : {}),
+        ...(hashes.length === 1 ? { configHash: hashes[0] } : {}),
+        measured,
+        ...(costKnown ? { estimated: { costUsdPerSuccess: sum(costs) / successes } } : {}),
+    };
+}
+/**
+ * Report over `records`, which must come from one suite version, one model, one mode and one
+ * provider/harness version. `configHash` also covers each task's own policy and budget, so it may
+ * differ between tasks but never between repetitions of one task; the report's
+ * `conditions.configHash` is then the hash of the sorted `taskId -> configHash` pairs (equal to the
+ * shared value when every task has the same one).
+ */
+export function buildReport(records, options = {}) {
+    const first = records[0];
+    if (!first)
+        throw new EvalReportError("Cannot build a report from zero runs.");
+    if (options.rates)
+        assertRates(options.rates);
+    const ids = new Set();
+    for (const run of records) {
+        if (ids.has(run.runId))
+            throw new EvalReportError(`Duplicate run ${run.runId}: a run can be aggregated only once.`);
+        ids.add(run.runId);
+    }
+    const hint = "Aggregate runs of one configuration; compare two reports with `eval compare`.";
+    for (const field of [
+        "suiteId",
+        "suiteVersion",
+        "model",
+        "mode",
+        "provider",
+        "harnessVersion",
+        "compactionHash",
+    ])
+        assertUniform(records, field, hint);
+    const byTask = new Map();
+    for (const run of records) {
+        const group = byTask.get(run.taskId);
+        if (group)
+            group.push(run);
+        else
+            byTask.set(run.taskId, [run]);
+    }
+    for (const [taskId, runs] of byTask)
+        assertUniform(runs, "configHash", `Repetitions of task "${taskId}" must share one configuration.`);
+    const sorted = [...byTask].sort(([a], [b]) => (a < b ? -1 : a > b ? 1 : 0));
+    const tasks = sorted.map(([taskId, runs]) => taskMetrics(taskId, runs, options.rates));
+    const taskHashes = sorted.map(([taskId, runs]) => [taskId, runs[0].configHash]);
+    const distinct = new Set(taskHashes.map(([, hash]) => hash));
+    const configHash = distinct.size === 1
+        ? taskHashes[0][1]
+        : `sha256:${createHash("sha256").update(JSON.stringify(taskHashes)).digest("hex")}`;
+    const count = (status) => records.filter((r) => r.status === status).length;
+    return {
+        schemaVersion: 1,
+        suiteId: first.suiteId,
+        suiteVersion: first.suiteVersion,
+        conditions: {
+            harnessVersion: first.harnessVersion,
+            configHash,
+            provider: first.provider,
+            model: first.model,
+            mode: first.mode,
+            k: options.k ?? Math.max(...[...byTask.values()].map((runs) => runs.length)),
+            seedBase: options.seedBase ?? Math.min(...records.map((r) => r.seed)),
+            ...(first.compactionHash ? { compactionHash: first.compactionHash } : {}),
+        },
+        ...(options.variant
+            ? { variant: { id: options.variant.id, overrides: { ...options.variant.overrides } } }
+            : {}),
+        notes: [...REPORT_NOTES],
+        tasks,
+        runs: records.map((r) => r.runId),
+        summary: {
+            tasks: tasks.length,
+            passed: count("pass"),
+            errored: count("error"),
+            budgetExceeded: count("budget_exceeded"),
+            runs: records.length,
+            failed: count("fail"),
+        },
+        ...(options.rates ? { rates: { ...options.rates } } : {}),
+    };
+}
+// ---------------------------------------------------------------------------------------------
+// Markdown
+// ---------------------------------------------------------------------------------------------
+const cell = (text) => text.replace(/\|/g, "\\|").replace(/\r?\n/g, " ");
+const percent = (value) => `${Math.round(value * 1000) / 10}%`;
+const seconds = (ms) => `${(Math.round(ms / 10) / 100).toFixed(2)} s`;
+const count = (value) => Math.round(value).toLocaleString("en-US");
+const optional = (value, format) => value === undefined ? "-" : format(value);
+const sd = (variance, format) => variance === undefined ? "-" : format(Math.sqrt(variance));
+function table(headers, rows) {
+    return [
+        `| ${headers.join(" | ")} |`,
+        `| ${headers.map(() => "---").join(" | ")} |`,
+        ...rows.map((row) => `| ${row.join(" | ")} |`),
+    ].join("\n");
+}
+/** Tasks whose judged runs fall below the power threshold. */
+export function lowPowerTasks(report) {
+    return report.tasks.filter((t) => t.k < MIN_POWER_K);
+}
+/** Markdown of a report: conditions, warnings, then Measured and Estimated as separate sections. */
+export function renderReportMarkdown(report) {
+    const { conditions: c } = report;
+    const lines = [
+        `# Reliability report: ${report.suiteId} ${report.suiteVersion}`,
+        "",
+        `- Suite: \`${report.suiteId}\` version \`${report.suiteVersion}\``,
+        `- Mode: **${c.mode}**`,
+        `- Provider / model: \`${c.provider}\` / \`${c.model}\``,
+        `- Harness version: \`${c.harnessVersion}\``,
+        `- Configuration hash: \`${c.configHash}\``,
+        `- Repetitions per task (k): ${c.k}; seed base: ${c.seedBase}`,
+        "",
+    ];
+    lines.push(c.mode === "replay"
+        ? "> Replay measures the harness, not the model: the provider answers come from a recording, so this says nothing about model quality."
+        : "> Live is not deterministic: the model can answer differently on every run. Repetitions and Wilson intervals estimate the noise; they do not remove it.", "");
+    const weak = lowPowerTasks(report);
+    if (weak.length > 0)
+        lines.push(`> **Low power:** ${weak.map((t) => `${t.taskId} (k=${t.k})`).join(", ")} ${weak.length === 1 ? "has" : "have"} fewer than ${MIN_POWER_K} judged runs. The intervals are wide; do not read an improvement or a regression from these numbers.`, "");
+    const { summary: s } = report;
+    lines.push(`Runs: ${s.runs ?? report.runs.length} (passed ${s.passed}, failed ${s.failed ?? "-"}, budget exceeded ${s.budgetExceeded}, errored ${s.errored}). Errored runs are infrastructure failures: they are excluded from every rate and spread below.`, "", "## Measured", "", "Provider-reported usage and wall time. Success is passes over judged runs, with a 95% Wilson interval; sd is the square root of the sample variance. Token figures are main-loop tokens (excludes compaction calls).", "", table([
+        "Task",
+        "k",
+        "Success [Wilson 95%]",
+        "pass@k",
+        "pass^k",
+        "Errored",
+        "Median wall",
+        "Wall sd",
+        "Tokens sd",
+        "Tokens / success",
+        "Wall / success",
+    ], report.tasks.map((t) => [
+        cell(t.taskId),
+        String(t.k),
+        t.k === 0
+            ? "n/a"
+            : `${percent(t.successRate.value)} [${percent(t.successRate.wilsonLow)}, ${percent(t.successRate.wilsonHigh)}]`,
+        t.passAtK ? "yes" : "no",
+        t.passPowK ? "yes" : "no",
+        String(t.errored ?? 0),
+        optional(t.measured.medianWallMs, seconds),
+        sd(t.measured.varianceWallMs, seconds),
+        sd(t.measured.varianceTokens, count),
+        optional(t.measured.tokensPerSuccess, count),
+        optional(t.measured.wallMsPerSuccess, seconds),
+    ])), "", "## Estimated", "");
+    if (!report.rates)
+        lines.push("Not computed: no rates were supplied, and Alisio does not guess prices.");
+    else {
+        lines.push(`Derived from the measured main-loop tokens (excludes compaction calls) at ${report.rates.inputPerMTokUsd} USD per million input tokens and ${report.rates.outputPerMTokUsd} USD per million output tokens (rates supplied by the caller). Cached-input and reasoning-token pricing are ignored. A task shows "-" when a run did not report usage on every turn or nothing succeeded.`, "", table(["Task", "Cost per success (USD)"], report.tasks.map((t) => [
+            cell(t.taskId),
+            optional(t.estimated?.costUsdPerSuccess, (v) => `$${v.toFixed(4)}`),
+        ])));
+    }
+    return `${lines.join("\n")}\n`;
+}

package/dist/eval/run-suite.d.ts

@@ -0,0 +1,68 @@
+/**
+ * Runs a suite k times (specs/010 D6): plans the (task, repetition) pairs, refuses the whole batch
+ * up front when a task is not allowed, then runs them one after another through `runTask` (which
+ * itself serializes: the process environment it scopes is global). The caller resolves the
+ * provider and the configuration; nothing here reads the network or the user's config.
+ */
+import type { EvalRunRecord, EvalTask, ModelProvider, RunEvent } from "@alisio/sdk";
+import type { BuiltinPlugin } from "../application.ts";
+import type { Config } from "../config.ts";
+import type { RunOptions } from "../core/runner.ts";
+import type { Cassette, CassetteRecorder } from "./cassette.ts";
+import { type EvalApplication, type EvalRunMode, type InvokerPermissions } from "./engine.ts";
+import type { LoadedSuite } from "./suite.ts";
+export interface RunEvaluationOptions {
+    suite: LoadedSuite;
+    /** Required except in `replay` mode, where each task's cassette supplies the provider. */
+    provider?: ModelProvider;
+    /** `live` (default), `record` (a live run that also fills a recorder) or `replay` (from cassettes). */
+    mode?: EvalRunMode;
+    /** The recorder (record) or the recorded cassette (replay) of one task repetition. */
+    cassetteFor?: (task: EvalTask, repetition: number) => CassetteRecorder | Cassette | undefined;
+    /** Receives every RunEvent of every run (tests and tooling). */
+    onEvent?: (event: RunEvent) => void;
+    /** Variant overrides are already applied (they must be before the configuration is hashed). */
+    config: Config;
+    invoker: InvokerPermissions;
+    /** Repetitions per task (a positive integer). */
+    k: number;
+    /** Plans the run order; each run gets `seedBase + repetition`. */
+    seedBase: number;
+    /** Subset of task ids (default: every task of the suite). */
+    taskIds?: string[];
+    allowSuiteFull?: boolean;
+    keep?: boolean;
+    builtins?: BuiltinPlugin[];
+    resolveAgent?: (app: EvalApplication, agent: string) => RunOptions | Promise<RunOptions>;
+    tmpRoot?: string;
+    onRecord?: (record: EvalRunRecord, progress: {
+        done: number;
+        total: number;
+    }) => void;
+}
+export interface RunEvaluationResult {
+    records: EvalRunRecord[];
+    /** Run directories kept because of `keep`. */
+    keptDirs: string[];
+}
+export interface RunVariantsOptions extends Omit<RunEvaluationOptions, "config" | "onRecord"> {
+    /** Each variant's overrides are already applied to its `config`. */
+    variants: Array<{
+        id: string;
+        config: Config;
+    }>;
+    onRecord?: (variantId: string, record: EvalRunRecord, progress: {
+        done: number;
+        total: number;
+    }) => void;
+}
+export declare function runEvaluation(options: RunEvaluationOptions): Promise<RunEvaluationResult>;
+/**
+ * Runs several variants over the SAME plan, interleaved at each plan step (A,B then B,A ...)
+ * instead of all of A and then all of B. The whole batch is refused up front like a single run.
+ * Remaining drift: the runs of one step still happen one after another, so provider load can
+ * change between A and B inside a step; interleaving spreads it, it does not control it.
+ */
+export declare function runVariantEvaluations(options: RunVariantsOptions): Promise<Array<{
+    id: string;
+} & RunEvaluationResult>>;

package/dist/eval/run-suite.js

@@ -0,0 +1,94 @@
+/**
+ * Runs a suite k times (specs/010 D6): plans the (task, repetition) pairs, refuses the whole batch
+ * up front when a task is not allowed, then runs them one after another through `runTask` (which
+ * itself serializes: the process environment it scopes is global). The caller resolves the
+ * provider and the configuration; nothing here reads the network or the user's config.
+ */
+import { checkTaskPolicy, EvalRunRefusedError, runTask, } from "./engine.js";
+import { interleaveVariants, planRuns } from "./plan.js";
+/** Validates the options and plans the runs shared by every variant. */
+function prepareRuns(options) {
+    if (!Number.isInteger(options.k) || options.k < 1)
+        throw new EvalRunRefusedError(`k must be a positive integer, got ${options.k}.`);
+    const all = options.suite.suite.tasks;
+    const selected = options.taskIds
+        ? [...new Set(options.taskIds)].map((id) => {
+            const found = all.find((t) => t.id === id);
+            if (!found)
+                throw new EvalRunRefusedError(`Unknown task "${id}" in suite "${options.suite.suite.suiteId}". Tasks: ${all.map((t) => t.id).join(", ")}.`);
+            return found;
+        })
+        : all;
+    for (const task of selected)
+        checkTaskPolicy(task, {
+            invoker: options.invoker,
+            ...(options.allowSuiteFull ? { allowSuiteFull: true } : {}),
+        });
+    const plan = planRuns(selected.map((t) => t.id), options.k, options.seedBase);
+    return { selected, plan };
+}
+export async function runEvaluation(options) {
+    const { selected, plan } = prepareRuns(options);
+    const records = [];
+    const keptDirs = [];
+    for (const step of plan) {
+        const task = selected.find((t) => t.id === step.taskId);
+        if (!task)
+            continue;
+        const { record, keptDir } = await runOne(options, options.config, task, step.repetition);
+        records.push(record);
+        if (keptDir)
+            keptDirs.push(keptDir);
+        options.onRecord?.(record, { done: records.length, total: plan.length });
+    }
+    return { records, keptDirs };
+}
+/**
+ * Runs several variants over the SAME plan, interleaved at each plan step (A,B then B,A ...)
+ * instead of all of A and then all of B. The whole batch is refused up front like a single run.
+ * Remaining drift: the runs of one step still happen one after another, so provider load can
+ * change between A and B inside a step; interleaving spreads it, it does not control it.
+ */
+export async function runVariantEvaluations(options) {
+    const { selected, plan } = prepareRuns(options);
+    const results = options.variants.map((variant) => ({
+        id: variant.id,
+        records: [],
+        keptDirs: [],
+    }));
+    const steps = interleaveVariants(plan, options.variants.map((v) => v.id));
+    let done = 0;
+    for (const step of steps) {
+        const task = selected.find((t) => t.id === step.taskId);
+        const variant = options.variants.find((v) => v.id === step.variantId);
+        const result = results.find((r) => r.id === step.variantId);
+        if (!task || !variant || !result)
+            continue;
+        const { record, keptDir } = await runOne(options, variant.config, task, step.repetition);
+        result.records.push(record);
+        if (keptDir)
+            result.keptDirs.push(keptDir);
+        done += 1;
+        options.onRecord?.(variant.id, record, { done, total: steps.length });
+    }
+    return results;
+}
+function runOne(options, config, task, repetition) {
+    return runTask(task, {
+        suite: options.suite,
+        ...(options.provider ? { provider: options.provider } : {}),
+        ...(options.mode ? { mode: options.mode } : {}),
+        ...(options.cassetteFor ? cassetteOption(options.cassetteFor(task, repetition)) : {}),
+        ...(options.onEvent ? { onEvent: options.onEvent } : {}),
+        config,
+        seed: options.seedBase + repetition,
+        repetition,
+        invoker: options.invoker,
+        ...(options.allowSuiteFull ? { allowSuiteFull: true } : {}),
+        ...(options.keep ? { keep: true } : {}),
+        ...(options.builtins ? { builtins: options.builtins } : {}),
+        ...(options.resolveAgent ? { resolveAgent: options.resolveAgent } : {}),
+        ...(options.tmpRoot ? { tmpRoot: options.tmpRoot } : {}),
+    });
+}
+const cassetteOption = (cassette) => cassette ? { cassette } : {};

package/dist/eval/secrets.d.ts

@@ -0,0 +1,12 @@
+/**
+ * Secret scan applied to everything a cassette stores and every diff excerpt it prints (specs/010
+ * 5.3, 10). It looks for well-known credential shapes plus the exact values of the credentials the
+ * run could hold. A hit on write REFUSES the cassette (nothing is stripped: a stripped cassette
+ * would silently stop matching what the model saw); diagnostics are redacted instead.
+ *
+ * Findings carry the rule name, never the matched text.
+ */
+/** Names of the rules `text` matches (each once). `exact` are credential values to look for verbatim. */
+export declare function findSecrets(text: string, exact?: readonly string[]): string[];
+/** `text` with every match replaced by `[redacted]`. */
+export declare function redactSecrets(text: string, exact?: readonly string[]): string;

package/dist/eval/secrets.js

@@ -0,0 +1,45 @@
+/**
+ * Secret scan applied to everything a cassette stores and every diff excerpt it prints (specs/010
+ * 5.3, 10). It looks for well-known credential shapes plus the exact values of the credentials the
+ * run could hold. A hit on write REFUSES the cassette (nothing is stripped: a stripped cassette
+ * would silently stop matching what the model saw); diagnostics are redacted instead.
+ *
+ * Findings carry the rule name, never the matched text.
+ */
+const RULES = [
+    { name: "Anthropic API key (sk-ant-...)", pattern: /sk-ant-[A-Za-z0-9_-]{16,}/ },
+    { name: "API key (sk-...)", pattern: /\bsk-[A-Za-z0-9_-]{20,}/ },
+    { name: "GitHub token (ghp_/gho_)", pattern: /\bgh[po]_[A-Za-z0-9]{20,}/ },
+    { name: "GitHub fine-grained token (github_pat_)", pattern: /\bgithub_pat_[A-Za-z0-9_]{20,}/ },
+    { name: "AWS access key id (AKIA...)", pattern: /\bAKIA[0-9A-Z]{16}\b/ },
+    { name: "Slack token (xoxb-/xoxp-)", pattern: /\bxox[bp]-[A-Za-z0-9-]{10,}/ },
+    { name: "Bearer token", pattern: /\bBearer\s+[A-Za-z0-9._~+/=-]{16,}/ },
+    { name: "PEM private key header", pattern: /-----BEGIN [A-Z ]*PRIVATE KEY-----/ },
+    {
+        name: "*_API_KEY assignment",
+        pattern: /\b[A-Z][A-Z0-9_]*_API_KEY\s*=\s*['"]?[^\s'"]{4,}/,
+    },
+];
+/** Exact credential values shorter than this are ignored: they would match ordinary text. */
+const MIN_EXACT_LENGTH = 8;
+const escape = (text) => text.replace(/[.*+?^${}()|[\]\\]/g, "\\$&");
+/** Names of the rules `text` matches (each once). `exact` are credential values to look for verbatim. */
+export function findSecrets(text, exact = []) {
+    const hits = [];
+    for (const rule of RULES)
+        if (rule.pattern.test(text))
+            hits.push(rule.name);
+    if (exact.some((value) => value.length >= MIN_EXACT_LENGTH && text.includes(value)))
+        hits.push("a credential of the provider profile or environment");
+    return hits;
+}
+/** `text` with every match replaced by `[redacted]`. */
+export function redactSecrets(text, exact = []) {
+    let out = text;
+    for (const rule of RULES)
+        out = out.replace(new RegExp(rule.pattern, "g"), "[redacted]");
+    for (const value of exact)
+        if (value.length >= MIN_EXACT_LENGTH)
+            out = out.replace(new RegExp(escape(value), "g"), "[redacted]");
+    return out;
+}

package/dist/eval/stats.d.ts

@@ -0,0 +1,17 @@
+/** Pure statistics of the reliability lab (specs/010 D6, D17). No I/O, no dependencies. */
+/**
+ * Wilson score interval of a proportion (default 95 %). Unlike the normal approximation it stays
+ * inside [0, 1] and is informative at 0/n and n/n. With no trials it carries no information.
+ */
+export declare function wilsonInterval(successes: number, trials: number, z?: number): {
+    low: number;
+    high: number;
+};
+/** Median; null for an empty set. Does not mutate the input. */
+export declare function median(values: number[]): number | null;
+/** Sample variance (n - 1), the same convention as `scripts/eval-baseline.ts`; null below two samples. */
+export declare function sampleVariance(values: number[]): number | null;
+/** pass@k: at least one of the k repetitions succeeded. No repetitions is not a pass. */
+export declare function passAtK(results: boolean[]): boolean;
+/** pass^k: every one of the k repetitions succeeded (consistency). No repetitions is not a pass. */
+export declare function passPowK(results: boolean[]): boolean;