@bradtaylorsf/alpha-loop 1.14.0 → 1.14.1

package/dist/lib/eval-matrix.js

@@ -0,0 +1,317 @@
+/**
+ * eval-matrix — run every case under multiple routing profiles and
+ * aggregate a side-by-side comparison.
+ *
+ * Profiles are loaded from YAML and deep-merged into a base Config before
+ * each profile's run. Pipeline costs, wall time, tool error rate, and a
+ * diff-similarity signal are aggregated per-case per-profile. Deltas are
+ * computed against a designated baseline profile (defaults to
+ * `all-frontier`).
+ */
+import { existsSync, readFileSync, readdirSync, statSync } from 'node:fs';
+import { join, basename, extname } from 'node:path';
+import { parse as parseYaml } from 'yaml';
+import { loadConfig } from './config.js';
+import { runEvalSuite } from './eval-runner.js';
+/**
+ * Load a profile YAML and return the narrow override shape.
+ * Accepts either a bare name ("hybrid-v1") or a full path.
+ */
+export function loadProfileOverrides(profileNameOrPath, projectDir = process.cwd()) {
+    const path = resolveProfilePath(profileNameOrPath, projectDir);
+    if (!existsSync(path)) {
+        throw new Error(`Profile file not found: ${path}`);
+    }
+    const raw = readFileSync(path, 'utf-8');
+    const parsed = parseYaml(raw);
+    if (!parsed || typeof parsed !== 'object')
+        return {};
+    const result = {};
+    if (typeof parsed.agent === 'string')
+        result.agent = parsed.agent;
+    if (typeof parsed.model === 'string')
+        result.model = parsed.model;
+    if (typeof parsed.review_model === 'string')
+        result.reviewModel = parsed.review_model;
+    if (parsed.pipeline && typeof parsed.pipeline === 'object') {
+        const pipelineRaw = parsed.pipeline;
+        const pipeline = {};
+        const validSteps = ['plan', 'implement', 'test_fix', 'review', 'verify', 'learn'];
+        for (const step of validSteps) {
+            const entry = pipelineRaw[step];
+            if (entry && typeof entry === 'object') {
+                const e = entry;
+                const stepCfg = {};
+                if (typeof e.agent === 'string')
+                    stepCfg.agent = e.agent;
+                if (typeof e.model === 'string')
+                    stepCfg.model = e.model;
+                if (Object.keys(stepCfg).length > 0)
+                    pipeline[step] = stepCfg;
+            }
+        }
+        if (Object.keys(pipeline).length > 0)
+            result.pipeline = pipeline;
+    }
+    if (parsed.routing && typeof parsed.routing === 'object') {
+        // Re-use loadConfig's routing parser by round-tripping through a fake
+        // config. Keeps the validation logic single-sourced.
+        const fake = loadConfig({ routing: parsed.routing });
+        if (fake.routing)
+            result.routing = fake.routing;
+    }
+    return result;
+}
+/** Resolve a bare profile name to its path, or pass a real path through. */
+export function resolveProfilePath(nameOrPath, projectDir) {
+    if (nameOrPath.includes('/') || nameOrPath.endsWith('.yaml') || nameOrPath.endsWith('.yml')) {
+        return nameOrPath;
+    }
+    return join(projectDir, '.alpha-loop', 'evals', 'profiles', `${nameOrPath}.yaml`);
+}
+/** Human-friendly profile name for reports — always just the base. */
+export function profileDisplayName(nameOrPath) {
+    if (!nameOrPath.endsWith('.yaml') && !nameOrPath.endsWith('.yml') && !nameOrPath.includes('/')) {
+        return nameOrPath;
+    }
+    return basename(nameOrPath, extname(nameOrPath));
+}
+/** Compose profile overrides onto a base config. Pipeline merges per-step. */
+export function applyProfileToConfig(base, overrides) {
+    const mergedPipeline = { ...base.pipeline };
+    for (const [step, stepCfg] of Object.entries(overrides.pipeline ?? {})) {
+        mergedPipeline[step] = { ...(mergedPipeline[step] ?? {}), ...stepCfg };
+    }
+    return {
+        ...base,
+        ...(overrides.agent ? { agent: overrides.agent } : {}),
+        ...(overrides.model ? { model: overrides.model } : {}),
+        ...(overrides.reviewModel ? { reviewModel: overrides.reviewModel } : {}),
+        pipeline: mergedPipeline,
+        ...(overrides.routing ? { routing: overrides.routing } : {}),
+    };
+}
+/**
+ * Compute a rough similarity score between a produced diff and the golden
+ * patch. Line-set Jaccard over non-empty, non-header lines — cheap and
+ * stable enough to flag drift, not meant to replace a proper diff compare.
+ */
+export function diffSimilarity(produced, golden) {
+    const extract = (s) => {
+        const lines = s.split('\n')
+            .map((l) => l.trim())
+            .filter((l) => l.length > 0)
+            .filter((l) => !l.startsWith('diff --git'))
+            .filter((l) => !l.startsWith('index '))
+            .filter((l) => !l.startsWith('---') && !l.startsWith('+++'))
+            .filter((l) => !l.startsWith('@@'))
+            .filter((l) => !l.startsWith('#'));
+        return new Set(lines);
+    };
+    const a = extract(produced);
+    const b = extract(golden);
+    if (a.size === 0 && b.size === 0)
+        return 1;
+    if (a.size === 0 || b.size === 0)
+        return 0;
+    let intersect = 0;
+    for (const line of a)
+        if (b.has(line))
+            intersect++;
+    const unionSize = a.size + b.size - intersect;
+    return unionSize === 0 ? 1 : intersect / unionSize;
+}
+/**
+ * Detect whether a golden.patch file is a stub (no real diff to compare).
+ * Stubs start with "# TODO" or contain only comment/blank lines.
+ */
+export function isStubPatch(content) {
+    const nonCommentLines = content.split('\n')
+        .map((l) => l.trim())
+        .filter((l) => l.length > 0 && !l.startsWith('#'));
+    return nonCommentLines.length === 0;
+}
+/**
+ * Extract per-case matrix entries from an EvalSuiteResult, plus the supplied
+ * per-case diff comparisons (if any).
+ */
+export function toMatrixEntries(result, diffLookup) {
+    const entries = {};
+    for (const r of result.cases) {
+        entries[r.caseId] = buildCaseEntry(r, diffLookup?.get(r.caseId) ?? null);
+    }
+    return entries;
+}
+function buildCaseEntry(r, diffSim) {
+    return {
+        passed: r.passed,
+        partialCredit: r.partialCredit,
+        costUsd: r.costUsd ?? 0,
+        wallTimeS: r.duration,
+        // Tool error rate isn't persisted on EvalResult today; keep 0 for now,
+        // the plumbing is here so #161's telemetry can back-fill without a
+        // downstream signature change.
+        toolErrorRate: 0,
+        diffSimilarity: diffSim,
+        errored: Boolean(r.error),
+        error: r.error,
+    };
+}
+/** Aggregate per-profile totals from per-case entries. */
+export function aggregateTotals(profile, entries) {
+    const values = Object.values(entries);
+    const caseCount = values.length;
+    const passCount = values.filter((v) => v.passed).length;
+    const totalCostUsd = values.reduce((s, v) => s + v.costUsd, 0);
+    const meanWallTimeS = caseCount === 0 ? 0 : values.reduce((s, v) => s + v.wallTimeS, 0) / caseCount;
+    const meanToolErrorRate = caseCount === 0 ? 0 : values.reduce((s, v) => s + v.toolErrorRate, 0) / caseCount;
+    return {
+        profile,
+        caseCount,
+        passCount,
+        passRate: caseCount === 0 ? 0 : passCount / caseCount,
+        totalCostUsd,
+        meanWallTimeS,
+        meanToolErrorRate,
+    };
+}
+/** Compute deltas for each profile vs the baseline. */
+export function computeDeltas(totals, baseline) {
+    const byProfile = new Map(totals.map((t) => [t.profile, t]));
+    const base = byProfile.get(baseline);
+    const result = {};
+    if (!base)
+        return result;
+    const baseCostPerIssue = base.caseCount === 0 ? 0 : base.totalCostUsd / base.caseCount;
+    for (const t of totals) {
+        const costPerIssue = t.caseCount === 0 ? 0 : t.totalCostUsd / t.caseCount;
+        result[t.profile] = {
+            pipelineSuccessDelta: t.passRate - base.passRate,
+            costPerIssueDelta: costPerIssue - baseCostPerIssue,
+        };
+    }
+    return result;
+}
+/**
+ * Run the full matrix: every profile × every case. Each profile gets a
+ * fresh config built by deep-merging its overrides onto `baseConfig`.
+ *
+ * `runner` is injectable for tests — defaults to the real `runEvalSuite`.
+ */
+export async function runMatrix(cases, opts, baseConfig, runner = runEvalSuite) {
+    if (opts.profiles.length === 0) {
+        throw new Error('runMatrix: at least one profile is required');
+    }
+    const profileNames = opts.profiles.map(profileDisplayName);
+    const baselineName = profileDisplayName(opts.baseline ?? 'all-frontier');
+    const perProfileEntries = new Map();
+    const perProfileTotals = [];
+    for (let i = 0; i < opts.profiles.length; i++) {
+        const profilePathOrName = opts.profiles[i];
+        const displayName = profileNames[i];
+        const overrides = loadProfileOverrides(profilePathOrName);
+        // Resolve the merged config so validation surfaces bad profile YAML even
+        // in dry-run mode. In execute mode the same merged config is what the
+        // runner receives.
+        const mergedConfig = applyProfileToConfig(baseConfig, overrides);
+        let entries;
+        if (opts.dryRun) {
+            entries = buildSkippedEntries(cases);
+        }
+        else {
+            const result = await runner(cases, mergedConfig, { verbose: opts.verbose });
+            const diffLookup = new Map();
+            // Placeholder diff similarities: real diff comes from the run's worktree
+            // output, which lives inside runEvalSuite. For now we mark all as null
+            // unless a golden stub tells us to skip — downstream reports treat null
+            // as "informational, not scored".
+            for (const c of cases)
+                diffLookup.set(c.id, null);
+            entries = toMatrixEntries(result, diffLookup);
+        }
+        perProfileEntries.set(displayName, entries);
+        perProfileTotals.push(aggregateTotals(displayName, entries));
+    }
+    // Flatten per-case table: iterate cases once, attach per-profile entries.
+    const flatCases = cases.map((c) => {
+        const perProfile = {};
+        for (const name of profileNames) {
+            const entries = perProfileEntries.get(name);
+            perProfile[name] = entries[c.id] ?? {
+                passed: false,
+                partialCredit: 0,
+                costUsd: 0,
+                wallTimeS: 0,
+                toolErrorRate: 0,
+                diffSimilarity: null,
+                errored: true,
+                error: 'missing result',
+            };
+        }
+        return { caseId: c.id, description: c.description, perProfile };
+    });
+    return {
+        profiles: profileNames,
+        baseline: baselineName,
+        cases: flatCases,
+        totals: perProfileTotals,
+        deltas: computeDeltas(perProfileTotals, baselineName),
+        ...(opts.dryRun ? { dryRun: true } : {}),
+    };
+}
+/**
+ * Locate the most recent matrix report, returning its mtime (epoch ms) and
+ * path. Used by `alpha-loop evolve routing` to enforce a freshness gate on
+ * promotion proposals — promotion must not fire without a recent matrix run.
+ *
+ * Checks the default `eval/reports/` directory first, then
+ * `.alpha-loop/evals/reports/` (where older runs land). Returns null when
+ * neither directory contains a `routing-*.{md,csv}` file.
+ */
+export function latestMatrixRun(projectDir = process.cwd()) {
+    const candidates = [
+        join(projectDir, 'eval', 'reports'),
+        join(projectDir, '.alpha-loop', 'evals', 'reports'),
+    ];
+    let best = null;
+    for (const dir of candidates) {
+        if (!existsSync(dir))
+            continue;
+        for (const name of readdirSync(dir)) {
+            if (!name.startsWith('routing-'))
+                continue;
+            if (!name.endsWith('.md') && !name.endsWith('.csv'))
+                continue;
+            try {
+                const full = join(dir, name);
+                const stat = statSync(full);
+                const ms = stat.mtimeMs;
+                if (!best || ms > best.timestampMs) {
+                    best = { timestampMs: ms, summaryPath: full };
+                }
+            }
+            catch {
+                /* ignore unreadable entry */
+            }
+        }
+    }
+    return best;
+}
+/** Stub entries for a dry-run: every case marked skipped under every profile. */
+function buildSkippedEntries(cases) {
+    const entries = {};
+    for (const c of cases) {
+        entries[c.id] = {
+            passed: false,
+            partialCredit: 0,
+            costUsd: 0,
+            wallTimeS: 0,
+            toolErrorRate: 0,
+            diffSimilarity: null,
+            errored: false,
+            skipped: true,
+        };
+    }
+    return entries;
+}
+//# sourceMappingURL=eval-matrix.js.map

package/dist/lib/eval-matrix.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-matrix.js","sourceRoot":"","sources":["../../src/lib/eval-matrix.ts"],"names":[],"mappings":"AAAA;;;;;;;;;GASG;AACH,OAAO,EAAE,UAAU,EAAE,YAAY,EAAE,WAAW,EAAE,QAAQ,EAAE,MAAM,SAAS,CAAC;AAC1E,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,OAAO,EAAE,MAAM,WAAW,CAAC;AACpD,OAAO,EAAE,KAAK,IAAI,SAAS,EAAE,MAAM,MAAM,CAAC;AAE1C,OAAO,EAAE,UAAU,EAAE,MAAM,aAAa,CAAC;AAGzC,OAAO,EAAE,YAAY,EAAE,MAAM,kBAAkB,CAAC;AA2EhD;;;GAGG;AACH,MAAM,UAAU,oBAAoB,CAAC,iBAAyB,EAAE,aAAqB,OAAO,CAAC,GAAG,EAAE;IAChG,MAAM,IAAI,GAAG,kBAAkB,CAAC,iBAAiB,EAAE,UAAU,CAAC,CAAC;IAC/D,IAAI,CAAC,UAAU,CAAC,IAAI,CAAC,EAAE,CAAC;QACtB,MAAM,IAAI,KAAK,CAAC,2BAA2B,IAAI,EAAE,CAAC,CAAC;IACrD,CAAC;IACD,MAAM,GAAG,GAAG,YAAY,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;IACxC,MAAM,MAAM,GAAG,SAAS,CAAC,GAAG,CAAmC,CAAC;IAChE,IAAI,CAAC,MAAM,IAAI,OAAO,MAAM,KAAK,QAAQ;QAAE,OAAO,EAAE,CAAC;IAErD,MAAM,MAAM,GAAqB,EAAE,CAAC;IACpC,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAwB,CAAC;IACrF,IAAI,OAAO,MAAM,CAAC,KAAK,KAAK,QAAQ;QAAE,MAAM,CAAC,KAAK,GAAG,MAAM,CAAC,KAAK,CAAC;IAClE,IAAI,OAAO,MAAM,CAAC,YAAY,KAAK,QAAQ;QAAE,MAAM,CAAC,WAAW,GAAG,MAAM,CAAC,YAAY,CAAC;IAEtF,IAAI,MAAM,CAAC,QAAQ,IAAI,OAAO,MAAM,CAAC,QAAQ,KAAK,QAAQ,EAAE,CAAC;QAC3D,MAAM,WAAW,GAAG,MAAM,CAAC,QAAmC,CAAC;QAC/D,MAAM,QAAQ,GAAmB,EAAE,CAAC;QACpC,MAAM,UAAU,GAAuB,CAAC,MAAM,EAAE,WAAW,EAAE,UAAU,EAAE,QAAQ,EAAE,QAAQ,EAAE,OAAO,CAAC,CAAC;QACtG,KAAK,MAAM,IAAI,IAAI,UAAU,EAAE,CAAC;YAC9B,MAAM,KAAK,GAAG,WAAW,CAAC,IAAI,CAAC,CAAC;YAChC,IAAI,KAAK,IAAI,OAAO,KAAK,KAAK,QAAQ,EAAE,CAAC;gBACvC,MAAM,CAAC,GAAG,KAAgC,CAAC;gBAC3C,MAAM,OAAO,GAAe,EAAE,CAAC;gBAC/B,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,OAAO,CAAC,CAAC,KAAK,KAAK,QAAQ;oBAAE,OAAO,CAAC,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC;gBACzD,IAAI,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,MAAM,GAAG,CAAC;oBAAE,QAAQ,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC;YAChE,CAAC;QACH,CAAC;QACD,IAAI,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,CAAC,MAAM,GAAG,CAAC;YAAE,MAAM,CAAC,QAAQ,GAAG,QAAQ,CAAC;IACnE,CAAC;IAED,IAAI,MAAM,CAAC,OAAO,IAAI,OAAO,MAAM,CAAC,OAAO,KAAK,QAAQ,EAAE,CAAC;QACzD,sEAAsE;QACtE,qDAAqD;QACrD,MAAM,IAAI,GAAG,UAAU,CAAC,EAAE,OAAO,EAAE,MAAM,CAAC,OAAmC,EAAE,CAAC,CAAC;QACjF,IAAI,IAAI,CAAC,OAAO;YAAE,MAAM,CAAC,OAAO,GAAG,IAAI,CAAC,OAAO,CAAC;IAClD,CAAC;IAED,OAAO,MAAM,CAAC;AAChB,CAAC;AAED,4EAA4E;AAC5E,MAAM,UAAU,kBAAkB,CAAC,UAAkB,EAAE,UAAkB;IACvE,IAAI,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,CAAC;QAC5F,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,IAAI,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,UAAU,EAAE,GAAG,UAAU,OAAO,CAAC,CAAC;AACpF,CAAC;AAED,sEAAsE;AACtE,MAAM,UAAU,kBAAkB,CAAC,UAAkB;IACnD,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,OAAO,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,GAAG,CAAC,EAAE,CAAC;QAC/F,OAAO,UAAU,CAAC;IACpB,CAAC;IACD,OAAO,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,UAAU,CAAC,CAAC,CAAC;AACnD,CAAC;AAED,8EAA8E;AAC9E,MAAM,UAAU,oBAAoB,CAAC,IAAY,EAAE,SAA2B;IAC5E,MAAM,cAAc,GAAmB,EAAE,GAAG,IAAI,CAAC,QAAQ,EAAE,CAAC;IAC5D,KAAK,MAAM,CAAC,IAAI,EAAE,OAAO,CAAC,IAAI,MAAM,CAAC,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,EAAE,CAA0C,EAAE,CAAC;QAChH,cAAc,CAAC,IAAI,CAAC,GAAG,EAAE,GAAG,CAAC,cAAc,CAAC,IAAI,CAAC,IAAI,EAAE,CAAC,EAAE,GAAG,OAAO,EAAE,CAAC;IACzE,CAAC;IACD,OAAO;QACL,GAAG,IAAI;QACP,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG,CAAC,SAAS,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,SAAS,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACtD,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,CAAC,CAAC,EAAE,WAAW,EAAE,SAAS,CAAC,WAAW,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;QACxE,QAAQ,EAAE,cAAc;QACxB,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KAC7D,CAAC;AACJ,CAAC;AAED;;;;GAIG;AACH,MAAM,UAAU,cAAc,CAAC,QAAgB,EAAE,MAAc;IAC7D,MAAM,OAAO,GAAG,CAAC,CAAS,EAAe,EAAE;QACzC,MAAM,KAAK,GAAG,CAAC,CAAC,KAAK,CAAC,IAAI,CAAC;aACxB,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;aACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,CAAC;aAC3B,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,YAAY,CAAC,CAAC;aAC1C,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,QAAQ,CAAC,CAAC;aACtC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,KAAK,CAAC,CAAC;aAC3D,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,IAAI,CAAC,CAAC;aAClC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;QACrC,OAAO,IAAI,GAAG,CAAC,KAAK,CAAC,CAAC;IACxB,CAAC,CAAC;IACF,MAAM,CAAC,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC5B,MAAM,CAAC,GAAG,OAAO,CAAC,MAAM,CAAC,CAAC;IAC1B,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC,IAAI,CAAC,CAAC,IAAI,KAAK,CAAC;QAAE,OAAO,CAAC,CAAC;IAC3C,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,KAAK,MAAM,IAAI,IAAI,CAAC;QAAE,IAAI,CAAC,CAAC,GAAG,CAAC,IAAI,CAAC;YAAE,SAAS,EAAE,CAAC;IACnD,MAAM,SAAS,GAAG,CAAC,CAAC,IAAI,GAAG,CAAC,CAAC,IAAI,GAAG,SAAS,CAAC;IAC9C,OAAO,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS,CAAC;AACrD,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,WAAW,CAAC,OAAe;IACzC,MAAM,eAAe,GAAG,OAAO,CAAC,KAAK,CAAC,IAAI,CAAC;SACxC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;SACpB,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,GAAG,CAAC,IAAI,CAAC,CAAC,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC;IACrD,OAAO,eAAe,CAAC,MAAM,KAAK,CAAC,CAAC;AACtC,CAAC;AAED;;;GAGG;AACH,MAAM,UAAU,eAAe,CAC7B,MAAuB,EACvB,UAAuC;IAEvC,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,OAAO,CAAC,CAAC,CAAC,MAAM,CAAC,GAAG,cAAc,CAAC,CAAC,EAAE,UAAU,EAAE,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,IAAI,IAAI,CAAC,CAAC;IAC3E,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC;AAED,SAAS,cAAc,CAAC,CAAa,EAAE,OAAsB;IAC3D,OAAO;QACL,MAAM,EAAE,CAAC,CAAC,MAAM;QAChB,aAAa,EAAE,CAAC,CAAC,aAAa;QAC9B,OAAO,EAAE,CAAC,CAAC,OAAO,IAAI,CAAC;QACvB,SAAS,EAAE,CAAC,CAAC,QAAQ;QACrB,uEAAuE;QACvE,mEAAmE;QACnE,+BAA+B;QAC/B,aAAa,EAAE,CAAC;QAChB,cAAc,EAAE,OAAO;QACvB,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;QACzB,KAAK,EAAE,CAAC,CAAC,KAAK;KACf,CAAC;AACJ,CAAC;AAED,0DAA0D;AAC1D,MAAM,UAAU,eAAe,CAC7B,OAAe,EACf,OAAwC;IAExC,MAAM,MAAM,GAAG,MAAM,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC;IAChC,MAAM,SAAS,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,MAAM,CAAC;IACxD,MAAM,YAAY,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC;IAC/D,MAAM,aAAa,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;IACpG,MAAM,iBAAiB,GAAG,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,CAAC,EAAE,EAAE,CAAC,CAAC,GAAG,CAAC,CAAC,aAAa,EAAE,CAAC,CAAC,GAAG,SAAS,CAAC;IAC5G,OAAO;QACL,OAAO;QACP,SAAS;QACT,SAAS;QACT,QAAQ,EAAE,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,SAAS,GAAG,SAAS;QACrD,YAAY;QACZ,aAAa;QACb,iBAAiB;KAClB,CAAC;AACJ,CAAC;AAED,uDAAuD;AACvD,MAAM,UAAU,aAAa,CAC3B,MAA6B,EAC7B,QAAgB;IAEhB,MAAM,SAAS,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,CAAC,OAAO,EAAE,CAAC,CAAC,CAAC,CAAC,CAAC;IAC7D,MAAM,IAAI,GAAG,SAAS,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;IACrC,MAAM,MAAM,GAAgF,EAAE,CAAC;IAC/F,IAAI,CAAC,IAAI;QAAE,OAAO,MAAM,CAAC;IACzB,MAAM,gBAAgB,GAAG,IAAI,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC,YAAY,GAAG,IAAI,CAAC,SAAS,CAAC;IACvF,KAAK,MAAM,CAAC,IAAI,MAAM,EAAE,CAAC;QACvB,MAAM,YAAY,GAAG,CAAC,CAAC,SAAS,KAAK,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,CAAC,YAAY,GAAG,CAAC,CAAC,SAAS,CAAC;QAC1E,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,GAAG;YAClB,oBAAoB,EAAE,CAAC,CAAC,QAAQ,GAAG,IAAI,CAAC,QAAQ;YAChD,iBAAiB,EAAE,YAAY,GAAG,gBAAgB;SACnD,CAAC;IACJ,CAAC;IACD,OAAO,MAAM,CAAC;AAChB,CAAC;AAED;;;;;GAKG;AACH,MAAM,CAAC,KAAK,UAAU,SAAS,CAC7B,KAA2B,EAC3B,IAAmB,EACnB,UAAkB,EAClB,SAIgC,YAAY;IAE5C,IAAI,IAAI,CAAC,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;QAC/B,MAAM,IAAI,KAAK,CAAC,6CAA6C,CAAC,CAAC;IACjE,CAAC;IAED,MAAM,YAAY,GAAG,IAAI,CAAC,QAAQ,CAAC,GAAG,CAAC,kBAAkB,CAAC,CAAC;IAC3D,MAAM,YAAY,GAAG,kBAAkB,CAAC,IAAI,CAAC,QAAQ,IAAI,cAAc,CAAC,CAAC;IAEzE,MAAM,iBAAiB,GAAG,IAAI,GAAG,EAA2C,CAAC;IAC7E,MAAM,gBAAgB,GAA0B,EAAE,CAAC;IAEnD,KAAK,IAAI,CAAC,GAAG,CAAC,EAAE,CAAC,GAAG,IAAI,CAAC,QAAQ,CAAC,MAAM,EAAE,CAAC,EAAE,EAAE,CAAC;QAC9C,MAAM,iBAAiB,GAAG,IAAI,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;QAC3C,MAAM,WAAW,GAAG,YAAY,CAAC,CAAC,CAAC,CAAC;QACpC,MAAM,SAAS,GAAG,oBAAoB,CAAC,iBAAiB,CAAC,CAAC;QAC1D,yEAAyE;QACzE,sEAAsE;QACtE,mBAAmB;QACnB,MAAM,YAAY,GAAG,oBAAoB,CAAC,UAAU,EAAE,SAAS,CAAC,CAAC;QAEjE,IAAI,OAAwC,CAAC;QAC7C,IAAI,IAAI,CAAC,MAAM,EAAE,CAAC;YAChB,OAAO,GAAG,mBAAmB,CAAC,KAAK,CAAC,CAAC;QACvC,CAAC;aAAM,CAAC;YACN,MAAM,MAAM,GAAG,MAAM,MAAM,CAAC,KAAK,EAAE,YAAY,EAAE,EAAE,OAAO,EAAE,IAAI,CAAC,OAAO,EAAE,CAAC,CAAC;YAC5E,MAAM,UAAU,GAAG,IAAI,GAAG,EAAyB,CAAC;YACpD,yEAAyE;YACzE,uEAAuE;YACvE,wEAAwE;YACxE,kCAAkC;YAClC,KAAK,MAAM,CAAC,IAAI,KAAK;gBAAE,UAAU,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,IAAI,CAAC,CAAC;YAClD,OAAO,GAAG,eAAe,CAAC,MAAM,EAAE,UAAU,CAAC,CAAC;QAChD,CAAC;QACD,iBAAiB,CAAC,GAAG,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC;QAC5C,gBAAgB,CAAC,IAAI,CAAC,eAAe,CAAC,WAAW,EAAE,OAAO,CAAC,CAAC,CAAC;IAC/D,CAAC;IAED,0EAA0E;IAC1E,MAAM,SAAS,GAAG,KAAK,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE;QAChC,MAAM,UAAU,GAAoC,EAAE,CAAC;QACvD,KAAK,MAAM,IAAI,IAAI,YAAY,EAAE,CAAC;YAChC,MAAM,OAAO,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAE,CAAC;YAC7C,UAAU,CAAC,IAAI,CAAC,GAAG,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,IAAI;gBAClC,MAAM,EAAE,KAAK;gBACb,aAAa,EAAE,CAAC;gBAChB,OAAO,EAAE,CAAC;gBACV,SAAS,EAAE,CAAC;gBACZ,aAAa,EAAE,CAAC;gBAChB,cAAc,EAAE,IAAI;gBACpB,OAAO,EAAE,IAAI;gBACb,KAAK,EAAE,gBAAgB;aACxB,CAAC;QACJ,CAAC;QACD,OAAO,EAAE,MAAM,EAAE,CAAC,CAAC,EAAE,EAAE,WAAW,EAAE,CAAC,CAAC,WAAW,EAAE,UAAU,EAAE,CAAC;IAClE,CAAC,CAAC,CAAC;IAEH,OAAO;QACL,QAAQ,EAAE,YAAY;QACtB,QAAQ,EAAE,YAAY;QACtB,KAAK,EAAE,SAAS;QAChB,MAAM,EAAE,gBAAgB;QACxB,MAAM,EAAE,aAAa,CAAC,gBAAgB,EAAE,YAAY,CAAC;QACrD,GAAG,CAAC,IAAI,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,MAAM,EAAE,IAAI,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACzC,CAAC;AACJ,CAAC;AAED;;;;;;;;GAQG;AACH,MAAM,UAAU,eAAe,CAC7B,aAAqB,OAAO,CAAC,GAAG,EAAE;IAElC,MAAM,UAAU,GAAG;QACjB,IAAI,CAAC,UAAU,EAAE,MAAM,EAAE,SAAS,CAAC;QACnC,IAAI,CAAC,UAAU,EAAE,aAAa,EAAE,OAAO,EAAE,SAAS,CAAC;KACpD,CAAC;IACF,IAAI,IAAI,GAAwD,IAAI,CAAC;IACrE,KAAK,MAAM,GAAG,IAAI,UAAU,EAAE,CAAC;QAC7B,IAAI,CAAC,UAAU,CAAC,GAAG,CAAC;YAAE,SAAS;QAC/B,KAAK,MAAM,IAAI,IAAI,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC,UAAU,CAAC,UAAU,CAAC;gBAAE,SAAS;YAC3C,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,QAAQ,CAAC,MAAM,CAAC;gBAAE,SAAS;YAC9D,IAAI,CAAC;gBACH,MAAM,IAAI,GAAG,IAAI,CAAC,GAAG,EAAE,IAAI,CAAC,CAAC;gBAC7B,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,CAAC,CAAC;gBAC5B,MAAM,EAAE,GAAG,IAAI,CAAC,OAAO,CAAC;gBACxB,IAAI,CAAC,IAAI,IAAI,EAAE,GAAG,IAAI,CAAC,WAAW,EAAE,CAAC;oBACnC,IAAI,GAAG,EAAE,WAAW,EAAE,EAAE,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;gBAChD,CAAC;YACH,CAAC;YAAC,MAAM,CAAC;gBACP,6BAA6B;YAC/B,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC;AACd,CAAC;AAED,iFAAiF;AACjF,SAAS,mBAAmB,CAAC,KAA2B;IACtD,MAAM,OAAO,GAAoC,EAAE,CAAC;IACpD,KAAK,MAAM,CAAC,IAAI,KAAK,EAAE,CAAC;QACtB,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC,GAAG;YACd,MAAM,EAAE,KAAK;YACb,aAAa,EAAE,CAAC;YAChB,OAAO,EAAE,CAAC;YACV,SAAS,EAAE,CAAC;YACZ,aAAa,EAAE,CAAC;YAChB,cAAc,EAAE,IAAI;YACpB,OAAO,EAAE,KAAK;YACd,OAAO,EAAE,IAAI;SACd,CAAC;IACJ,CAAC;IACD,OAAO,OAAO,CAAC;AACjB,CAAC"}

package/dist/lib/eval-report.d.ts

@@ -0,0 +1,12 @@
+/**
+ * eval-report — render matrix results as Markdown or CSV.
+ *
+ * The markdown form is what's posted to the tracking epic by the nightly
+ * workflow. The CSV form is what downstream spreadsheets / Grafana /
+ * analytic scripts consume.
+ */
+import type { MatrixResult } from './eval-matrix.js';
+/** Render a matrix result as Markdown suitable for a GitHub comment. */
+export declare function renderMatrixMarkdown(result: MatrixResult, title?: string): string;
+/** Render a matrix result as CSV. One row per (case, profile). */
+export declare function renderMatrixCsv(result: MatrixResult): string;

package/dist/lib/eval-report.js

@@ -0,0 +1,132 @@
+/** Render a matrix result as Markdown suitable for a GitHub comment. */
+export function renderMatrixMarkdown(result, title) {
+    const lines = [];
+    const heading = title ?? `# Routing regression — ${new Date().toISOString().slice(0, 10)}`;
+    lines.push(heading);
+    lines.push('');
+    lines.push(`Baseline: \`${result.baseline}\` · ${result.cases.length} case(s) · ${result.profiles.length} profile(s)`);
+    lines.push('');
+    if (result.dryRun) {
+        lines.push('> **Dry-run** — no pipelines executed. This report validates profile loading and case structure only. Pass `--execute` to run for real (requires fixture isolation; see CASE_FORMAT.md).');
+        lines.push('');
+    }
+    // Per-profile summary table. In dry-run every numeric column is zero by
+    // construction (no pipelines ran), so we show a validation table instead
+    // to avoid misreading "0/N" as real regressions.
+    lines.push('## Per-profile summary');
+    lines.push('');
+    if (result.dryRun) {
+        lines.push('| Profile | Cases loaded | Profile applied | Executed |');
+        lines.push('| --- | --- | --- | --- |');
+        for (const t of result.totals) {
+            lines.push(`| \`${t.profile}\` | ${t.caseCount} | yes | no (dry-run) |`);
+        }
+    }
+    else {
+        lines.push('| Profile | Pass | Pass rate | Total cost | Mean wall time | Mean tool-error rate |');
+        lines.push('| --- | --- | --- | --- | --- | --- |');
+        for (const t of result.totals) {
+            lines.push(`| \`${t.profile}\` | ${t.passCount}/${t.caseCount} | ${percent(t.passRate)} | ${usd(t.totalCostUsd)} | ${seconds(t.meanWallTimeS)} | ${percent(t.meanToolErrorRate)} |`);
+        }
+    }
+    lines.push('');
+    // Per-case grid
+    lines.push('## Per-case results');
+    lines.push('');
+    const header = ['Case', ...result.profiles];
+    lines.push(`| ${header.join(' | ')} |`);
+    lines.push(`| ${header.map(() => '---').join(' | ')} |`);
+    for (const c of result.cases) {
+        const cells = [c.caseId];
+        for (const profile of result.profiles) {
+            const entry = c.perProfile[profile];
+            cells.push(formatCaseCell(entry));
+        }
+        lines.push(`| ${cells.join(' | ')} |`);
+    }
+    lines.push('');
+    // Deltas vs baseline — only meaningful when pipelines actually ran.
+    if (!result.dryRun) {
+        lines.push(`## Deltas vs \`${result.baseline}\``);
+        lines.push('');
+        lines.push('| Profile | Δ pipeline_success | Δ cost_per_issue |');
+        lines.push('| --- | --- | --- |');
+        for (const t of result.totals) {
+            const d = result.deltas[t.profile];
+            if (!d)
+                continue;
+            lines.push(`| \`${t.profile}\` | ${deltaPct(d.pipelineSuccessDelta)} | ${deltaUsd(d.costPerIssueDelta)} |`);
+        }
+        lines.push('');
+    }
+    return lines.join('\n');
+}
+/** Render a matrix result as CSV. One row per (case, profile). */
+export function renderMatrixCsv(result) {
+    const header = [
+        'case_id',
+        'profile',
+        'passed',
+        'partial_credit',
+        'cost_usd',
+        'wall_time_s',
+        'tool_error_rate',
+        'diff_similarity',
+        'errored',
+    ];
+    const rows = [header.join(',')];
+    for (const c of result.cases) {
+        for (const profile of result.profiles) {
+            const entry = c.perProfile[profile];
+            rows.push([
+                csvField(c.caseId),
+                csvField(profile),
+                entry.passed ? '1' : '0',
+                entry.partialCredit.toFixed(3),
+                entry.costUsd.toFixed(4),
+                entry.wallTimeS.toFixed(1),
+                entry.toolErrorRate.toFixed(3),
+                entry.diffSimilarity === null ? '' : entry.diffSimilarity.toFixed(3),
+                entry.errored ? '1' : '0',
+            ].join(','));
+        }
+    }
+    return rows.join('\n') + '\n';
+}
+/** Format a single per-case cell in the markdown grid. */
+function formatCaseCell(entry) {
+    if (!entry)
+        return '—';
+    if (entry.skipped)
+        return 'SKIP';
+    if (entry.errored)
+        return 'ERR';
+    const tag = entry.passed ? 'PASS' : 'FAIL';
+    return `${tag} (${usd(entry.costUsd)})`;
+}
+function percent(value) {
+    return `${(value * 100).toFixed(1)}%`;
+}
+function usd(value) {
+    return `$${value.toFixed(2)}`;
+}
+function seconds(value) {
+    return `${value.toFixed(0)}s`;
+}
+function deltaPct(value) {
+    const sign = value > 0 ? '+' : '';
+    return `${sign}${(value * 100).toFixed(1)} pp`;
+}
+function deltaUsd(value) {
+    if (value >= 0)
+        return `+$${value.toFixed(2)}`;
+    return `-$${Math.abs(value).toFixed(2)}`;
+}
+/** Escape a CSV field if it contains commas, quotes, or newlines. */
+function csvField(value) {
+    if (/[",\n\r]/.test(value)) {
+        return `"${value.replace(/"/g, '""')}"`;
+    }
+    return value;
+}
+//# sourceMappingURL=eval-report.js.map

package/dist/lib/eval-report.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"eval-report.js","sourceRoot":"","sources":["../../src/lib/eval-report.ts"],"names":[],"mappings":"AASA,wEAAwE;AACxE,MAAM,UAAU,oBAAoB,CAAC,MAAoB,EAAE,KAAc;IACvE,MAAM,KAAK,GAAa,EAAE,CAAC;IAC3B,MAAM,OAAO,GAAG,KAAK,IAAI,0BAA0B,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE,CAAC,KAAK,CAAC,CAAC,EAAE,EAAE,CAAC,EAAE,CAAC;IAC3F,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC;IACpB,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,KAAK,CAAC,IAAI,CAAC,eAAe,MAAM,CAAC,QAAQ,QAAQ,MAAM,CAAC,KAAK,CAAC,MAAM,cAAc,MAAM,CAAC,QAAQ,CAAC,MAAM,aAAa,CAAC,CAAC;IACvH,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,0LAA0L,CAAC,CAAC;QACvM,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,wEAAwE;IACxE,yEAAyE;IACzE,iDAAiD;IACjD,KAAK,CAAC,IAAI,CAAC,wBAAwB,CAAC,CAAC;IACrC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;QAClB,KAAK,CAAC,IAAI,CAAC,yDAAyD,CAAC,CAAC;QACtE,KAAK,CAAC,IAAI,CAAC,2BAA2B,CAAC,CAAC;QACxC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,QAAQ,CAAC,CAAC,SAAS,yBAAyB,CAAC,CAAC;QAC3E,CAAC;IACH,CAAC;SAAM,CAAC;QACN,KAAK,CAAC,IAAI,CAAC,qFAAqF,CAAC,CAAC;QAClG,KAAK,CAAC,IAAI,CAAC,uCAAuC,CAAC,CAAC;QACpD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,KAAK,CAAC,IAAI,CACR,OAAO,CAAC,CAAC,OAAO,QAAQ,CAAC,CAAC,SAAS,IAAI,CAAC,CAAC,SAAS,MAAM,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC,YAAY,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,aAAa,CAAC,MAAM,OAAO,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CACzK,CAAC;QACJ,CAAC;IACH,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,gBAAgB;IAChB,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;IAClC,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACf,MAAM,MAAM,GAAG,CAAC,MAAM,EAAE,GAAG,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC5C,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACxC,KAAK,CAAC,IAAI,CAAC,KAAK,MAAM,CAAC,GAAG,CAAC,GAAG,EAAE,CAAC,KAAK,CAAC,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzD,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,MAAM,KAAK,GAAG,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC;QACzB,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACpC,KAAK,CAAC,IAAI,CAAC,cAAc,CAAC,KAAK,CAAC,CAAC,CAAC;QACpC,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,KAAK,KAAK,CAAC,IAAI,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;IACzC,CAAC;IACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IAEf,oEAAoE;IACpE,IAAI,CAAC,MAAM,CAAC,MAAM,EAAE,CAAC;QACnB,KAAK,CAAC,IAAI,CAAC,kBAAkB,MAAM,CAAC,QAAQ,IAAI,CAAC,CAAC;QAClD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;QACf,KAAK,CAAC,IAAI,CAAC,qDAAqD,CAAC,CAAC;QAClE,KAAK,CAAC,IAAI,CAAC,qBAAqB,CAAC,CAAC;QAClC,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,MAAM,EAAE,CAAC;YAC9B,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;YACnC,IAAI,CAAC,CAAC;gBAAE,SAAS;YACjB,KAAK,CAAC,IAAI,CAAC,OAAO,CAAC,CAAC,OAAO,QAAQ,QAAQ,CAAC,CAAC,CAAC,oBAAoB,CAAC,MAAM,QAAQ,CAAC,CAAC,CAAC,iBAAiB,CAAC,IAAI,CAAC,CAAC;QAC9G,CAAC;QACD,KAAK,CAAC,IAAI,CAAC,EAAE,CAAC,CAAC;IACjB,CAAC;IAED,OAAO,KAAK,CAAC,IAAI,CAAC,IAAI,CAAC,CAAC;AAC1B,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,eAAe,CAAC,MAAoB;IAClD,MAAM,MAAM,GAAG;QACb,SAAS;QACT,SAAS;QACT,QAAQ;QACR,gBAAgB;QAChB,UAAU;QACV,aAAa;QACb,iBAAiB;QACjB,iBAAiB;QACjB,SAAS;KACV,CAAC;IACF,MAAM,IAAI,GAAa,CAAC,MAAM,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;IAC1C,KAAK,MAAM,CAAC,IAAI,MAAM,CAAC,KAAK,EAAE,CAAC;QAC7B,KAAK,MAAM,OAAO,IAAI,MAAM,CAAC,QAAQ,EAAE,CAAC;YACtC,MAAM,KAAK,GAAG,CAAC,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;YACpC,IAAI,CAAC,IAAI,CAAC;gBACR,QAAQ,CAAC,CAAC,CAAC,MAAM,CAAC;gBAClB,QAAQ,CAAC,OAAO,CAAC;gBACjB,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;gBACxB,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9B,KAAK,CAAC,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;gBACxB,KAAK,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC1B,KAAK,CAAC,aAAa,CAAC,OAAO,CAAC,CAAC,CAAC;gBAC9B,KAAK,CAAC,cAAc,KAAK,IAAI,CAAC,CAAC,CAAC,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,cAAc,CAAC,OAAO,CAAC,CAAC,CAAC;gBACpE,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,GAAG;aAC1B,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;QACf,CAAC;IACH,CAAC;IACD,OAAO,IAAI,CAAC,IAAI,CAAC,IAAI,CAAC,GAAG,IAAI,CAAC;AAChC,CAAC;AAED,0DAA0D;AAC1D,SAAS,cAAc,CAAC,KAA4F;IAClH,IAAI,CAAC,KAAK;QAAE,OAAO,GAAG,CAAC;IACvB,IAAI,KAAK,CAAC,OAAO;QAAE,OAAO,MAAM,CAAC;IACjC,IAAI,KAAK,CAAC,OAAO;QAAE,OAAO,KAAK,CAAC;IAChC,MAAM,GAAG,GAAG,KAAK,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,MAAM,CAAC;IAC3C,OAAO,GAAG,GAAG,KAAK,GAAG,CAAC,KAAK,CAAC,OAAO,CAAC,GAAG,CAAC;AAC1C,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AACxC,CAAC;AAED,SAAS,GAAG,CAAC,KAAa;IACxB,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAChC,CAAC;AAED,SAAS,OAAO,CAAC,KAAa;IAC5B,OAAO,GAAG,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,GAAG,CAAC;AAChC,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,MAAM,IAAI,GAAG,KAAK,GAAG,CAAC,CAAC,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,CAAC;IAClC,OAAO,GAAG,IAAI,GAAG,CAAC,KAAK,GAAG,GAAG,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,KAAK,CAAC;AACjD,CAAC;AAED,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,KAAK,IAAI,CAAC;QAAE,OAAO,KAAK,KAAK,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/C,OAAO,KAAK,IAAI,CAAC,GAAG,CAAC,KAAK,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,EAAE,CAAC;AAC3C,CAAC;AAED,qEAAqE;AACrE,SAAS,QAAQ,CAAC,KAAa;IAC7B,IAAI,UAAU,CAAC,IAAI,CAAC,KAAK,CAAC,EAAE,CAAC;QAC3B,OAAO,IAAI,KAAK,CAAC,OAAO,CAAC,IAAI,EAAE,IAAI,CAAC,GAAG,CAAC;IAC1C,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC"}

package/dist/lib/eval-secret-scan.d.ts

@@ -0,0 +1,41 @@
+export type SecretRule = {
+    /** Short id for the rule (used in messages and tests). */
+    id: string;
+    /** Human-readable description of what this detects. */
+    description: string;
+    /** Regex to match against file contents. */
+    pattern: RegExp;
+};
+/**
+ * Built-in secret patterns. Conservative — we'd rather false-positive on
+ * an obvious looking token than let a real one slip through.
+ */
+export declare const SECRET_RULES: SecretRule[];
+export type SecretFinding = {
+    ruleId: string;
+    description: string;
+    match: string;
+    line: number;
+};
+export type SecretScanResult = {
+    /** Path of the file that was scanned. */
+    path: string;
+    /** Non-empty when the file contains a secret. */
+    findings: SecretFinding[];
+};
+/**
+ * Scan a single string of content against the rule set.
+ * Returned matches are truncated to 40 characters so secrets don't leak
+ * into CI logs when the scan itself reports failures.
+ */
+export declare function scanContent(content: string): SecretFinding[];
+/**
+ * Walk a case directory and scan each text file. Returns per-file results.
+ * Directories that don't exist return an empty array.
+ */
+export declare function scanCaseDir(dirPath: string): SecretScanResult[];
+/**
+ * Format findings as a single human-readable block. Used by the build
+ * script and CLI to print a dirty-case report.
+ */
+export declare function formatFindings(results: SecretScanResult[]): string;