@stupify/cli 0.0.3 → 0.0.4

package/dist/search-bench.js

@@ -0,0 +1,675 @@
+import { execFile } from "node:child_process";
+import { mkdir, mkdtemp, readFile, readdir, rm, writeFile } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import path from "node:path";
+import { promisify } from "node:util";
+const execFileAsync = promisify(execFile);
+export async function runSearchBench(configPath) {
+    const startedAt = new Date();
+    const configFile = path.resolve(configPath);
+    const configDir = path.dirname(configFile);
+    const config = JSON.parse(await readFile(configFile, "utf8"));
+    const outputDir = path.resolve("experiments/results", `${safeSegment(config.name)}-${startedAt.toISOString().replace(/[:.]/g, "-")}`);
+    const profilesDir = path.join(outputDir, "profiles");
+    const runsDir = path.join(outputDir, "runs");
+    const replayDir = path.join(outputDir, "real-replay");
+    await mkdir(profilesDir, { recursive: true });
+    await mkdir(runsDir, { recursive: true });
+    await mkdir(replayDir, { recursive: true });
+    const profilePaths = await resolveProfilePaths(config.profiles, configDir);
+    const profiles = await Promise.all(profilePaths.map(readProfile));
+    await Promise.all(profiles.map(({ profile, filePath }) => writeFile(path.join(profilesDir, `${safeSegment(profile.id)}.json`), JSON.stringify({ source: filePath, ...profile }, null, 2))));
+    const fixturePaths = await resolveGlob(config.fixtures, configDir);
+    const fixtures = await Promise.all(fixturePaths.map(readFixture));
+    const allRuns = [];
+    const replayRuns = [];
+    for (const { profile, filePath: profilePath } of profiles) {
+        for (const { fixture } of fixtures) {
+            const run = await runFixture(profile.id, profilePath, fixture);
+            allRuns.push(run);
+            await writeRunFiles(runsDir, `${fixture.id}__${profile.id}`, run, fixture.description);
+        }
+        for (const smoke of config.realSmokeRuns ?? []) {
+            const run = await runSmoke(profile.id, profilePath, smoke);
+            allRuns.push(run);
+            await writeRunFiles(runsDir, `${smoke.id}__${profile.id}`, run, "Real repo smoke run");
+        }
+    }
+    for (const replay of config.realCommitReplay ?? []) {
+        const runs = await runCommitReplay(replay, profiles, replayDir);
+        replayRuns.push(...runs);
+    }
+    const leaderboard = summarize(profiles.map(({ profile }) => profile), fixtures.map(({ fixture }) => fixture), allRuns);
+    const perCheck = summarizeByCheck(allRuns);
+    const summary = {
+        name: config.name,
+        outputDir,
+        generatedAt: startedAt.toISOString(),
+        runs: allRuns,
+        realReplayRuns: replayRuns,
+        leaderboard,
+        perCheck,
+    };
+    await writeFile(path.join(outputDir, "summary.json"), JSON.stringify(summary, null, 2));
+    const leaderboardText = renderLeaderboard(leaderboard, perCheck);
+    await writeFile(path.join(outputDir, "leaderboard.md"), leaderboardText);
+    await writeFile(path.join(outputDir, "real-replay-summary.json"), JSON.stringify(replayRuns, null, 2));
+    await writeFile(path.join(outputDir, "real-replay.md"), renderReplayMarkdown(replayRuns));
+    await writeFile(path.join(outputDir, "real-replay-review.md"), renderReplayReviewMarkdown(replayRuns));
+    return `Search bench complete.
+Results: ${outputDir}
+
+${leaderboardText}`;
+}
+async function runFixture(profileId, profilePath, fixture) {
+    const tempDir = await mkdtemp(path.join(tmpdir(), "stupify-search-fixture-"));
+    try {
+        await execFileAsync("git", ["init", "-q"], { cwd: tempDir });
+        const patchPath = path.join(tempDir, "fixture.patch");
+        await writeFile(patchPath, fixture.stagedPatch);
+        await execFileAsync("git", ["apply", "--recount", "--whitespace=nowarn", patchPath], { cwd: tempDir, maxBuffer: 32 * 1024 * 1024 });
+        await rm(patchPath, { force: true });
+        await execFileAsync("git", ["add", "-A"], { cwd: tempDir });
+        const result = await runCli(tempDir, ["--staged", "--json", "--search-profile", profilePath]);
+        const run = resultToBenchRun(profileId, result, { fixtureId: fixture.id, expected: fixture.expected });
+        return {
+            ...run,
+            score: scoreFixtureRun(run, fixture.expected),
+        };
+    }
+    catch (error) {
+        return errorRun(profileId, { fixtureId: fixture.id, expected: fixture.expected }, error);
+    }
+    finally {
+        await rm(tempDir, { recursive: true, force: true });
+    }
+}
+async function runSmoke(profileId, profilePath, smoke) {
+    const cwd = resolveSmokeCwd(smoke.cwd);
+    if (!cwd) {
+        return {
+            profileId,
+            smokeId: smoke.id,
+            elapsedMs: 0,
+            modelCalls: 0,
+            patterns: [],
+            targets: 0,
+            targetsByPattern: {},
+            inputTokens: 0,
+            skipped: true,
+            skipReason: "missing_cwd",
+            matches: [],
+            targetsPreview: [],
+            matchesUsingCounterReasonAsProof: 0,
+            score: -5,
+            error: "Smoke cwd is not configured. Set BEVYL_REPO or provide cwd.",
+        };
+    }
+    try {
+        const result = await runCli(cwd, [...smoke.args, "--json", "--search-profile", profilePath]);
+        const run = resultToBenchRun(profileId, result, { smokeId: smoke.id });
+        return {
+            ...run,
+            score: scoreSmokeRun(run),
+        };
+    }
+    catch (error) {
+        return errorRun(profileId, { smokeId: smoke.id }, error);
+    }
+}
+async function runCommitReplay(replay, profiles, replayDir) {
+    const cwd = resolveReplayCwd(replay);
+    if (!cwd) {
+        return replay.profiles.map((profileId) => replayErrorRun(replay.id, profileId, { sha: "", shortSha: "(none)" }, new Error(`Replay cwd is not configured. Set ${replay.repoEnv ?? "repo env"} or provide cwd.`)));
+    }
+    const commits = await replayCommits(cwd, replay);
+    const profilesById = new Map(profiles.map((profile) => [profile.profile.id, profile]));
+    const runs = [];
+    for (const commit of commits) {
+        for (const profileId of replay.profiles) {
+            const profile = profilesById.get(profileId);
+            const run = profile
+                ? await runReplayCommit(cwd, replay.id, commit, profile.profile.id, profile.filePath)
+                : replayErrorRun(replay.id, profileId, commit, new Error(`Unknown replay profile: ${profileId}`));
+            runs.push(run);
+            await writeFile(path.join(replayDir, `${safeSegment(replay.id)}__${safeSegment(commit.shortSha)}__${safeSegment(profileId)}.json`), JSON.stringify(run, null, 2));
+        }
+    }
+    return runs;
+}
+async function replayCommits(cwd, replay) {
+    const args = ["log", "--format=%H", `-${replay.limit}`];
+    if (replay.nonMerge)
+        args.push("--no-merges");
+    if (replay.since)
+        args.push(`--since=${replay.since}`);
+    const { stdout } = await execFileAsync("git", args, { cwd, maxBuffer: 32 * 1024 * 1024 });
+    return stdout
+        .split(/\r?\n/)
+        .map((sha) => sha.trim())
+        .filter(Boolean)
+        .map((sha) => ({ sha, shortSha: sha.slice(0, 7) }));
+}
+async function runReplayCommit(repoCwd, replayId, commit, profileId, profilePath) {
+    const tempDir = await mkdtemp(path.join(tmpdir(), "stupify-replay-"));
+    let worktreeAdded = false;
+    try {
+        const parent = `${commit.sha}^`;
+        const stats = await commitStats(repoCwd, parent, commit.sha);
+        await execFileAsync("git", ["worktree", "add", "--detach", tempDir, parent], { cwd: repoCwd, maxBuffer: 64 * 1024 * 1024 });
+        worktreeAdded = true;
+        const { stdout: patch } = await execFileAsync("git", ["diff", "--binary", parent, commit.sha], { cwd: repoCwd, maxBuffer: 128 * 1024 * 1024 });
+        const patchPath = path.join(tempDir, "commit.patch");
+        await writeFile(patchPath, patch);
+        await execFileAsync("git", ["apply", "--cached", "--whitespace=nowarn", patchPath], { cwd: tempDir, maxBuffer: 128 * 1024 * 1024 });
+        await rm(patchPath, { force: true });
+        const result = await runCli(tempDir, ["--staged", "--json", "--search-profile", profilePath]);
+        return replayResult(replayId, profileId, commit, result, stats);
+    }
+    catch (error) {
+        return replayErrorRun(replayId, profileId, commit, error);
+    }
+    finally {
+        if (worktreeAdded) {
+            await execFileAsync("git", ["worktree", "remove", "--force", tempDir], { cwd: repoCwd, maxBuffer: 64 * 1024 * 1024 }).catch(async () => {
+                await rm(tempDir, { recursive: true, force: true });
+                await execFileAsync("git", ["worktree", "prune"], { cwd: repoCwd }).catch(() => undefined);
+            });
+        }
+        else {
+            await rm(tempDir, { recursive: true, force: true });
+        }
+    }
+}
+async function commitStats(cwd, parent, target) {
+    const { stdout } = await execFileAsync("git", ["diff", "--numstat", parent, target], { cwd, maxBuffer: 32 * 1024 * 1024 });
+    let changedFiles = 0;
+    let addedLines = 0;
+    let deletedLines = 0;
+    for (const line of stdout.split(/\r?\n/).filter(Boolean)) {
+        const [added, deleted] = line.split(/\s+/);
+        changedFiles += 1;
+        addedLines += numericStat(added);
+        deletedLines += numericStat(deleted);
+    }
+    return { changedFiles, addedLines, deletedLines };
+}
+function replayResult(replayId, profileId, commit, result, stats) {
+    return {
+        replayId,
+        profileId,
+        commitId: commit.shortSha,
+        outcome: replayOutcome(result),
+        changedFiles: stats.changedFiles,
+        addedLines: stats.addedLines,
+        deletedLines: stats.deletedLines,
+        elapsedMs: result.stats.elapsedMs,
+        skipped: result.stats.skipped ?? false,
+        skipReason: result.stats.skipReason,
+        targets: result.stats.searchTargets ?? result.stats.candidates ?? 0,
+        inputTokens: result.stats.inputTokens ?? 0,
+        repomixPackedTokens: result.stats.repomixTokens,
+        modelCalls: result.stats.modelCalls,
+        matches: result.matches,
+        matchesByPattern: countMatches(result.matches),
+    };
+}
+function replayOutcome(result) {
+    if (result.stats.skipReason === "input_too_large")
+        return "skipped_input_too_large";
+    if (result.stats.skipReason === "no_candidates")
+        return "no_candidates";
+    if (result.matches.length > 0)
+        return "ran_with_matches";
+    return "ran_no_matches";
+}
+function replayErrorRun(replayId, profileId, commit, error) {
+    return {
+        replayId,
+        profileId,
+        commitId: commit.shortSha,
+        outcome: "error",
+        changedFiles: 0,
+        addedLines: 0,
+        deletedLines: 0,
+        elapsedMs: 0,
+        skipped: true,
+        skipReason: "error",
+        targets: 0,
+        inputTokens: 0,
+        modelCalls: 0,
+        matches: [],
+        matchesByPattern: {},
+        error: error instanceof Error ? error.message : String(error),
+    };
+}
+async function runCli(cwd, args) {
+    const startedAt = Date.now();
+    const cliPath = process.argv[1];
+    const { stdout } = await execFileAsync(process.execPath, [cliPath, ...args], {
+        cwd,
+        env: process.env,
+        maxBuffer: 128 * 1024 * 1024,
+    });
+    const parsed = JSON.parse(stdout);
+    return {
+        ...parsed,
+        stats: {
+            ...parsed.stats,
+            elapsedMs: parsed.stats.elapsedMs || Date.now() - startedAt,
+        },
+    };
+}
+function resultToBenchRun(profileId, result, identity) {
+    return {
+        profileId,
+        fixtureId: identity.fixtureId,
+        smokeId: identity.smokeId,
+        elapsedMs: result.stats.elapsedMs,
+        modelCalls: result.stats.modelCalls,
+        patterns: result.patterns,
+        targets: result.stats.searchTargets ?? result.stats.candidates ?? 0,
+        targetsByPattern: result.stats.targetsByPattern ?? {},
+        inputTokens: result.stats.inputTokens ?? 0,
+        repomixPackedTokens: result.stats.repomixTokens,
+        skipped: result.stats.skipped ?? false,
+        skipReason: result.stats.skipReason,
+        matches: result.matches,
+        expected: identity.expected,
+        targetsPreview: result.stats.targetsPreview ?? [],
+        matchesUsingCounterReasonAsProof: countCounterReasonProofs(result.matches),
+    };
+}
+function errorRun(profileId, identity, error) {
+    return {
+        profileId,
+        fixtureId: identity.fixtureId,
+        smokeId: identity.smokeId,
+        elapsedMs: 0,
+        modelCalls: 0,
+        patterns: [],
+        targets: 0,
+        targetsByPattern: {},
+        inputTokens: 0,
+        skipped: true,
+        skipReason: "error",
+        matches: [],
+        expected: identity.expected,
+        targetsPreview: [],
+        matchesUsingCounterReasonAsProof: 0,
+        score: identity.fixtureId ? -3 : -5,
+        error: error instanceof Error ? error.message : String(error),
+    };
+}
+function scoreFixtureRun(run, expected) {
+    const activePatterns = new Set(run.patterns.map((pattern) => pattern));
+    const activeExpected = expected.filter((item) => activePatterns.has(item.patternId));
+    let score = run.skipped && activeExpected.some((item) => item.shouldMatch) ? -3 : 0;
+    const matchCounts = countMatches(run.matches);
+    const expectedPatterns = new Set(activeExpected.map((item) => item.patternId));
+    for (const item of activeExpected) {
+        const matched = (matchCounts[item.patternId] ?? 0) > 0;
+        if (item.shouldMatch && matched)
+            score += 5;
+        if (item.shouldMatch && !matched)
+            score -= 4;
+        if (!item.shouldMatch && !matched)
+            score += 2;
+        if (!item.shouldMatch && matched)
+            score -= 10;
+    }
+    for (const match of run.matches) {
+        const id = match.patternId;
+        if (!expectedPatterns.has(id))
+            score -= 6;
+    }
+    score -= (run.elapsedMs / 1000) * 0.05;
+    score -= (run.inputTokens / 1000) * 0.001;
+    return round(score);
+}
+function scoreSmokeRun(run) {
+    let score = 0;
+    if (run.skipped)
+        score -= 5;
+    if (run.matches.length > 3)
+        score -= 3;
+    if (run.elapsedMs > 60_000)
+        score -= 5;
+    if (run.inputTokens > 12_000 && run.skipped)
+        score -= 5;
+    score -= (run.elapsedMs / 1000) * 0.05;
+    score -= (run.inputTokens / 1000) * 0.001;
+    return round(score);
+}
+function summarize(profiles, fixtures, runs) {
+    const rows = profiles.map((profile) => {
+        const fixtureRuns = runs.filter((run) => run.profileId === profile.id && run.fixtureId);
+        const smokeRuns = runs.filter((run) => run.profileId === profile.id && run.smokeId);
+        const counts = fixtureRuns.reduce((acc, run) => addFixtureCounts(acc, run), emptyCounts());
+        const positiveFixtureCount = fixtureRuns
+            .flatMap((run) => (run.expected ?? []).filter((item) => run.patterns.some((pattern) => pattern === item.patternId)))
+            .filter((expected) => expected.shouldMatch).length;
+        const avgMs = fixtureRuns.length === 0
+            ? 0
+            : fixtureRuns.reduce((sum, run) => sum + run.elapsedMs, 0) / fixtureRuns.length;
+        const decision = decisionForProfile(counts, positiveFixtureCount, smokeRuns);
+        return {
+            profileId: profile.id,
+            fixtureScore: round(fixtureRuns.reduce((sum, run) => sum + (run.score ?? 0), 0)),
+            falsePositives: counts.fp,
+            falseNegatives: counts.fn,
+            truePositives: counts.tp,
+            trueNegatives: counts.tn,
+            wrongPatterns: counts.wp,
+            assignedCheckFalsePositives: counts.assignedFp,
+            avgMs: Math.round(avgMs),
+            smokeMatches: smokeRuns.reduce((sum, run) => sum + run.matches.length, 0),
+            smokeSkipped: smokeRuns.filter((run) => run.skipped).length,
+            matchesUsingCounterReasonAsProof: fixtureRuns.reduce((sum, run) => sum + run.matchesUsingCounterReasonAsProof, 0),
+            decision,
+        };
+    });
+    return rows.sort((a, b) => b.fixtureScore - a.fixtureScore);
+}
+function summarizeByCheck(runs) {
+    const counts = new Map();
+    for (const run of runs.filter((item) => item.fixtureId)) {
+        const expected = run.expected ?? [];
+        const activePatterns = new Set(run.patterns.map((pattern) => pattern));
+        const activeExpected = expected.filter((item) => activePatterns.has(item.patternId));
+        for (const item of activeExpected) {
+            const current = counts.get(item.patternId) ?? emptyCounts();
+            const matched = run.matches.some((match) => match.patternId === item.patternId);
+            if (item.shouldMatch && matched)
+                current.tp += 1;
+            if (item.shouldMatch && !matched)
+                current.fn += 1;
+            if (!item.shouldMatch && matched) {
+                current.fp += 1;
+                current.assignedFp += 1;
+            }
+            if (!item.shouldMatch && !matched)
+                current.tn += 1;
+            counts.set(item.patternId, current);
+        }
+        const expectedPatterns = new Set(activeExpected.map((item) => item.patternId));
+        for (const match of run.matches) {
+            const id = match.patternId;
+            if (expectedPatterns.has(id))
+                continue;
+            const current = counts.get(id) ?? emptyCounts();
+            current.fp += 1;
+            current.wp += 1;
+            counts.set(id, current);
+        }
+    }
+    return [...counts.entries()]
+        .map(([checkId, count]) => ({
+        checkId,
+        truePositives: count.tp,
+        falsePositives: count.fp,
+        falseNegatives: count.fn,
+        wrongPatterns: count.wp,
+        assignedCheckFalsePositives: count.assignedFp,
+        decision: checkDecision(count),
+    }))
+        .sort((a, b) => a.checkId.localeCompare(b.checkId));
+}
+function addFixtureCounts(counts, run) {
+    const expected = run.expected ?? [];
+    const activePatterns = new Set(run.patterns.map((pattern) => pattern));
+    const activeExpected = expected.filter((item) => activePatterns.has(item.patternId));
+    const matchCounts = countMatches(run.matches);
+    const expectedPatterns = new Set(activeExpected.map((item) => item.patternId));
+    for (const item of activeExpected) {
+        const matched = (matchCounts[item.patternId] ?? 0) > 0;
+        if (item.shouldMatch && matched)
+            counts.tp += 1;
+        if (item.shouldMatch && !matched)
+            counts.fn += 1;
+        if (!item.shouldMatch && !matched)
+            counts.tn += 1;
+        if (!item.shouldMatch && matched) {
+            counts.fp += 1;
+            counts.assignedFp += 1;
+        }
+    }
+    for (const match of run.matches) {
+        const id = match.patternId;
+        if (!expectedPatterns.has(id)) {
+            if (!expectedPatterns.has(id))
+                counts.fp += 1;
+            counts.wp += 1;
+        }
+    }
+    return counts;
+}
+function emptyCounts() {
+    return { tp: 0, tn: 0, fp: 0, fn: 0, wp: 0, assignedFp: 0 };
+}
+function decisionForProfile(counts, positiveFixtureCount, smokeRuns) {
+    if (counts.fp > 0)
+        return "reject: false positives";
+    if (counts.wp > 0)
+        return "reject: wrong pattern";
+    if (counts.tp < Math.ceil(positiveFixtureCount * 0.6))
+        return "reject: low recall";
+    if (smokeRuns.some((run) => run.matches.length > 3))
+        return "reject: noisy smoke";
+    if (smokeRuns.some((run) => run.elapsedMs > 60_000))
+        return "reject: slow smoke";
+    if (smokeRuns.some((run) => run.skipped))
+        return "fixture candidate";
+    return "candidate hook default";
+}
+function checkDecision(counts) {
+    if (counts.fp > 0)
+        return "not search-safe";
+    if (counts.tp === 0 && counts.fn > 0)
+        return "blind";
+    if (counts.fn > counts.tp)
+        return "low recall";
+    return "candidate";
+}
+function countMatches(matches) {
+    const counts = {};
+    for (const match of matches)
+        counts[match.patternId] = (counts[match.patternId] ?? 0) + 1;
+    return counts;
+}
+function countCounterReasonProofs(matches) {
+    return matches.filter((match) => /counter_reason/i.test(match.proof)).length;
+}
+function emptyReplayOutcomeCounts() {
+    return {
+        runs: 0,
+        no_candidates: 0,
+        ran_no_matches: 0,
+        ran_with_matches: 0,
+        skipped_input_too_large: 0,
+        error: 0,
+        matches: 0,
+        modelCalls: 0,
+        targets: 0,
+    };
+}
+async function writeRunFiles(runsDir, id, run, description) {
+    const safeId = safeSegment(id);
+    await writeFile(path.join(runsDir, `${safeId}.json`), JSON.stringify(run, null, 2));
+    await writeFile(path.join(runsDir, `${safeId}.md`), renderRunMarkdown(run, description));
+}
+function renderRunMarkdown(run, description) {
+    return `# ${run.fixtureId ?? run.smokeId}
+
+Profile: ${run.profileId}
+Description: ${description}
+Runtime: ${run.elapsedMs}ms
+Targets: ${run.targets}
+Model calls: ${run.modelCalls}
+Input tokens: ${run.inputTokens}
+Counter-reason proofs: ${run.matchesUsingCounterReasonAsProof}
+Skipped: ${run.skipped ? `${run.skipReason ?? "yes"}` : "no"}
+Score: ${run.score ?? "n/a"}
+
+## Matches
+${run.matches.length === 0 ? "(none)" : run.matches.map((match, index) => `${index + 1}. ${match.patternId} (${match.targetId})
+   reason: ${match.reason}
+   proof: ${match.proof}`).join("\n")}
+
+## Expected
+${(run.expected ?? []).length === 0 ? "(none)" : (run.expected ?? []).map((expected) => `- ${expected.patternId}: ${expected.shouldMatch ? "match" : "no match"}`).join("\n")}
+
+## Targets
+${run.targetsPreview.length === 0 ? "(none)" : run.targetsPreview.map((target) => `- ${target.targetId}: ${target.patternId} ${target.entityKind ?? ""} ${target.sourceKind ?? ""}`.trim()).join("\n")}
+
+${run.error ? `## Error\n${run.error}\n` : ""}`;
+}
+function renderLeaderboard(rows, perCheck) {
+    const table = rows.map((row, index) => `| ${index + 1} | ${row.profileId} | ${row.fixtureScore} | ${row.falsePositives} | ${row.wrongPatterns} | ${row.assignedCheckFalsePositives} | ${row.falseNegatives} | ${row.truePositives} | ${row.matchesUsingCounterReasonAsProof} | ${row.avgMs} | ${row.smokeMatches} | ${row.smokeSkipped} | ${row.decision} |`).join("\n");
+    const checkTable = perCheck.map((row) => `| ${row.checkId} | ${row.truePositives} | ${row.falsePositives} | ${row.wrongPatterns} | ${row.assignedCheckFalsePositives} | ${row.falseNegatives} | ${row.decision} |`).join("\n");
+    return `# Search Bench Leaderboard
+
+| rank | profile | fixture score | FP | wrong FP | assigned FP | FN | TP | counter-proof | avg ms | smoke matches | smoke skipped | decision |
+|---|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|---:|---|
+${table}
+
+## Per-Check Summary
+
+| check | TP | FP | wrong FP | assigned FP | FN | decision |
+|---|---:|---:|---:|---:|---:|---|
+${checkTable}
+`;
+}
+function renderReplayMarkdown(runs) {
+    const table = runs.map((run) => {
+        const patterns = Object.entries(run.matchesByPattern)
+            .filter(([, count]) => count > 0)
+            .map(([pattern, count]) => count === 1 ? pattern : `${pattern}(${count})`)
+            .join(", ") || "-";
+        return `| ${run.profileId} | ${run.commitId} | ${run.changedFiles} | +${run.addedLines}/-${run.deletedLines} | ${run.outcome} | ${run.elapsedMs} | ${run.targets} | ${run.inputTokens} | ${run.matches.length} | ${patterns} | |`;
+    }).join("\n");
+    const outcomeTable = renderReplayOutcomeSummary(runs);
+    return `# Real Staged Replay
+
+${outcomeTable}
+
+| profile | commit | files | +/- | outcome | ms | targets | input tokens | matches | patterns | manual |
+|---|---|---:|---:|---|---:|---:|---:|---:|---|---|
+${table}
+`;
+}
+function renderReplayReviewMarkdown(runs) {
+    const matched = runs.filter((run) => run.matches.length > 0);
+    if (matched.length === 0)
+        return "# Real Replay Review\n\nNo real replay matches.\n";
+    return `# Real Replay Review
+
+${matched.flatMap((run) => run.matches.map((match) => `## ${run.profileId} / ${run.commitId}
+
+Pattern: ${match.patternId}
+Target: ${match.targetId}
+Reason: ${match.reason}
+Proof: ${match.proof}
+Manual label: [good / maybe / bad]
+Notes:
+`)).join("\n")}`;
+}
+function renderReplayOutcomeSummary(runs) {
+    const byProfile = new Map();
+    for (const run of runs) {
+        const current = byProfile.get(run.profileId) ?? emptyReplayOutcomeCounts();
+        current.runs += 1;
+        current[run.outcome] += 1;
+        current.matches += run.matches.length;
+        current.modelCalls += run.modelCalls;
+        current.targets += run.targets;
+        byProfile.set(run.profileId, current);
+    }
+    const table = [...byProfile.entries()].map(([profile, counts]) => `| ${profile} | ${counts.runs} | ${counts.no_candidates} | ${counts.ran_no_matches} | ${counts.ran_with_matches} | ${counts.skipped_input_too_large} | ${counts.error} | ${counts.matches} | ${counts.modelCalls} | ${counts.targets} |`).join("\n");
+    return `## Outcome Summary
+
+| profile | runs | no candidates | ran no matches | ran with matches | input too large | errors | matches | model calls | targets |
+|---|---:|---:|---:|---:|---:|---:|---:|---:|---:|
+${table}`;
+}
+async function resolveProfilePaths(profilePaths, configDir) {
+    return Promise.all(profilePaths.map((profilePath) => resolvePath(profilePath, configDir)));
+}
+async function resolveGlob(pattern, configDir) {
+    const resolved = await resolvePath(pattern, configDir, false);
+    if (!resolved.includes("*"))
+        return [resolved];
+    const before = resolved.slice(0, resolved.indexOf("*"));
+    const after = resolved.slice(resolved.indexOf("*") + 1);
+    const dir = before.endsWith(path.sep) ? before.slice(0, -1) : path.dirname(before);
+    const prefix = before.endsWith(path.sep) ? "" : path.basename(before);
+    const entries = await readdir(dir);
+    return entries
+        .filter((entry) => entry.startsWith(prefix) && entry.endsWith(after))
+        .map((entry) => path.join(dir, entry))
+        .sort();
+}
+async function resolvePath(input, configDir, mustExist = true) {
+    const expanded = input.startsWith("~/") ? path.join(process.env.HOME ?? "", input.slice(2)) : input;
+    const fromCwd = path.resolve(expanded);
+    const fromConfig = path.resolve(configDir, expanded);
+    if (!mustExist || await exists(fromCwd))
+        return fromCwd;
+    if (await exists(fromConfig))
+        return fromConfig;
+    return fromCwd;
+}
+async function readProfile(filePath) {
+    const profile = JSON.parse(await readFile(filePath, "utf8"));
+    if (!profile.id)
+        throw new Error(`Search profile missing id: ${filePath}`);
+    return { filePath, profile };
+}
+async function readFixture(filePath) {
+    const fixture = JSON.parse(await readFile(filePath, "utf8"));
+    if (!fixture.id)
+        throw new Error(`Search fixture missing id: ${filePath}`);
+    return { filePath, fixture };
+}
+async function exists(filePath) {
+    try {
+        await readFile(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function resolveSmokeCwd(cwd) {
+    if (!cwd)
+        return process.cwd();
+    if (cwd === "$BEVYL_REPO")
+        return process.env.BEVYL_REPO ?? null;
+    if (cwd.startsWith("$BEVYL_REPO/")) {
+        const root = process.env.BEVYL_REPO;
+        return root ? path.join(root, cwd.slice("$BEVYL_REPO/".length)) : null;
+    }
+    return cwd.startsWith("~/") ? path.join(process.env.HOME ?? "", cwd.slice(2)) : cwd;
+}
+function resolveReplayCwd(replay) {
+    if (replay.cwd)
+        return expandPath(replay.cwd);
+    if (replay.repoEnv) {
+        const value = process.env[replay.repoEnv];
+        return value ? expandPath(value) : null;
+    }
+    return process.cwd();
+}
+function expandPath(input) {
+    return input.startsWith("~/") ? path.join(process.env.HOME ?? "", input.slice(2)) : input;
+}
+function safeSegment(value) {
+    return value.replace(/[^A-Za-z0-9._-]+/g, "_").replace(/^_+|_+$/g, "") || "run";
+}
+function round(value) {
+    return Math.round(value * 1000) / 1000;
+}
+function numericStat(value) {
+    const parsed = Number(value);
+    return Number.isFinite(parsed) ? parsed : 0;
+}

package/dist/search-profile.d.ts

@@ -0,0 +1,6 @@
+import type { RepomixSearchConfig, SearchProfile, StupifyCheck } from "./types.ts";
+export declare function loadSearchProfile(profilePath: string | null): Promise<SearchProfile | null>;
+export declare function effectiveSearchChecks(explicitCheckIds: readonly string[] | null, profile: SearchProfile | null): readonly StupifyCheck[];
+export declare function effectiveMaxCandidates(defaultValue: number, profile: SearchProfile | null): number;
+export declare function effectiveMaxSearchInputTokens(defaultValue: number, profile: SearchProfile | null): number;
+export declare function effectiveRepomixConfig(defaultValue: RepomixSearchConfig, profile: SearchProfile | null): RepomixSearchConfig;