flowseeker 0.1.7

package/dist/cli/runEvaluation.js

@@ -0,0 +1,789 @@
+"use strict";
+var __createBinding = (this && this.__createBinding) || (Object.create ? (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    var desc = Object.getOwnPropertyDescriptor(m, k);
+    if (!desc || ("get" in desc ? !m.__esModule : desc.writable || desc.configurable)) {
+      desc = { enumerable: true, get: function() { return m[k]; } };
+    }
+    Object.defineProperty(o, k2, desc);
+}) : (function(o, m, k, k2) {
+    if (k2 === undefined) k2 = k;
+    o[k2] = m[k];
+}));
+var __setModuleDefault = (this && this.__setModuleDefault) || (Object.create ? (function(o, v) {
+    Object.defineProperty(o, "default", { enumerable: true, value: v });
+}) : function(o, v) {
+    o["default"] = v;
+});
+var __importStar = (this && this.__importStar) || (function () {
+    var ownKeys = function(o) {
+        ownKeys = Object.getOwnPropertyNames || function (o) {
+            var ar = [];
+            for (var k in o) if (Object.prototype.hasOwnProperty.call(o, k)) ar[ar.length] = k;
+            return ar;
+        };
+        return ownKeys(o);
+    };
+    return function (mod) {
+        if (mod && mod.__esModule) return mod;
+        var result = {};
+        if (mod != null) for (var k = ownKeys(mod), i = 0; i < k.length; i++) if (k[i] !== "default") __createBinding(result, mod, k[i]);
+        __setModuleDefault(result, mod);
+        return result;
+    };
+})();
+Object.defineProperty(exports, "__esModule", { value: true });
+const fs = __importStar(require("fs/promises"));
+const fssync = __importStar(require("fs"));
+const path = __importStar(require("path"));
+const loadConfigFromPath_1 = require("../config/loadConfigFromPath");
+const defaultConfig_1 = require("../config/defaultConfig");
+const evaluationMetrics_1 = require("../pipeline/evaluationMetrics");
+const fileGroups_1 = require("../pipeline/fileGroups");
+const runHeadless_1 = require("../pipeline/runHeadless");
+const tokenSavings_1 = require("../pipeline/tokenSavings");
+const text_1 = require("../utils/text");
+void main().catch((error) => {
+    const message = error instanceof Error ? error.stack ?? error.message : String(error);
+    console.error(message);
+    process.exitCode = 1;
+});
+async function main() {
+    const options = parseArgs(process.argv.slice(2));
+    const loaded = await loadManifest(options);
+    const manifest = loaded.manifest;
+    validateManifest(manifest, loaded.source);
+    if (options.validateOnly) {
+        const issues = await validateManifestCases(manifest);
+        if (issues.length > 0) {
+            console.error(`FlowSeeker eval validation failed: ${issues.length} issue(s).`);
+            for (const issue of issues.slice(0, 200)) {
+                console.error(`- ${issue}`);
+            }
+            if (issues.length > 200) {
+                console.error(`...and ${issues.length - 200} more issue(s).`);
+            }
+            process.exitCode = 1;
+        }
+        else {
+            console.log("FlowSeeker eval validation passed.");
+        }
+        return;
+    }
+    const results = [];
+    const startedAt = Date.now();
+    console.log(`FlowSeeker headless evaluation: ${manifest.name ?? loaded.source}`);
+    console.log(`${loaded.kind}: ${loaded.source}`);
+    for (const workspace of manifest.workspaces) {
+        const workspacePath = path.resolve(workspace.path);
+        const workspaceName = workspace.name ?? path.basename(workspacePath);
+        const config = (0, defaultConfig_1.mergeConfig)(await (0, loadConfigFromPath_1.loadConfigFromPath)(workspacePath), workspace.config ?? {});
+        const suites = filterSuitesByCaseId(await loadSuitesForWorkspace(workspacePath, workspace), options.caseId);
+        const totalCases = suites.reduce((sum, suite) => sum + suite.cases.length, 0);
+        if (totalCases === 0) {
+            continue;
+        }
+        console.log(`\nWorkspace: ${workspaceName}`);
+        console.log(`Path: ${workspacePath}`);
+        console.log(`Suites: ${suites.length}; cases: ${totalCases}`);
+        for (const suite of suites) {
+            for (const caseDef of suite.cases) {
+                const caseStartedAt = Date.now();
+                console.log(`- ${suite.name}/${caseDef.id}: ${caseDef.task}`);
+                const result = await (0, runHeadless_1.runFlowSeekerHeadless)(workspacePath, caseDef.task, config, {
+                    onProgress: options.verbose ? (message) => console.log(`  ${message}`) : undefined
+                });
+                const evaluated = evaluateCase(workspaceName, workspacePath, suite.name, caseDef, result, Date.now() - caseStartedAt);
+                results.push(evaluated);
+                console.log(`  ${formatCaseStatus(evaluated)}; ${(0, tokenSavings_1.formatTokenSavingsLine)(result.tokenSavings)}; ${formatContextCoverage(result)}; ${result.stats.scannedFiles} scanned; ${result.stats.rawEvidenceCount} raw${formatIndexConsole(result.stats)}; ${(evaluated.durationMs / 1000).toFixed(2)}s`);
+            }
+        }
+    }
+    if (options.caseId && results.length === 0) {
+        throw new Error(`No eval case matched --case-id '${options.caseId}'.`);
+    }
+    const reportPath = path.resolve(options.reportPath ?? manifest.reportPath ?? defaultReportPath());
+    await fs.mkdir(path.dirname(reportPath), { recursive: true });
+    await fs.writeFile(reportPath, renderReport(manifest, results, Date.now() - startedAt), "utf8");
+    console.log(`\nReport: ${reportPath}`);
+    console.log(formatSummary(results));
+    console.log(formatTokenSavingsSummary(results));
+    if (options.failOnMiss && results.some((result) => result.pass === false)) {
+        process.exitCode = 1;
+    }
+}
+function parseArgs(args) {
+    const options = {
+        expectedPaths: [],
+        avoidPaths: [],
+        failOnMiss: false,
+        verbose: false,
+        validateOnly: false
+    };
+    for (let index = 0; index < args.length; index += 1) {
+        const arg = args[index];
+        if (arg === "--manifest" || arg === "-m") {
+            options.manifestPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--workspace" || arg === "-w") {
+            options.workspacePath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--name") {
+            options.workspaceName = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--task" || arg === "-t") {
+            options.task = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--case-id") {
+            options.caseId = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--expected") {
+            options.expectedPaths.push(...splitList(requireValue(args, index, arg)));
+            index += 1;
+            continue;
+        }
+        if (arg === "--avoid") {
+            options.avoidPaths.push(...splitList(requireValue(args, index, arg)));
+            index += 1;
+            continue;
+        }
+        if (arg === "--pass-at-k") {
+            options.passAtK = Number(requireValue(args, index, arg));
+            if (!Number.isFinite(options.passAtK) || options.passAtK < 1) {
+                throw new Error("--pass-at-k must be a positive number.");
+            }
+            index += 1;
+            continue;
+        }
+        if (arg === "--expected-mode") {
+            const mode = requireValue(args, index, arg);
+            if (mode !== "any" && mode !== "all" && mode !== "subset") {
+                throw new Error("--expected-mode must be either 'any', 'all', or 'subset'.");
+            }
+            options.expectedMode = mode;
+            index += 1;
+            continue;
+        }
+        if (arg === "--report" || arg === "-r") {
+            options.reportPath = requireValue(args, index, arg);
+            index += 1;
+            continue;
+        }
+        if (arg === "--fail-on-miss") {
+            options.failOnMiss = true;
+            continue;
+        }
+        if (arg === "--verbose" || arg === "-v") {
+            options.verbose = true;
+            continue;
+        }
+        if (arg === "--validate-only") {
+            options.validateOnly = true;
+            continue;
+        }
+        if (arg === "--help" || arg === "-h") {
+            printUsage();
+            process.exit(0);
+        }
+        throw new Error(`Unknown argument: ${arg}`);
+    }
+    return options;
+}
+async function loadManifest(options) {
+    if (options.workspacePath) {
+        if (!options.task?.trim()) {
+            throw new Error("--workspace requires --task for ad-hoc evaluation.");
+        }
+        const workspacePath = path.resolve(options.workspacePath);
+        const caseId = options.caseId ?? slugify(options.task);
+        return {
+            source: workspacePath,
+            kind: "Ad-hoc workspace",
+            manifest: {
+                name: `Ad-hoc evaluation: ${options.workspaceName ?? path.basename(workspacePath)}`,
+                reportPath: options.reportPath,
+                workspaces: [
+                    {
+                        name: options.workspaceName ?? path.basename(workspacePath),
+                        path: workspacePath,
+                        discoverSuites: false,
+                        suiteFiles: [],
+                        cases: [
+                            {
+                                id: caseId,
+                                task: options.task.trim(),
+                                expectedPaths: options.expectedPaths.length > 0 ? options.expectedPaths : undefined,
+                                avoidPaths: options.avoidPaths.length > 0 ? options.avoidPaths : undefined,
+                                passAtK: options.passAtK,
+                                expectedMode: options.expectedMode
+                            }
+                        ]
+                    }
+                ]
+            }
+        };
+    }
+    const manifestPath = path.resolve(options.manifestPath ?? path.join(".flowseeker", "eval-workspaces.json"));
+    return {
+        source: manifestPath,
+        kind: "Manifest",
+        manifest: await readJson(manifestPath)
+    };
+}
+function requireValue(args, index, flag) {
+    const value = args[index + 1];
+    if (!value || value.startsWith("-")) {
+        throw new Error(`${flag} requires a value.`);
+    }
+    return cleanArgValue(value);
+}
+function cleanArgValue(value) {
+    return value.replace(/\^/g, "");
+}
+function splitList(value) {
+    return value
+        .split(",")
+        .map((item) => item.trim())
+        .filter(Boolean);
+}
+function slugify(value) {
+    const slug = (0, text_1.normalizeText)(value)
+        .replace(/[^a-z0-9]+/g, "-")
+        .replace(/^-+|-+$/g, "")
+        .slice(0, 64);
+    return slug || "ad-hoc-task";
+}
+function printUsage() {
+    console.log([
+        "Usage:",
+        "  node dist/cli/runEvaluation.js --manifest .flowseeker/eval-workspaces.json",
+        "  node dist/cli/runEvaluation.js --workspace D:/repo --task \"Fix timestamp based on browser timezone\"",
+        "",
+        "Options:",
+        "  --manifest, -m      Path to a multi-workspace eval manifest.",
+        "  --workspace, -w     Run one ad-hoc task against this workspace path without a manifest.",
+        "  --task, -t          Natural-language task for ad-hoc workspace mode.",
+        "  --name             Optional workspace name for ad-hoc mode.",
+        "  --case-id          Optional case id for ad-hoc mode, or filter manifest cases by id.",
+        "  --expected         Optional expected path(s), comma-separated or repeated.",
+        "  --avoid            Optional avoid path(s), comma-separated or repeated.",
+        "  --pass-at-k        Optional pass threshold for expected paths. Default: 10.",
+        "  --expected-mode    Expected path pass mode: any, subset, or all. Default: any.",
+        "  --report, -r        Override report output path.",
+        "  --fail-on-miss      Exit with code 1 if a labeled case misses passAtK.",
+        "  --verbose, -v       Print scan progress.",
+        "  --validate-only     Validate eval labels and schema without scanning.",
+        ""
+    ].join("\n"));
+}
+async function loadSuitesForWorkspace(workspacePath, workspace) {
+    const suites = [];
+    if (workspace.cases?.length) {
+        suites.push({
+            name: `${workspace.name ?? path.basename(workspacePath)} direct cases`,
+            cases: workspace.cases
+        });
+    }
+    const shouldDiscoverSuites = workspace.discoverSuites !== false;
+    const suiteFiles = workspace.suiteFiles?.length ? workspace.suiteFiles.map((file) => path.resolve(workspacePath, file)) : shouldDiscoverSuites ? await discoverSuiteFiles(workspacePath) : [];
+    for (const suiteFile of suiteFiles) {
+        suites.push(await readSuite(suiteFile));
+    }
+    if (suites.length === 0) {
+        throw new Error(`No eval suites found for workspace: ${workspacePath}`);
+    }
+    return suites;
+}
+function filterSuitesByCaseId(suites, caseId) {
+    if (!caseId) {
+        return suites;
+    }
+    return suites
+        .map((suite) => ({
+        ...suite,
+        cases: suite.cases.filter((caseDef) => caseDef.id === caseId || `${suite.name}/${caseDef.id}` === caseId)
+    }))
+        .filter((suite) => suite.cases.length > 0);
+}
+async function discoverSuiteFiles(workspacePath) {
+    const files = [];
+    const direct = path.join(workspacePath, ".flowseeker", "eval.json");
+    if (await pathExists(direct)) {
+        files.push(direct);
+    }
+    const evalDir = path.join(workspacePath, ".flowseeker", "evals");
+    if (await pathExists(evalDir)) {
+        await collectJsonFiles(evalDir, files);
+    }
+    return files.sort((left, right) => left.localeCompare(right));
+}
+async function collectJsonFiles(directoryPath, output) {
+    const entries = await fs.readdir(directoryPath, { withFileTypes: true });
+    for (const entry of entries) {
+        const absolutePath = path.join(directoryPath, entry.name);
+        if (entry.isDirectory()) {
+            await collectJsonFiles(absolutePath, output);
+        }
+        else if (entry.isFile() && entry.name.endsWith(".json")) {
+            output.push(absolutePath);
+        }
+    }
+}
+async function readSuite(suitePath) {
+    const suite = await readJson(suitePath);
+    if (!suite.name || !Array.isArray(suite.cases)) {
+        throw new Error(`Invalid eval suite: ${suitePath}`);
+    }
+    suite.cases.forEach((caseDef, index) => {
+        if (!caseDef?.id || !caseDef?.task) {
+            throw new Error(`Invalid eval case at ${suitePath}#${index + 1}: id and task are required.`);
+        }
+    });
+    return {
+        name: suite.name,
+        description: suite.description,
+        cases: suite.cases
+    };
+}
+function evaluateCase(workspaceName, workspacePath, suiteName, caseDef, result, durationMs) {
+    const passAtK = caseDef.passAtK ?? 10;
+    const schemaIssues = validateCaseDefinition(workspacePath, caseDef);
+    const validGroups = resolveRequiredGroups(caseDef);
+    const expectedMetrics = (0, evaluationMetrics_1.computeExpectedPathMetrics)(result.units, caseDef.expectedPaths ?? [], passAtK, caseDef.expectedMode ?? "any");
+    const groupResults = validGroups.length > 0 ? (0, evaluationMetrics_1.computeGroupResults)(result.units, validGroups, passAtK) : undefined;
+    const allGroupsPass = groupResults ? groupResults.every((g) => g.pass) : undefined;
+    const hardAvoidHitsTop10 = findAvoidHits(result.units.slice(0, 10), caseDef.avoidPaths ?? [], "hard");
+    const contrastAvoidHitsTop10 = findAvoidHits(result.units.slice(0, 10), caseDef.avoidPaths ?? [], "contrast");
+    const wrongSubsystemAvoidHitsTop10 = findWrongSubsystemAvoidHits(result, caseDef.avoidPaths ?? []);
+    // When requiredGroups are present, pass depends on all groups passing (and expectedPaths pass if provided)
+    const pass = groupResults && groupResults.length > 0
+        ? (expectedMetrics.pass !== false && allGroupsPass === true)
+        : expectedMetrics.pass;
+    const contaminationHitsTop10 = findContaminationHits((0, fileGroups_1.aggregateEvidenceFiles)(result.units).slice(0, 10).map((group) => group.relativePath));
+    return {
+        workspaceName,
+        workspacePath,
+        suiteName,
+        caseDef,
+        result,
+        durationMs,
+        firstHitRank: expectedMetrics.firstUnitHitRank,
+        firstFileHitRank: expectedMetrics.firstFileHitRank,
+        expectedMetrics: { ...expectedMetrics, groupResults, allGroupsPass },
+        pass: pass,
+        hitAt5: expectedMetrics.coverageAt5 > 0,
+        hitAt10: expectedMetrics.coverageAt10 > 0,
+        hitAt20: expectedMetrics.coverageAt20 > 0,
+        hitAt40: expectedMetrics.coverageAt40 > 0,
+        avoidHitsTop10: uniqueStrings([...hardAvoidHitsTop10, ...contrastAvoidHitsTop10]),
+        hardAvoidHitsTop10,
+        contrastAvoidHitsTop10,
+        wrongSubsystemAvoidHitsTop10,
+        groupResults,
+        allGroupsPass,
+        schemaIssues,
+        contaminationHitsTop10,
+        rankingQuality: (0, evaluationMetrics_1.computeRankingQualityMetrics)(result)
+    };
+}
+function findWrongSubsystemAvoidHits(result, avoidPaths) {
+    if (avoidPaths.length === 0) {
+        return [];
+    }
+    const topGroups = (0, fileGroups_1.aggregateEvidenceFiles)(result.units).slice(0, 10);
+    const primarySubsystem = result.units.find((unit) => unit.subsystem && !unit.contrastContext)?.subsystem;
+    if (!primarySubsystem) {
+        return [];
+    }
+    return uniqueStrings(topGroups
+        .filter((group) => group.subsystem && group.subsystem !== primarySubsystem)
+        .filter((group) => avoidPaths.some((avoidPath) => (0, evaluationMetrics_1.pathMatches)(group.relativePath, avoidPath)))
+        .map((group) => group.relativePath));
+}
+function findAvoidHits(units, avoidPaths, mode = "hard") {
+    if (avoidPaths.length === 0) {
+        return [];
+    }
+    return uniqueStrings(units
+        .filter((unit) => avoidPaths.some((avoidPath) => (0, evaluationMetrics_1.pathMatches)(unit.relativePath, avoidPath)))
+        .filter((unit) => mode === "contrast" ? unit.contrastContext : !unit.contrastContext)
+        .map((unit) => unit.relativePath));
+}
+function resolveRequiredGroups(caseDef) {
+    const groups = caseDef.requiredGroups;
+    if (!Array.isArray(groups)) {
+        return [];
+    }
+    const explicitGroups = groups.filter(isRequiredGroup);
+    if (explicitGroups.length > 0) {
+        return explicitGroups;
+    }
+    if (groups.some((group) => typeof group === "string")) {
+        return inferRequiredGroups(caseDef.expectedPaths ?? [], caseDef.expectedMode ?? "any");
+    }
+    return [];
+}
+function isRequiredGroup(group) {
+    const value = group;
+    return Boolean(value &&
+        typeof value.name === "string" &&
+        Array.isArray(value.paths) &&
+        value.paths.every((item) => typeof item === "string") &&
+        (value.mode === "any" || value.mode === "all"));
+}
+function inferRequiredGroups(expectedPaths, expectedMode) {
+    const byName = new Map();
+    for (const expectedPath of expectedPaths) {
+        const groupName = inferGroupName(expectedPath);
+        byName.set(groupName, [...(byName.get(groupName) ?? []), expectedPath]);
+    }
+    const groups = Array.from(byName.entries()).map(([name, paths]) => ({
+        name,
+        paths,
+        mode: "any"
+    }));
+    if (groups.length === 1 && expectedMode === "all") {
+        return groups.map((group) => ({ ...group, mode: "all" }));
+    }
+    return groups;
+}
+function inferGroupName(relativePath) {
+    const normalized = normalizeCasePath(relativePath);
+    if (/(^|\/)(__tests__|tests?|specs?)(\/|$)|\.(test|spec)\./.test(normalized)) {
+        return "verification";
+    }
+    if (/(^|\/)(config|configs?|settings?|environments?)(\/|$)|\.(env|toml|ya?ml|properties|ini)$/.test(normalized)) {
+        return "config";
+    }
+    if (/(^|\/)(migrations?|schema|schemas|models?|entities?|repositories?|prisma)(\/|$)|\.(sql|prisma)$/.test(normalized)) {
+        return "data";
+    }
+    if (/(^|\/)(jobs?|workers?|events?|listeners?|mailers?|mail|notifications?|webhooks?|queues?)(\/|$)/.test(normalized)) {
+        return "side_effect";
+    }
+    if (/(^|\/)(policies?|permissions?|guards?|middleware|auth)(\/|$)/.test(normalized)) {
+        return "permission";
+    }
+    if (/(^|\/)(routes?|router|controllers?|handlers?|api)(\/|$)|\+page\.(svelte|ts|tsx)$|page\.(ts|tsx|jsx|svelte)$/.test(normalized)) {
+        return "entry";
+    }
+    if (/(^|\/)(components?|views?|templates?|pages?|store|stores|hooks?|context)(\/|$)|\.(svelte|vue|tsx|jsx|erb|blade\.php)$/.test(normalized)) {
+        return "ui";
+    }
+    if (/(^|\/)(services?|usecases?|domain|lib|utils?|clients?)(\/|$)/.test(normalized)) {
+        return "core_logic";
+    }
+    return "context";
+}
+function validateCaseDefinition(workspacePath, caseDef) {
+    const issues = [];
+    const expectedPaths = caseDef.expectedPaths ?? [];
+    const avoidPaths = caseDef.avoidPaths ?? [];
+    if (caseDef.expectedMode && caseDef.expectedMode !== "any" && caseDef.expectedMode !== "all" && caseDef.expectedMode !== "subset") {
+        issues.push(`invalid expectedMode '${caseDef.expectedMode}'`);
+    }
+    const expectedSet = new Set(expectedPaths.map(normalizeCasePath));
+    for (const avoidPath of avoidPaths) {
+        if (expectedSet.has(normalizeCasePath(avoidPath))) {
+            issues.push(`expected/avoid overlap: ${avoidPath}`);
+        }
+    }
+    for (const expectedPath of expectedPaths) {
+        if (!casePathExists(workspacePath, expectedPath)) {
+            issues.push(`missing expected path: ${expectedPath}`);
+        }
+    }
+    for (const avoidPath of avoidPaths) {
+        if (!casePathExists(workspacePath, avoidPath)) {
+            issues.push(`missing avoid path: ${avoidPath}`);
+        }
+    }
+    if (Array.isArray(caseDef.requiredGroups)) {
+        for (const group of caseDef.requiredGroups) {
+            if (typeof group === "string" && expectedPaths.length > 0) {
+                continue;
+            }
+            if (!isRequiredGroup(group)) {
+                issues.push("invalid requiredGroups entry; use { name, paths, mode }");
+                continue;
+            }
+            for (const groupPath of group.paths) {
+                if (!casePathExists(workspacePath, groupPath)) {
+                    issues.push(`missing group path (${group.name}): ${groupPath}`);
+                }
+            }
+        }
+    }
+    return Array.from(new Set(issues));
+}
+function casePathExists(workspacePath, relativePath) {
+    if (relativePath.includes("*")) {
+        return true;
+    }
+    return fssync.existsSync(path.join(workspacePath, relativePath));
+}
+function normalizeCasePath(relativePath) {
+    return relativePath.replace(/\\/g, "/").replace(/^\/+/, "").toLowerCase();
+}
+function findContaminationHits(relativePaths) {
+    return relativePaths.filter(isContaminationPath);
+}
+function isContaminationPath(relativePath) {
+    const normalized = normalizeCasePath(relativePath);
+    return /(^|\/)\.flowseeker(\/|$)|(^|\/)_gen[^/]*\.(py|sh)$|(^|\/)_bulk_write\.js$/.test(normalized);
+}
+function renderReport(manifest, results, durationMs) {
+    const lines = [
+        "# FlowSeeker Headless Evaluation Report",
+        "",
+        `Generated: ${new Date().toISOString()}`,
+        `Benchmark: ${manifest.name ?? "Unnamed benchmark"}`,
+        `Workspaces: ${new Set(results.map((result) => result.workspaceName)).size}`,
+        `Cases: ${results.length}`,
+        `Duration: ${(durationMs / 1000).toFixed(2)}s`,
+        "",
+        "## Summary",
+        "",
+        formatSummary(results),
+        formatTokenSavingsSummary(results),
+        "",
+        "## Workspace Summary",
+        "",
+        ...renderWorkspaceSummary(results),
+        "",
+        "## Cases",
+        ""
+    ];
+    for (const result of results) {
+        lines.push(...renderCase(result));
+    }
+    return `${lines.join("\n")}\n`;
+}
+function renderWorkspaceSummary(results) {
+    const byWorkspace = new Map();
+    for (const result of results) {
+        byWorkspace.set(result.workspaceName, [...(byWorkspace.get(result.workspaceName) ?? []), result]);
+    }
+    return Array.from(byWorkspace.entries()).map(([workspaceName, workspaceResults]) => `- ${workspaceName}: ${formatSummary(workspaceResults)}; ${formatTokenSavingsSummary(workspaceResults)}`);
+}
+function renderCase(result) {
+    const status = result.pass === "unlabeled" ? "UNLABELED" : result.pass ? "PASS" : "FAIL";
+    const stats = result.result.stats;
+    const topFiles = (0, fileGroups_1.aggregateEvidenceFiles)(result.result.units).slice(0, 15);
+    return [
+        `### ${status} - ${result.workspaceName}/${result.suiteName}/${result.caseDef.id}`,
+        "",
+        `Workspace: ${result.workspacePath}`,
+        `Task: ${result.caseDef.task}`,
+        ...(result.caseDef.notes ? [`Notes: ${result.caseDef.notes}`] : []),
+        `Intent: ${result.result.profile.intent}`,
+        `Keywords: ${result.result.profile.keywords.join(", ") || "none"}`,
+        `Strategies: ${result.result.profile.strategies.join(", ") || "none"}`,
+        `Expected paths: ${(result.caseDef.expectedPaths ?? []).join(", ") || "none"}`,
+        `Expected mode: ${result.caseDef.expectedMode ?? "any"}`,
+        `Avoid paths: ${(result.caseDef.avoidPaths ?? []).join(", ") || "none"}`,
+        `First expected unit rank: ${result.firstHitRank ?? "none"}`,
+        `First expected file rank: ${result.firstFileHitRank ?? "none"}`,
+        `Coverage@5/10/20/40: ${result.expectedMetrics.coverageAt5}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt10}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt20}/${result.expectedMetrics.expectedCount}, ${result.expectedMetrics.coverageAt40}/${result.expectedMetrics.expectedCount}`,
+        `Missing expected at 10: ${result.expectedMetrics.missingAt10.join(", ") || "none"}`,
+        `Missing expected at 20: ${result.expectedMetrics.missingAt20.join(", ") || "none"}`,
+        `Avoid hits in top 10: ${result.hardAvoidHitsTop10.join(", ") || "none"}`,
+        `Contrast avoid hits in top 10: ${result.contrastAvoidHitsTop10.join(", ") || "none"}`,
+        `Wrong-subsystem avoid hits in top 10: ${result.wrongSubsystemAvoidHitsTop10.join(", ") || "none"}`,
+        `Contamination hits in top 10: ${result.contaminationHitsTop10.join(", ") || "none"}`,
+        `Schema issues: ${result.schemaIssues.join("; ") || "none"}`,
+        ...(result.groupResults && result.groupResults.length > 0 ? [
+            `All groups pass: ${result.allGroupsPass ? "yes" : "no"}`,
+            ...result.groupResults.map((g) => `Group ${g.name} [${g.mode}]: pass@${g.passAtK}=${g.pass}, rank=${g.firstHitRank ?? "none"}, coverage@${g.passAtK}=${g.coverageAtPassK}/${g.expectedCount}, missing=${g.missingAtPassK.join(", ") || "none"}`)
+        ] : []),
+        `Token savings: ${formatCaseTokenSavings(result)}`,
+        `Context coverage: ${formatContextCoverage(result.result)}`,
+        `Ranking quality: uniqueTop10Files=${result.rankingQuality.uniqueTop10Files}, duplicateEvidenceTop10=${result.rankingQuality.duplicateEvidenceTop10}, slotCoverageTop10=${result.rankingQuality.slotCoverageTop10}/${result.rankingQuality.requiredSlotCount}${result.rankingQuality.missingRequiredSlotsTop10.length ? `, missingSlots=${result.rankingQuality.missingRequiredSlotsTop10.join("|")}` : ""}, contrastContextTop10=${result.rankingQuality.contrastContextTop10}, primarySubsystem=${result.rankingQuality.primarySubsystemTop10 ?? "none"}, subsystemPurityTop10=${result.rankingQuality.subsystemPurityTop10.toFixed(2)}, symbolEvidenceTop10=${result.rankingQuality.symbolEvidenceTop10}`,
+        `Duration: ${(result.durationMs / 1000).toFixed(2)}s`,
+        `Scan: discovered=${stats.discoveredFiles}, scanned=${stats.scannedFiles}, skipped=${stats.skippedFiles}, bytes=${formatBytes(stats.scannedBytes)}, rawEvidence=${stats.rawEvidenceCount}, ranked=${stats.rankedEvidenceCount}, capped=${stats.capped}, stoppedEarly=${stats.stoppedEarly}${stats.stopReason ? ` (${stats.stopReason})` : ""}${formatIndexStats(stats)}`,
+        "",
+        "Top files:",
+        "",
+        ...topFiles.map((group, index) => `${index + 1}. ${group.relativePath} [${group.role}/${group.slot}] score=${group.score.toFixed(1)} evidence=${group.unitCount} dup=${group.duplicateUnitCount} contrast=${group.contrastContext} subsystem=${group.subsystem ?? "none"} kinds=${group.kinds.join(",")}`),
+        "",
+        "Top evidence:",
+        "",
+        ...result.result.units.slice(0, 15).map((unit, index) => {
+            const range = unit.range ? `:${unit.range.startLine}-${unit.range.endLine}` : "";
+            const passes = unit.retrievalPasses.length > 0 ? unit.retrievalPasses.join("+") : "none";
+            const meta = [unit.isDuplicate ? "duplicate" : "", unit.contrastContext ? "contrast" : "", unit.frameworkEdgeReason ? "framework" : ""].filter(Boolean).join(",");
+            return `${index + 1}. ${unit.relativePath}${range} [${unit.role}/${unit.kind}/${unit.slot ?? "unknown"}${meta ? `/${meta}` : ""}] score=${unit.score.toFixed(1)} passes=${passes}`;
+        }),
+        ""
+    ];
+}
+function formatSummary(results) {
+    const labeled = results.filter((result) => result.pass !== "unlabeled");
+    const passed = labeled.filter((result) => result.pass === true);
+    const hitAt5 = count(results, (result) => result.hitAt5);
+    const hitAt10 = count(results, (result) => result.hitAt10);
+    const hitAt20 = count(results, (result) => result.hitAt20);
+    const hitAt40 = count(results, (result) => result.hitAt40);
+    const allExpectedAt10 = count(labeled, (result) => result.expectedMetrics.expectedCount > 0 && result.expectedMetrics.coverageAt10 === result.expectedMetrics.expectedCount);
+    const allExpectedAt20 = count(labeled, (result) => result.expectedMetrics.expectedCount > 0 && result.expectedMetrics.coverageAt20 === result.expectedMetrics.expectedCount);
+    const avgExpectedCoverageAt10 = average(labeled.map((result) => result.expectedMetrics.expectedCount > 0 ? result.expectedMetrics.coverageAt10 / result.expectedMetrics.expectedCount : 0));
+    const avgMrr = average(labeled.map((result) => (result.firstFileHitRank ? 1 / result.firstFileHitRank : 0)));
+    const avgDuration = average(results.map((result) => result.durationMs));
+    const withGroups = results.filter((r) => r.allGroupsPass !== undefined);
+    const groupsPassed = withGroups.filter((r) => r.allGroupsPass === true).length;
+    const avoidCases = results.filter((result) => (result.caseDef.avoidPaths ?? []).length > 0).length;
+    const avoidHitCases = results.filter((result) => result.hardAvoidHitsTop10.length > 0).length;
+    const contrastAvoidCases = results.filter((result) => result.contrastAvoidHitsTop10.length > 0).length;
+    const wrongSubsystemAvoidCases = results.filter((result) => result.wrongSubsystemAvoidHitsTop10.length > 0).length;
+    const contaminationCases = results.filter((result) => result.contaminationHitsTop10.length > 0).length;
+    const schemaIssueCases = results.filter((result) => result.schemaIssues.length > 0).length;
+    const avgUniqueTop10Files = average(results.map((result) => result.rankingQuality.uniqueTop10Files));
+    const avgDuplicateEvidenceTop10 = average(results.map((result) => result.rankingQuality.duplicateEvidenceTop10));
+    const avgSlotCoverageTop10 = average(results.map((result) => result.rankingQuality.requiredSlotCount > 0 ? result.rankingQuality.slotCoverageTop10 / result.rankingQuality.requiredSlotCount : 1));
+    const avgSubsystemPurityTop10 = average(results.map((result) => result.rankingQuality.subsystemPurityTop10));
+    const avgSymbolEvidenceTop10 = average(results.map((result) => result.rankingQuality.symbolEvidenceTop10));
+    const groupSummary = withGroups.length > 0
+        ? `, groupsPass=${groupsPassed}/${withGroups.length}`
+        : "";
+    const avgTokenReduction = average(results
+        .map((result) => result.result.tokenSavings)
+        .filter((item) => item !== undefined && item.status !== "unavailable")
+        .map((item) => item.reductionPercent / 100));
+    const avoidHitRate = avoidCases > 0 ? avoidHitCases / avoidCases : 0;
+    const qualityScore = (0, evaluationMetrics_1.computeQualityScore)({
+        hitAt10Norm: labeled.length > 0 ? count(labeled, (result) => result.hitAt10) / labeled.length : 0,
+        contextCoverageNorm: avgSlotCoverageTop10,
+        avoidHitRate,
+        mrr: avgMrr,
+        tokenReductionNorm: avgTokenReduction
+    });
+    return `quality=${qualityScore.toFixed(3)}, labeled=${labeled.length}, pass=${passed.length}/${labeled.length}, hit@5=${hitAt5}/${results.length}, hit@10=${hitAt10}/${results.length}, hit@20=${hitAt20}/${results.length}, hit@40=${hitAt40}/${results.length}, allExpected@10=${allExpectedAt10}/${labeled.length}, allExpected@20=${allExpectedAt20}/${labeled.length}, avgExpectedCoverage@10=${avgExpectedCoverageAt10.toFixed(3)}, MRR=${avgMrr.toFixed(3)}, avg=${(avgDuration / 1000).toFixed(2)}s${groupSummary}, avoidHitCases=${avoidHitCases}/${avoidCases}, contrastAvoidCases=${contrastAvoidCases}/${avoidCases}, wrongSubsystemAvoidCases=${wrongSubsystemAvoidCases}/${avoidCases}, contaminationCases=${contaminationCases}/${results.length}, schemaIssueCases=${schemaIssueCases}/${results.length}, uniqueTop10FilesAvg=${avgUniqueTop10Files.toFixed(1)}, duplicateEvidenceTop10Avg=${avgDuplicateEvidenceTop10.toFixed(1)}, slotCoverageTop10Avg=${avgSlotCoverageTop10.toFixed(2)}, subsystemPurityTop10Avg=${avgSubsystemPurityTop10.toFixed(2)}, symbolEvidenceTop10Avg=${avgSymbolEvidenceTop10.toFixed(1)}`;
+}
+function formatTokenSavingsSummary(results) {
+    const metrics = results.map((result) => result.result.tokenSavings).filter((item) => item !== undefined && item.status !== "unavailable");
+    if (metrics.length === 0) {
+        return "tokenSavings=unavailable";
+    }
+    const avgReduction = average(metrics.map((item) => item.reductionPercent));
+    const avgPacketTokens = average(metrics.map((item) => item.solvePacketTokensEstimate));
+    const avgWorkspaceTokens = average(metrics.map((item) => item.workspaceTokensEstimate));
+    const partial = metrics.filter((item) => item.status === "partial").length;
+    return `tokenSavings avg=${(0, tokenSavings_1.formatReductionPercent)(avgReduction)}, packetAvg=${(0, tokenSavings_1.formatTokenCount)(avgPacketTokens)}, workspaceAvg=${(0, tokenSavings_1.formatTokenCount)(avgWorkspaceTokens)}, partial=${partial}/${metrics.length}`;
+}
+function formatCaseTokenSavings(result) {
+    const metrics = result.result.tokenSavings;
+    if (!metrics || metrics.status === "unavailable") {
+        return "unavailable";
+    }
+    const note = metrics.reason ? `; note=${metrics.reason}` : "";
+    const warnings = metrics.proofWarnings.length > 0 ? `; warnings=${metrics.proofWarnings.join(" | ")}` : "";
+    return `Solve Packet ${(0, tokenSavings_1.formatTokenCount)(metrics.solvePacketTokensEstimate)} / workspace ${(0, tokenSavings_1.formatTokenCount)(metrics.workspaceTokensEstimate)} tokens, candidates=${(0, tokenSavings_1.formatTokenCount)(metrics.candidateTokensEstimate)}, reduction ${(0, tokenSavings_1.formatReductionPercent)(metrics.reductionPercent)}, method=${metrics.method}, tokenizer=${metrics.tokenizer}, basis=${metrics.basis}, status=${metrics.status}, proof=${metrics.proofStatus}, files=${metrics.rankedFileCount}/${metrics.workspaceFileCount}${note}${warnings}`;
+}
+function formatContextCoverage(result) {
+    const coverage = result.contextCoverage;
+    if (!coverage) {
+        return "coverage unavailable";
+    }
+    const missing = coverage.missingRequiredSlots.length > 0 ? `, missing=${coverage.missingRequiredSlots.join("|")}` : "";
+    return `coverage ${coverage.foundRequiredSlots.length}/${coverage.requiredSlots.length} required (${coverage.requiredCoveragePercent.toFixed(0)}%)${missing}`;
+}
+function formatCaseStatus(result) {
+    if (result.pass === "unlabeled") {
+        return "UNLABELED";
+    }
+    const base = result.pass ? `PASS fileRank=${result.firstFileHitRank ?? "n/a"}` : "FAIL missing expected coverage";
+    if (result.groupResults && result.groupResults.length > 0) {
+        const groupStatus = result.groupResults.map((g) => `${g.name}=${g.pass ? "PASS" : "FAIL"}`).join(" ");
+        const allG = result.allGroupsPass ? "groups=ALL_PASS" : "groups=SOME_FAIL";
+        return `${base}; ${allG}; ${groupStatus}`;
+    }
+    return base;
+}
+function formatIndexConsole(stats) {
+    if (stats.indexedFiles === undefined) {
+        return "";
+    }
+    return `; ${stats.candidateFiles ?? 0}/${stats.indexedFiles} candidates; index ${stats.indexReusedFiles ?? 0} reused/${stats.indexUpdatedFiles ?? 0} updated`;
+}
+function formatIndexStats(stats) {
+    if (stats.indexedFiles === undefined) {
+        return "";
+    }
+    return `, indexed=${stats.indexedFiles}, candidates=${stats.candidateFiles ?? "unknown"}, indexReused=${stats.indexReusedFiles ?? 0}, indexUpdated=${stats.indexUpdatedFiles ?? 0}`;
+}
+async function readJson(filePath) {
+    const text = await fs.readFile(filePath, "utf8");
+    return JSON.parse(text);
+}
+function validateManifest(manifest, manifestPath) {
+    if (!Array.isArray(manifest.workspaces) || manifest.workspaces.length === 0) {
+        throw new Error(`Invalid manifest ${manifestPath}: workspaces must be a non-empty array.`);
+    }
+    manifest.workspaces.forEach((workspace, index) => {
+        if (!workspace.path) {
+            throw new Error(`Invalid manifest ${manifestPath}: workspaces[${index}].path is required.`);
+        }
+    });
+}
+async function validateManifestCases(manifest) {
+    const issues = [];
+    for (const workspace of manifest.workspaces) {
+        const workspacePath = path.resolve(workspace.path);
+        const workspaceName = workspace.name ?? path.basename(workspacePath);
+        const suites = await loadSuitesForWorkspace(workspacePath, workspace);
+        for (const suite of suites) {
+            for (const caseDef of suite.cases) {
+                const prefix = `${workspaceName}/${suite.name}/${caseDef.id}`;
+                for (const issue of validateCaseDefinition(workspacePath, caseDef)) {
+                    issues.push(`${prefix}: ${issue}`);
+                }
+            }
+        }
+    }
+    return issues;
+}
+async function pathExists(filePath) {
+    try {
+        await fs.access(filePath);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+function defaultReportPath() {
+    const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
+    return path.join(".flowseeker", "reports", `headless-eval-${timestamp}.md`);
+}
+function yesNo(value) {
+    return value ? "yes" : "no";
+}
+function uniqueStrings(values) {
+    return Array.from(new Set(values));
+}
+function count(items, predicate) {
+    return items.filter(predicate).length;
+}
+function average(values) {
+    if (values.length === 0) {
+        return 0;
+    }
+    return values.reduce((sum, value) => sum + value, 0) / values.length;
+}
+function formatBytes(bytes) {
+    if (bytes < 1024 * 1024) {
+        return `${(bytes / 1024).toFixed(1)}KB`;
+    }
+    return `${(bytes / 1024 / 1024).toFixed(1)}MB`;
+}
+//# sourceMappingURL=runEvaluation.js.map