@fodx/codelens 2.5.0 → 2.5.1

package/build/src/eval/evaluator.js

@@ -11,19 +11,23 @@ import { ensureFreshIndex } from "../index/reindex.js";
 import { scanFiles } from "../index/scanner.js";
 import { setPendingPaths } from "../index/staleness.js";
 import { ctxImpact } from "../tools/impact.js";
+import { DEFAULT_WEIGHTS } from "../search/rank.js";
 import { ctxSearch } from "../tools/search.js";
 import { isInsideRepo, resolveReal } from "../util/paths.js";
 import { runRgBaseline } from "./baseline.js";
-import { aggregateRuns, artifactPaths, evaluateObservation, writeEvalArtifacts } from "./report.js";
+import { aggregateRuns, artifactPaths, evaluateObservation, graphAggregates, retrievalAggregates, writeEvalArtifacts, } from "./report.js";
+import { loadEvalTaskFile, requiredTaskPaths } from "./task-file.js";
 import { generateEvalTasks, seededShuffle } from "./tasks.js";
+const RETRIEVAL_ARMS = ["full", "lexical", "fts", "rg"];
+const NON_GRAPH_WEIGHT = 1 - DEFAULT_WEIGHTS.graph;
 const LEXICAL_WEIGHTS = {
-    fts: 0.50,
-    symbol: 0.22,
+    fts: DEFAULT_WEIGHTS.fts / NON_GRAPH_WEIGHT,
+    symbol: DEFAULT_WEIGHTS.symbol / NON_GRAPH_WEIGHT,
     graph: 0,
-    code: 0.05,
-    pathHit: 0.10,
-    exact: 0.13,
-    recency: 0,
+    code: DEFAULT_WEIGHTS.code / NON_GRAPH_WEIGHT,
+    pathHit: DEFAULT_WEIGHTS.pathHit / NON_GRAPH_WEIGHT,
+    exact: DEFAULT_WEIGHTS.exact / NON_GRAPH_WEIGHT,
+    recency: DEFAULT_WEIGHTS.recency,
 };
 const FTS_ONLY_WEIGHTS = {
     fts: 1,
@@ -69,9 +73,7 @@ class EvalProgressTracker {
         try {
             this.callback?.({ ...event, elapsedMs: performance.now() - this.started });
         }
-        catch {
-            // Progress reporting must never break an evaluation.
-        }
+        catch { /* progress is best-effort */ }
     }
 }
 export function defaultEvalOptions(repoRoot) {
@@ -82,12 +84,13 @@ export function defaultEvalOptions(repoRoot) {
         repeats: 1,
         resultLimit: 10,
         scales: [500, 2000, "all"],
+        suites: ["retrieval", "graph", "freshness"],
         quick: false,
-        freshness: true,
         thresholds: {
             minRecallAtK: 0.6,
             minMrr: 0.5,
             minSuccessRate: 0.7,
+            minGraphPrecision: 0.5,
         },
     };
 }
@@ -96,7 +99,7 @@ export function quickEvalOptions(repoRoot) {
         ...defaultEvalOptions(repoRoot),
         taskLimit: 20,
         scales: [500],
-        freshness: false,
+        suites: ["retrieval", "graph"],
         quick: true,
     };
 }
@@ -106,6 +109,7 @@ export function runRepositoryEval(input) {
     const progress = new EvalProgressTracker(input.onProgress, started);
     let temp;
     try {
+        const suites = normalizeSuites(input.suites);
         progress.start("repository", `Validating repository ${input.repoRoot}`);
         const requestedRoot = resolveReal(input.repoRoot);
         const scope = detectScope(requestedRoot);
@@ -119,21 +123,34 @@ export function runRepositoryEval(input) {
         if (scanned.length === 0)
             throw new Error(`no indexable files found: ${repoRoot}`);
         progress.complete("scan", `Found ${scanned.length} indexable files`, { current: scanned.length, total: scanned.length });
-        progress.start("setup", "Preparing evaluation scales and output paths");
+        progress.start("setup", "Preparing frozen tasks, nested scales, and output paths");
         const outputDirectory = input.outputDir ? canonicalOutputPath(input.outputDir) : defaultOutputDirectory(repoRoot);
         if (isInsideRepo(repoRoot, outputDirectory))
             throw new Error("evaluation output must be outside the target repository");
         const artifacts = artifactPaths(outputDirectory);
+        const loaded = input.taskFile ? loadEvalTaskFile(input.taskFile) : undefined;
+        let frozenTasks = (loaded?.tasks ?? []).filter((task) => suites.includes(task.suite));
+        const taskSuites = suites.filter((suite) => suite === "retrieval" || suite === "graph");
+        const taskSource = taskSuites.length === 0
+            ? { kind: "none", independentGroundTruth: true }
+            : loaded?.source ?? { kind: "automatic-self-evaluation", independentGroundTruth: false };
+        const scaleSpecs = taskSuites.length > 0 ? resolveScaleSpecs(input.scales, scanned.length) : [];
+        const required = loaded ? requiredTaskPaths(frozenTasks) : new Set();
+        ensureRequiredPaths(scanned, required);
+        if (scaleSpecs.length > 0 && required.size > scaleSpecs[0].count) {
+            throw new Error(`smallest scale (${scaleSpecs[0].count}) cannot contain ${required.size} required task paths`);
+        }
+        const orderedFiles = nestedFileOrder(scanned, required, input.seed);
         temp = mkdtempSync(join(tmpdir(), "codelens-eval-"));
         const skipped = [];
-        const scaleSpecs = resolveScaleSpecs(input.scales, scanned.length);
         const scales = [];
-        progress.complete("setup", `Prepared ${scaleSpecs.length} scale${scaleSpecs.length === 1 ? "" : "s"}: ${scaleSpecs.map((spec) => spec.label).join(", ")}`);
+        progress.complete("setup", `Prepared ${scaleSpecs.length} nested scale${scaleSpecs.length === 1 ? "" : "s"}${loaded ? ` from frozen tasks (${taskSource.source ?? "task file"})` : " from automatic self-evaluation"}`);
         for (let scaleIndex = 0; scaleIndex < scaleSpecs.length; scaleIndex++) {
             const spec = scaleSpecs[scaleIndex];
             const scaleCounts = { scale: spec.label, current: scaleIndex + 1, total: scaleSpecs.length };
             progress.start("scale", `Starting scale ${spec.label} with ${spec.count} files`, scaleCounts);
-            const selected = selectFiles(scanned, spec.count, input.seed + scaleIndex * 1009);
+            const selected = orderedFiles.slice(0, spec.count).sort((a, b) => a.path.localeCompare(b.path));
+            const inventory = selected.map((file) => file.path);
             const dbPath = join(temp, `eval-${scaleIndex}.db`);
             const db = openDb(dbPath);
             try {
@@ -144,28 +161,40 @@ export function runRepositoryEval(input) {
                 });
                 const indexMs = performance.now() - indexStart;
                 progress.complete("index", `Indexed ${build.indexedFiles} files and ${build.totalChunks} chunks for scale ${spec.label} in ${formatDuration(indexMs)} (${build.skipped} skipped)`, { scale: spec.label, current: selected.length, total: selected.length });
-                const inventory = selected.map((file) => file.path);
-                const scaleTaskLimit = distributedLimit(input.taskLimit, scaleSpecs.length, scaleIndex);
-                progress.start("tasks", `Generating up to ${scaleTaskLimit} tasks for scale ${spec.label}`, { scale: spec.label });
-                const tasks = generateEvalTasks(db, build.indexId, repoRoot, inventory, scaleTaskLimit, input.seed + scaleIndex * 7919);
-                progress.complete("tasks", `Generated ${tasks.length} tasks for scale ${spec.label}`, { scale: spec.label, current: tasks.length, total: scaleTaskLimit });
-                if (scaleTaskLimit === 0)
-                    skipped.push(`${spec.label}: no tasks allocated because --tasks is lower than the number of scales.`);
-                for (const type of ["locate", "callers", "tests", "history"]) {
-                    if (!tasks.some((task) => task.type === type))
-                        skipped.push(`${spec.label}: no ${type} tasks were generated.`);
+                if (!loaded && scaleIndex === 0) {
+                    progress.start("tasks", `Generating up to ${input.taskLimit} automatic self-evaluation tasks`, { scale: spec.label });
+                    frozenTasks = generateEvalTasks(db, build.indexId, repoRoot, inventory, input.taskLimit, input.seed, suites);
+                    progress.complete("tasks", `Froze ${frozenTasks.length} tasks for every scale`, { scale: spec.label, current: frozenTasks.length, total: input.taskLimit });
                 }
-                const expectedRuns = tasks.length * 4 * Math.max(1, input.repeats);
-                progress.start("retrieval", `Running ${expectedRuns} retrieval checks for scale ${spec.label}`, { scale: spec.label, current: 0, total: expectedRuns });
-                const runs = runTaskSet(db, repoRoot, tasks, input.repeats, input.resultLimit, (current, total) => {
-                    progress.update("retrieval", `Completed ${current}/${total} retrieval checks for scale ${spec.label}`, { scale: spec.label, current, total });
-                });
-                progress.complete("retrieval", `Completed ${runs.length} retrieval checks for scale ${spec.label}`, { scale: spec.label, current: runs.length, total: expectedRuns });
-                if (runs.some((run) => run.arm === "rg" && run.error === "ripgrep is not installed")) {
-                    skipped.push("rg baseline unavailable because ripgrep is not installed.");
-                    progress.skipped("retrieval", "rg baseline unavailable because ripgrep is not installed", { scale: spec.label });
+                const tasks = frozenTasks;
+                for (const suite of taskSuites) {
+                    if (!tasks.some((task) => task.suite === suite))
+                        skipped.push(`${suite}: no tasks were available.`);
+                }
+                const retrievalTasks = tasks.filter((task) => task.suite === "retrieval");
+                const graphTasks = tasks.filter((task) => task.suite === "graph" && (loaded !== undefined || scaleIndex === 0));
+                if (!loaded && scaleIndex > 0 && tasks.some((task) => task.suite === "graph")) {
+                    skipped.push("automatic graph self-consistency runs only at the label-generating smallest tier; frozen independent graph tasks may run across every tier.");
+                }
+                const runs = [];
+                if (retrievalTasks.length > 0) {
+                    const total = retrievalTasks.length * RETRIEVAL_ARMS.length * Math.max(1, input.repeats);
+                    progress.start("retrieval", `Running ${total} counterbalanced retrieval checks for scale ${spec.label}`, { scale: spec.label, current: 0, total });
+                    runs.push(...runRetrievalTaskSet(db, repoRoot, inventory, retrievalTasks, input.repeats, input.resultLimit, (current) => {
+                        progress.update("retrieval", `Completed ${current}/${total} retrieval checks for scale ${spec.label}`, { scale: spec.label, current, total });
+                    }));
+                    progress.complete("retrieval", `Completed ${total} retrieval checks for scale ${spec.label}`, { scale: spec.label, current: total, total });
                 }
-                const aggregates = aggregateByArm(runs);
+                if (graphTasks.length > 0) {
+                    const total = graphTasks.length * Math.max(1, input.repeats);
+                    progress.start("graph", `Running ${total} known-target graph checks for scale ${spec.label}`, { scale: spec.label, current: 0, total });
+                    runs.push(...runGraphTaskSet(db, graphTasks, input.repeats, input.resultLimit, (current) => {
+                        progress.update("graph", `Completed ${current}/${total} graph checks for scale ${spec.label}`, { scale: spec.label, current, total });
+                    }));
+                    progress.complete("graph", `Completed ${total} graph checks for scale ${spec.label}`, { scale: spec.label, current: total, total });
+                }
+                if (runs.some((run) => run.arm === "rg" && run.error === "ripgrep is not installed"))
+                    skipped.push("rg baseline unavailable because ripgrep is not installed.");
                 scales.push({
                     label: spec.label,
                     requestedFiles: spec.requested,
@@ -176,44 +205,72 @@ export function runRepositoryEval(input) {
                     dbBytes: databaseBytes(dbPath),
                     tasks,
                     runs,
-                    aggregates,
+                    retrieval: retrievalAggregates(tasks, runs, input.seed + scaleIndex * 1009),
+                    graph: graphAggregates(tasks, runs, input.seed + scaleIndex * 2003),
                 });
-                progress.complete("scale", `Finished scale ${spec.label}: ${formatAggregateSummary(aggregates.full)}`, scaleCounts);
+                progress.complete("scale", `Finished scale ${spec.label}: ${retrievalTasks.length} retrieval and ${graphTasks.length} graph tasks`, scaleCounts);
             }
             finally {
                 db.close();
             }
         }
         let freshness;
-        if (input.freshness) {
+        if (suites.includes("freshness")) {
             progress.start("freshness", "Starting detached-worktree edit/delete freshness probe");
             freshness = runFreshnessEval(repoRoot, input.seed, progress);
-            const freshnessMessage = freshness.attempted
+            const message = freshness.attempted
                 ? `Freshness probe ${freshness.passed ? "passed" : "failed"} in ${formatDuration(freshness.elapsedMs ?? 0)}`
                 : `Freshness probe skipped: ${freshness.skippedReason ?? "not attempted"}`;
             if (freshness.attempted && freshness.passed)
-                progress.complete("freshness", freshnessMessage);
+                progress.complete("freshness", message);
             else if (freshness.attempted)
-                progress.error("freshness", freshnessMessage);
+                progress.error("freshness", message);
             else
-                progress.skipped("freshness", freshnessMessage);
+                progress.skipped("freshness", message);
         }
         else {
             freshness = disabledFreshness();
-            progress.skipped("freshness", "Freshness probe disabled");
+            progress.skipped("freshness", "Freshness suite disabled");
         }
         if (freshness.skippedReason)
             skipped.push(`freshness: ${freshness.skippedReason}`);
-        progress.start("thresholds", "Checking configured quality thresholds");
+        progress.start("thresholds", "Checking quality thresholds on eligible task types");
+        const hasThresholdEligibleRetrieval = frozenTasks.some((task) => task.suite === "retrieval" && task.contributesToThresholds);
+        const hasThresholdEligibleGraph = frozenTasks.some((task) => task.suite === "graph" && task.groundTruth.independent && task.contributesToThresholds);
+        if (suites.includes("retrieval") && !hasThresholdEligibleRetrieval) {
+            skipped.push("no threshold-eligible retrieval tasks; retrieval quality is informational only.");
+        }
+        if (suites.includes("graph") && !hasThresholdEligibleGraph) {
+            skipped.push("no independently labeled threshold-eligible graph tasks; graph self-consistency is informational only.");
+        }
         const thresholdFailures = scales.flatMap((scale) => thresholdFailuresFor(scale, input));
-        if (input.freshness && freshness.attempted && !freshness.passed)
+        for (const scale of scales) {
+            const executionErrors = new Set(scale.runs
+                .filter((run) => run.error && !(run.arm === "rg" && run.error === "ripgrep is not installed"))
+                .map((run) => `${run.arm}: ${run.error}`));
+            for (const error of executionErrors)
+                thresholdFailures.push(`${scale.label}: evaluation arm error — ${error}`);
+        }
+        if (suites.includes("freshness") && freshness.attempted && !freshness.passed)
             thresholdFailures.push("Freshness edit/delete probe failed.");
         if (initialStatus !== gitStatus(repoRoot))
             thresholdFailures.push("Target repository worktree changed during evaluation.");
-        progress.complete("thresholds", thresholdFailures.length === 0 ? "All configured thresholds passed" : `${thresholdFailures.length} configured threshold${thresholdFailures.length === 1 ? "" : "s"} failed`);
-        const completedAt = new Date().toISOString();
+        progress.complete("thresholds", thresholdFailures.length === 0 ? "All applicable thresholds passed" : `${thresholdFailures.length} threshold${thresholdFailures.length === 1 ? "" : "s"} failed`);
+        const graphTasks = frozenTasks.filter((task) => task.suite === "graph");
+        const graphIndependentGroundTruth = graphTasks.length > 0 && graphTasks.every((task) => task.groundTruth.independent);
+        const hasTaskEvidence = taskSuites.length === 0 || scales.some((scale) => scale.runs.length > 0);
+        const hasFreshnessEvidence = !suites.includes("freshness") || freshness.attempted || !!freshness.skippedReason;
         const result = {
-            version: 1,
+            version: 2,
+            methodology: {
+                taskSource,
+                taskSetDigest: taskSetDigest(frozenTasks),
+                retrievalComparable: true,
+                graphIndependentGroundTruth,
+                repeatsAreTimingSamples: true,
+                scaleTasksFrozen: true,
+                notes: methodologyNotes(taskSource, suites, graphTasks.length > 0, graphIndependentGroundTruth),
+            },
             repository: {
                 root: repoRoot,
                 branch: scope.branch,
@@ -227,14 +284,14 @@ export function runRepositoryEval(input) {
                 repeats: input.repeats,
                 resultLimit: input.resultLimit,
                 scales: input.scales,
+                suites,
                 quick: input.quick,
-                freshness: input.freshness,
                 thresholds: input.thresholds,
             },
             startedAt,
-            completedAt,
+            completedAt: new Date().toISOString(),
             durationMs: performance.now() - started,
-            pass: scales.some((scale) => scale.tasks.length > 0) && thresholdFailures.length === 0,
+            pass: hasTaskEvidence && hasFreshnessEvidence && thresholdFailures.length === 0,
             thresholdFailures,
             skipped: [...new Set(skipped)],
             scales,
@@ -291,64 +348,113 @@ function buildSubsetIndex(db, scope, files, onProgress) {
     setPendingPaths(indexId, []);
     return { indexId, indexedFiles, totalChunks, skipped };
 }
-function runTaskSet(db, repoRoot, tasks, repeats, limit, onProgress) {
+function runRetrievalTaskSet(db, repoRoot, inventory, tasks, repeats, limit, onProgress) {
+    const repeatCount = Math.max(1, repeats);
+    if (tasks[0]) {
+        for (const arm of RETRIEVAL_ARMS)
+            runRetrievalArm(db, repoRoot, inventory, tasks[0], limit, arm);
+    }
+    const runs = [];
+    const nextTypeOffset = new Map();
+    const typeOffsets = tasks.map((task) => {
+        const offset = nextTypeOffset.get(task.type) ?? 0;
+        nextTypeOffset.set(task.type, offset + 1);
+        return offset;
+    });
+    for (let repeat = 0; repeat < repeatCount; repeat++) {
+        for (let taskIndex = 0; taskIndex < tasks.length; taskIndex++) {
+            const task = tasks[taskIndex];
+            const offset = (typeOffsets[taskIndex] + repeat) % RETRIEVAL_ARMS.length;
+            const orderedArms = [...RETRIEVAL_ARMS.slice(offset), ...RETRIEVAL_ARMS.slice(0, offset)];
+            for (let order = 0; order < orderedArms.length; order++) {
+                const arm = orderedArms[order];
+                const observation = runRetrievalArm(db, repoRoot, inventory, task, limit, arm);
+                runs.push(evaluateObservation(task, arm, observation, limit, repeat, order));
+                onProgress?.(runs.length);
+            }
+        }
+    }
+    return runs;
+}
+function runGraphTaskSet(db, tasks, repeats, limit, onProgress) {
+    if (tasks[0])
+        runGraphArm(db, tasks[0], limit);
     const runs = [];
-    const arms = ["full", "lexical", "fts", "rg"];
-    const total = tasks.length * arms.length * Math.max(1, repeats);
-    const progressInterval = Math.max(1, Math.ceil(total / 20));
     for (let repeat = 0; repeat < Math.max(1, repeats); repeat++) {
         for (const task of tasks) {
-            for (const arm of arms) {
-                const observation = arm === "rg"
-                    ? runRgBaseline(repoRoot, task, limit)
-                    : runCodeLens(db, task, limit, arm);
-                runs.push(evaluateObservation(task, arm, observation, limit));
-                if (runs.length === total || runs.length % progressInterval === 0)
-                    onProgress?.(runs.length, total);
-            }
+            const observation = runGraphArm(db, task, limit);
+            runs.push(evaluateObservation(task, "graph", observation, limit, repeat, 0));
+            onProgress?.(runs.length);
         }
     }
     return runs;
 }
-function runCodeLens(db, task, limit, arm) {
+function runRetrievalArm(db, repoRoot, inventory, task, limit, arm) {
+    if (arm === "rg")
+        return runRgBaseline(repoRoot, task, limit, inventory);
     const start = performance.now();
     try {
-        let result;
-        let foundPaths;
-        if (arm === "full" && (task.type === "callers" || task.type === "tests") && task.sourcePath) {
-            result = ctxImpact(db, { path: task.sourcePath, depth: 2 });
-            const impact = result;
-            foundPaths = [
-                ...(task.type === "tests" ? (impact.affectedTests ?? []).map((item) => item.path) : []),
-                ...(impact.callers ?? []).map((item) => item.path),
-                ...(impact.affectedFiles ?? []).map((item) => item.path),
-                impact.target?.path,
-            ].filter((path) => !!path);
-        }
-        else {
-            const weights = arm === "lexical" ? LEXICAL_WEIGHTS : arm === "fts" ? FTS_ONLY_WEIGHTS : undefined;
-            result = ctxSearch(db, task.query, { limit, snippet: "none", weights });
-            foundPaths = result.results.map((item) => item.path);
+        const weights = arm === "lexical" ? LEXICAL_WEIGHTS : arm === "fts" ? FTS_ONLY_WEIGHTS : undefined;
+        const foundPaths = [];
+        const seen = new Set();
+        let cursor;
+        let toolCalls = 0;
+        let bytesServed = 0;
+        // ctxSearch ranks chunks. Page through its bounded candidate set so every
+        // retrieval arm is scored on up to K unique files, matching rg's unit.
+        for (let page = 0; page < 4 && foundPaths.length < limit; page++) {
+            const result = ctxSearch(db, task.query, { limit, cursor, snippet: "none", weights });
+            toolCalls++;
+            bytesServed += Buffer.byteLength(JSON.stringify(result));
+            for (const item of result.results) {
+                if (seen.has(item.path))
+                    continue;
+                seen.add(item.path);
+                foundPaths.push(item.path);
+                if (foundPaths.length >= limit)
+                    break;
+            }
+            if (!result.nextCursor)
+                break;
+            cursor = result.nextCursor;
         }
-        foundPaths = [...new Set(foundPaths)].slice(0, limit);
         return {
             foundPaths,
-            toolCalls: 1,
-            bytesServed: Buffer.byteLength(JSON.stringify(result)),
-            bytesRead: 0,
+            toolCalls,
+            bytesServed,
             elapsedMs: performance.now() - start,
         };
     }
     catch (error) {
+        return failedObservation(start, error);
+    }
+}
+function runGraphArm(db, task, limit) {
+    const start = performance.now();
+    try {
+        if (!task.targetPath || (task.type !== "callers" && task.type !== "tests"))
+            throw new Error("graph task requires callers/tests type and targetPath");
+        const result = ctxImpact(db, { path: task.targetPath, depth: 1, includeTests: true });
+        const paths = task.type === "tests" ? result.affectedTests.map((item) => item.path) : result.callers.map((item) => item.path);
         return {
-            foundPaths: [],
+            foundPaths: [...new Set(paths)].slice(0, limit),
             toolCalls: 1,
-            bytesServed: 0,
-            bytesRead: 0,
+            bytesServed: Buffer.byteLength(JSON.stringify(result)),
             elapsedMs: performance.now() - start,
-            error: error instanceof Error ? error.message : String(error),
         };
     }
+    catch (error) {
+        return failedObservation(start, error);
+    }
+}
+function failedObservation(start, error) {
+    return {
+        foundPaths: [],
+        toolCalls: 1,
+        bytesServed: 0,
+        elapsedMs: performance.now() - start,
+        error: error instanceof Error ? error.message : String(error),
+    };
 }
 function runFreshnessEval(repoRoot, seed, progress) {
     const start = performance.now();
@@ -367,18 +473,14 @@ function runFreshnessEval(repoRoot, seed, progress) {
         worktreeAdded = true;
         progress.complete("freshness-worktree", `Created detached worktree at ${worktree}`);
         const scope = detectScope(worktree);
-        if (!scope) {
-            progress.skipped("freshness-worktree", "Temporary worktree scope detection failed");
+        if (!scope)
             return { enabled: true, attempted: false, passed: false, skippedReason: "temporary worktree scope detection failed" };
-        }
         progress.start("freshness-scan", "Scanning temporary worktree for a safe probe file");
         const scanned = scanFiles(worktree);
         const candidates = seededShuffle(scanned.filter(isFreshnessCandidate), seed);
         const selected = candidates.map((file) => ({ file, path: safeFreshnessPath(worktree, file) })).find((item) => item.path !== null);
-        if (!selected?.path) {
-            progress.skipped("freshness-scan", "No safe regular text file for freshness probe");
+        if (!selected?.path)
             return { enabled: true, attempted: false, passed: false, skippedReason: "no safe regular text file for freshness probe" };
-        }
         progress.complete("freshness-scan", `Selected ${selected.file.path} from ${scanned.length} files`);
         db = openDb(join(temp, "freshness.db"));
         progress.start("freshness-index", `Building freshness index for ${scanned.length} files`, { current: 0, total: scanned.length });
@@ -397,25 +499,12 @@ function runFreshnessEval(repoRoot, seed, progress) {
         ensureFreshIndex(db, scope, { budgetMs: 30000 });
         const deletedRemoved = !ctxSearch(db, token, { snippet: "none" }).results.some((result) => result.path === selected.file.path);
         progress.complete("freshness-delete", `Deleted content ${deletedRemoved ? "was removed" : "remained searchable"}`);
-        return {
-            enabled: true,
-            attempted: true,
-            passed: modifiedVisible && deletedRemoved,
-            modifiedVisible,
-            deletedRemoved,
-            elapsedMs: performance.now() - start,
-        };
+        return { enabled: true, attempted: true, passed: modifiedVisible && deletedRemoved, modifiedVisible, deletedRemoved, elapsedMs: performance.now() - start };
     }
     catch (error) {
         const message = error instanceof Error ? error.message : String(error);
         progress.error("freshness", message);
-        return {
-            enabled: true,
-            attempted: true,
-            passed: false,
-            elapsedMs: performance.now() - start,
-            error: message,
-        };
+        return { enabled: true, attempted: true, passed: false, elapsedMs: performance.now() - start, error: message };
     }
     finally {
         progress.start("freshness-cleanup", "Removing temporary worktree and Git metadata");
@@ -469,9 +558,8 @@ function cleanupTemporaryWorktree(repoRoot, temp, worktree, worktreeAdded) {
         return;
     }
     const pruned = spawnSync("git", ["worktree", "prune", "--expire", "now"], { cwd: repoRoot, encoding: "utf-8" });
-    if (pruned.error || pruned.status !== 0) {
+    if (pruned.error || pruned.status !== 0)
         throw new Error(`failed to prune temporary worktree metadata: ${pruned.error?.message ?? (pruned.stderr ?? "").trim()}`);
-    }
     const listed = spawnSync("git", ["worktree", "list", "--porcelain"], { cwd: repoRoot, encoding: "utf-8" });
     if (listed.error || listed.status !== 0)
         throw new Error(`failed to verify temporary worktree cleanup: ${listed.error?.message ?? "git worktree list failed"}`);
@@ -481,53 +569,104 @@ function cleanupTemporaryWorktree(repoRoot, temp, worktree, worktreeAdded) {
     if (tempFailure)
         throw new Error(`failed to remove evaluation temp directory: ${tempFailure}`);
 }
+function normalizeSuites(suites) {
+    const normalized = [...new Set(suites)];
+    if (normalized.length === 0)
+        throw new Error("at least one evaluation suite is required");
+    for (const suite of normalized) {
+        if (suite !== "retrieval" && suite !== "graph" && suite !== "freshness")
+            throw new Error(`unknown evaluation suite: ${suite}`);
+    }
+    return normalized;
+}
 function resolveScaleSpecs(scales, total) {
-    const resolved = [];
-    const seenCounts = new Set();
-    const requestedScales = scales.length > 0 ? scales : ["all"];
-    for (const scale of requestedScales) {
+    const byCount = new Map();
+    for (const scale of scales.length > 0 ? scales : ["all"]) {
         const count = scale === "all" ? total : Math.max(1, Math.min(total, Math.floor(scale)));
-        if (seenCounts.has(count))
-            continue;
-        seenCounts.add(count);
-        resolved.push({ label: count === total ? "all" : String(count), requested: scale, count });
-    }
-    if (resolved.length === 0)
-        resolved.push({ label: "all", requested: "all", count: total });
-    return resolved;
+        if (!byCount.has(count) || scale === "all")
+            byCount.set(count, scale);
+    }
+    return [...byCount.entries()].sort(([a], [b]) => a - b).map(([count, requested]) => ({
+        label: count === total ? "all" : String(count),
+        requested,
+        count,
+    }));
 }
-function selectFiles(files, count, seed) {
-    if (count >= files.length)
-        return [...files].sort((a, b) => a.path.localeCompare(b.path));
-    return seededShuffle(files, seed).slice(0, count).sort((a, b) => a.path.localeCompare(b.path));
+function nestedFileOrder(files, requiredPaths, seed) {
+    const required = [];
+    const remainder = [];
+    for (const file of files)
+        (requiredPaths.has(file.path) ? required : remainder).push(file);
+    required.sort((a, b) => a.path.localeCompare(b.path));
+    return [...required, ...seededShuffle(remainder, seed)];
 }
-function aggregateByArm(runs) {
-    const output = {};
-    for (const arm of ["full", "lexical", "fts", "rg"]) {
-        const armRuns = runs.filter((run) => run.arm === arm);
-        if (armRuns.length > 0)
-            output[arm] = aggregateRuns(armRuns);
-    }
-    return output;
+function ensureRequiredPaths(files, required) {
+    const known = new Set(files.map((file) => file.path));
+    const missing = [...required].filter((path) => !known.has(path));
+    if (missing.length > 0)
+        throw new Error(`task paths are missing or not indexable: ${missing.slice(0, 5).join(", ")}${missing.length > 5 ? ` (+${missing.length - 5} more)` : ""}`);
 }
 function thresholdFailuresFor(scale, options) {
-    if (scale.tasks.length === 0)
-        return [];
-    const metric = scale.aggregates.full;
-    if (!metric)
-        return [`${scale.label}: full CodeLens arm produced no metrics.`];
     const failures = [];
-    if (metric.recallAtK < options.thresholds.minRecallAtK)
-        failures.push(`${scale.label}: full recall@${options.resultLimit} ${metric.recallAtK.toFixed(3)} < ${options.thresholds.minRecallAtK.toFixed(3)}`);
-    if (metric.mrr < options.thresholds.minMrr)
-        failures.push(`${scale.label}: full MRR ${metric.mrr.toFixed(3)} < ${options.thresholds.minMrr.toFixed(3)}`);
-    if (metric.successRate < options.thresholds.minSuccessRate)
-        failures.push(`${scale.label}: full success rate ${metric.successRate.toFixed(3)} < ${options.thresholds.minSuccessRate.toFixed(3)}`);
+    for (const type of ["locate", "history"]) {
+        const eligible = new Set(scale.tasks.filter((task) => task.suite === "retrieval" && task.type === type && task.contributesToThresholds).map((task) => task.id));
+        if (eligible.size === 0)
+            continue;
+        const runs = scale.runs.filter((run) => run.arm === "full" && eligible.has(run.taskId));
+        const metric = aggregateRuns(runs, options.seed + type.length * 313);
+        if (metric.recallAtK < options.thresholds.minRecallAtK)
+            failures.push(`${scale.label}/${type}: full recall@${options.resultLimit} ${metric.recallAtK.toFixed(3)} < ${options.thresholds.minRecallAtK.toFixed(3)}`);
+        if (metric.mrr < options.thresholds.minMrr)
+            failures.push(`${scale.label}/${type}: full MRR ${metric.mrr.toFixed(3)} < ${options.thresholds.minMrr.toFixed(3)}`);
+        if (metric.successRate < options.thresholds.minSuccessRate)
+            failures.push(`${scale.label}/${type}: full success rate ${metric.successRate.toFixed(3)} < ${options.thresholds.minSuccessRate.toFixed(3)}`);
+    }
+    for (const type of ["callers", "tests"]) {
+        const eligible = new Set(scale.tasks.filter((task) => task.suite === "graph" && task.type === type && task.groundTruth.independent && task.contributesToThresholds).map((task) => task.id));
+        if (eligible.size === 0)
+            continue;
+        const runs = scale.runs.filter((run) => run.arm === "graph" && eligible.has(run.taskId));
+        const metric = aggregateRuns(runs, options.seed + type.length * 419);
+        if (metric.recallAtK < options.thresholds.minRecallAtK)
+            failures.push(`${scale.label}/${type}: graph recall@${options.resultLimit} ${metric.recallAtK.toFixed(3)} < ${options.thresholds.minRecallAtK.toFixed(3)}`);
+        if (metric.precisionAtK < options.thresholds.minGraphPrecision)
+            failures.push(`${scale.label}/${type}: graph precision@${options.resultLimit} ${metric.precisionAtK.toFixed(3)} < ${options.thresholds.minGraphPrecision.toFixed(3)}`);
+        if (metric.successRate < options.thresholds.minSuccessRate)
+            failures.push(`${scale.label}/${type}: graph success rate ${metric.successRate.toFixed(3)} < ${options.thresholds.minSuccessRate.toFixed(3)}`);
+    }
     return failures;
 }
-function distributedLimit(total, count, index) {
-    const base = Math.floor(total / count);
-    return base + (index < total % count ? 1 : 0);
+function methodologyNotes(taskSource, suites, hasGraph, graphIndependent) {
+    const notes = [];
+    const hasTaskSuite = suites.includes("retrieval") || suites.includes("graph");
+    if (suites.includes("retrieval")) {
+        notes.push("Retrieval arms receive identical natural-language queries and the same selected file inventory; chunk-ranked CodeLens pages are deduplicated to the same file-level top-K unit as rg, and the lexical ablation removes only graph weight before proportionally renormalizing the remaining default weights.", "The rg arm is a deterministic OR-term file-match heuristic with path-term tie-breaking; it is not a native ranked-search system.", "Retrieval arm order is deterministically counterbalanced and one warmup per arm is discarded before timing.", "Latency is end-to-end and environment-specific; rg measurements include process startup while CodeLens arms run in-process.");
+    }
+    if (hasTaskSuite) {
+        notes.push("Scale tiers are nested and reuse one fixed task set; larger tiers add distractor files rather than changing labels.", "Repeats are timing samples; unique tasks determine quality metrics and confidence intervals.", "Low-confidence tasks may be reported but only tasks marked contributesToThresholds affect pass/fail.");
+    }
+    if (taskSource.kind === "automatic-self-evaluation")
+        notes.push("Automatic tasks are a self-evaluation and are not independent benchmark ground truth; automatic graph self-consistency runs only at the label-generating smallest tier.");
+    if (hasGraph)
+        notes.push(graphIndependent
+            ? "Graph labels are declared independent by the frozen task file."
+            : "Graph labels come from the evaluated CodeLens index and measure self-consistency only.");
+    if (suites.includes("freshness"))
+        notes.push("Freshness mutations run only in a detached temporary worktree and cleanup is verified.");
+    return notes;
+}
+function taskSetDigest(tasks) {
+    const canonical = tasks.map((task) => ({
+        id: task.id,
+        suite: task.suite,
+        type: task.type,
+        query: task.query,
+        targetPath: task.targetPath,
+        expectedPaths: task.expectedPaths,
+        confidence: task.confidence,
+        contributesToThresholds: task.contributesToThresholds,
+    }));
+    return createHash("sha256").update(JSON.stringify(canonical)).digest("hex");
 }
 function isFreshnessCandidate(file) {
     if (!file.language || file.size > 512 * 1024)
@@ -542,17 +681,13 @@ function freshnessComment(language, token) {
         return `\n<!-- ${token} -->\n`;
     return `\n// ${token}\n`;
 }
-function disabledFreshness() {
-    return { enabled: false, attempted: false, passed: true };
-}
-function databaseBytes(path) {
-    try {
-        return statSync(path).size;
-    }
-    catch {
-        return 0;
-    }
+function disabledFreshness() { return { enabled: false, attempted: false, passed: true }; }
+function databaseBytes(path) { try {
+    return statSync(path).size;
 }
+catch {
+    return 0;
+} }
 function canonicalOutputPath(path) {
     let current = resolve(path);
     const missing = [];
@@ -578,11 +713,6 @@ function formatDuration(elapsedMs) {
     const minutes = Math.floor(elapsedMs / 60_000);
     return `${minutes}m ${((elapsedMs % 60_000) / 1000).toFixed(0)}s`;
 }
-function formatAggregateSummary(metric) {
-    if (!metric)
-        return "full arm produced no metrics";
-    return `full recall ${(metric.recallAtK * 100).toFixed(1)}%, MRR ${metric.mrr.toFixed(3)}, success ${(metric.successRate * 100).toFixed(1)}%`;
-}
 function gitStatus(repoRoot) {
     const result = spawnSync("git", ["status", "--porcelain=v1", "-z"], { cwd: repoRoot, encoding: "utf-8" });
     return result.status === 0 ? result.stdout ?? "" : "";