@dailephd/my-dev-kit-lab 0.9.3 → 0.9.5

package/dist/src/evaluation/runMyDevKitRetrieval.js

@@ -2,7 +2,7 @@ import path from "node:path";
 import { countEstimatedTokens, countTextChars, tokenCountMethod } from "../core/countTokens.js";
 import { runMeasuredCommand } from "../core/runMeasuredCommand.js";
 import { interpretToolVersionOutput } from "./indexSnapshot.js";
-import { buildRetrievalEvidenceFromCommands } from "./retrievalQuality/buildRetrievalEvidence.js";
+import { buildRetrievalEvidenceFromCommands, normalizeRetrievedRepositoryPath } from "./retrievalQuality/buildRetrievalEvidence.js";
 import { buildQueryStrategyEvidenceFromRetrievalEvidence } from "./retrievalQueryStrategyEvidence.js";
 const VERSION_PROBE_TIMEOUT_MS = 30_000;
 function parseJsonIfPossible(text) {
@@ -253,6 +253,39 @@ function selectRetrievalStrategyContextText(strategyId, outputs) {
                     : [outputs.source, outputs.slice, outputs.lookup, outputs.search];
     return ordered.find((text) => text && text.trim().length > 0) ?? "";
 }
+export const BOUNDED_SOURCE_PROTOCOL_ID = "bounded-multi-source-v1";
+export const DEFAULT_BOUNDED_SOURCE_OPTIONS = { maxSourceFiles: 8, searchLimit: 40 };
+export function isBoundedRetrievalResult(result) {
+    return result.boundedCoverage?.protocol === BOUNDED_SOURCE_PROTOCOL_ID;
+}
+/**
+ * Pure, deterministic candidate plan over search results. Only `file`/`symbol` entries (or legacy entries that
+ * carry no `kind`) with a node id and a safe repository-relative path are eligible; edges and other kinds are
+ * ignored. The first occurrence of each normalized file wins, rank order is preserved, and the plan is cut at
+ * `maxSourceFiles`. Never inspects an answer key.
+ */
+export function planBoundedSourceCandidates(results, maxSourceFiles) {
+    const budget = Number.isInteger(maxSourceFiles) && maxSourceFiles > 0 ? maxSourceFiles : 0;
+    const seen = new Set();
+    const eligible = [];
+    let rejectedCount = 0;
+    for (const entry of results) {
+        const kind = entry.kind;
+        if (kind !== undefined && kind !== "file" && kind !== "symbol")
+            continue;
+        const fields = pickCandidateFields(entry);
+        const file = normalizeRetrievedRepositoryPath(fields.file);
+        if (file === null || fields.nodeId === undefined) {
+            rejectedCount += 1;
+            continue;
+        }
+        if (seen.has(file))
+            continue;
+        seen.add(file);
+        eligible.push({ file, nodeId: fields.nodeId });
+    }
+    return { candidates: eligible.slice(0, budget), eligibleFileCount: eligible.length, rejectedCount };
+}
 const STRATEGY_DOWNSTREAM_COMMANDS = {
     "keyword-search": [],
     "symbol-lookup": ["lookup"],
@@ -275,7 +308,15 @@ export async function runMyDevKitRetrievalStrategyFromIndex(options) {
         commandString: options.kitCommand,
         cwd: process.cwd(),
         outDir: commandsDir,
-        extraArgs: ["search", "--index", indexDir, "--query", options.evaluationCase.query, "--json"]
+        extraArgs: [
+            "search",
+            "--index",
+            indexDir,
+            "--query",
+            options.evaluationCase.query,
+            ...(options.boundedSource ? ["--limit", String(options.boundedSource.searchLimit)] : []),
+            "--json"
+        ]
     });
     commands.push(searchCommand);
     if (!searchCommand.ok) {
@@ -324,8 +365,28 @@ export async function runMyDevKitRetrievalStrategyFromIndex(options) {
             warnings.push("No my-dev-kit node id was available after search.");
         }
     }
-    const contextText = selectRetrievalStrategyContextText(options.strategyId, outputs);
-    const filesRead = selectedFile ? [selectedFile] : [];
+    let contextText = selectRetrievalStrategyContextText(options.strategyId, outputs);
+    let filesRead = selectedFile ? [selectedFile] : [];
+    let boundedCoverage;
+    if (options.boundedSource && downstream.includes("source")) {
+        const bounded = await expandBoundedSourceCandidates({
+            bounded: options.boundedSource,
+            candidates,
+            selectedNodeId,
+            selectedSourceOk: commands.some((command) => command.commandId === "source" && command.ok),
+            kitCommand: options.kitCommand,
+            indexDir,
+            commandsDir,
+            commands,
+            warnings
+        });
+        boundedCoverage = bounded.coverage;
+        // Bounded mode: filesRead names only files whose own `source` command succeeded, in rank order.
+        filesRead = bounded.coverage.attempts.filter((attempt) => attempt.sourced).map((attempt) => attempt.file);
+        if (bounded.extraSourceTexts.length > 0) {
+            contextText = [contextText, ...bounded.extraSourceTexts].filter((text) => text.length > 0).join("\n\n");
+        }
+    }
     const retrievalEvidence = buildRetrievalEvidenceFromCommands(commands, { nodeId: selectedNodeId, file: selectedFile });
     return {
         caseId: options.evaluationCase.id,
@@ -342,9 +403,81 @@ export async function runMyDevKitRetrievalStrategyFromIndex(options) {
         selectedSymbol,
         retrievalEvidence,
         queryStrategyEvidence: buildQueryStrategyEvidenceFromRetrievalEvidence(options.strategyId, retrievalEvidence),
+        ...(boundedCoverage ? { boundedCoverage } : {}),
         durationMs: Date.now() - started
     };
 }
+async function expandBoundedSourceCandidates(args) {
+    const plan = planBoundedSourceCandidates(args.candidates, args.bounded.maxSourceFiles);
+    const attempts = [];
+    const extraSourceTexts = [];
+    for (const [index, candidate] of plan.candidates.entries()) {
+        if (candidate.nodeId === args.selectedNodeId) {
+            // The first search candidate already received its `source` command in the historical lifecycle.
+            attempts.push({ ...candidate, sourced: args.selectedSourceOk });
+            continue;
+        }
+        const command = await runMeasuredCommand({
+            commandId: `source-candidate-${index + 1}`,
+            commandString: args.kitCommand,
+            cwd: process.cwd(),
+            outDir: args.commandsDir,
+            extraArgs: ["source", "--index", args.indexDir, "--node", candidate.nodeId, "--max-lines", "160", "--format", "numbered"]
+        });
+        args.commands.push(command);
+        if (!command.ok) {
+            args.warnings.push(`my-dev-kit source command failed for bounded candidate ${index + 1}.`);
+        }
+        else if (command.stdout.length > 0) {
+            extraSourceTexts.push(command.stdout);
+        }
+        attempts.push({ ...candidate, sourced: command.ok });
+    }
+    const resultCount = args.candidates.length;
+    return {
+        coverage: {
+            protocol: BOUNDED_SOURCE_PROTOCOL_ID,
+            maxSourceFiles: args.bounded.maxSourceFiles,
+            searchLimit: args.bounded.searchLimit,
+            searchResultCount: resultCount,
+            searchTruncated: resultCount >= args.bounded.searchLimit,
+            eligibleCandidateFileCount: plan.eligibleFileCount,
+            candidateBudgetExhausted: plan.eligibleFileCount > plan.candidates.length,
+            rejectedCandidateCount: plan.rejectedCount,
+            attempts
+        },
+        extraSourceTexts
+    };
+}
+/**
+ * Opt-in bounded multi-source retrieval for evaluations that must observe more than one file (for example
+ * multi-file required-file evidence). Reuses the shared strategy executor; default top-one retrieval and the
+ * meaning of its `filesRead` are unchanged for every other caller. The result always carries
+ * `boundedCoverage`, including when the search yielded no usable candidate.
+ */
+export async function runMyDevKitBoundedRetrievalFromIndex(options) {
+    const boundedSource = options.boundedSource ?? DEFAULT_BOUNDED_SOURCE_OPTIONS;
+    const result = await runMyDevKitRetrievalStrategyFromIndex({ ...options, strategyId: "combined-graph-guided", boundedSource });
+    if (isBoundedRetrievalResult(result))
+        return result;
+    // Skipped before expansion (search failed or yielded no candidate): still declare the protocol, nothing sourced.
+    const skipped = {
+        ...result,
+        filesRead: [],
+        boundedCoverage: {
+            protocol: BOUNDED_SOURCE_PROTOCOL_ID,
+            maxSourceFiles: boundedSource.maxSourceFiles,
+            searchLimit: boundedSource.searchLimit,
+            searchResultCount: 0,
+            searchTruncated: false,
+            eligibleCandidateFileCount: 0,
+            candidateBudgetExhausted: false,
+            rejectedCandidateCount: 0,
+            attempts: []
+        }
+    };
+    return skipped;
+}
 /**
  * v0.8.0 compatibility entrypoint: the historical search -> lookup -> slice -> source lifecycle,
  * delegating to the `combined-graph-guided` strategy. Never invokes `index`.

package/dist/src/evaluation/syntheticRepository/materialize.js

@@ -20,6 +20,19 @@ export class SyntheticRepositoryMaterializationError extends Error {
     }
 }
 const STAGING_PREFIX = ".synthetic-staging-";
+/** Total attempts at the publication rename (initial try plus two retries); Windows EPERM only. */
+const PUBLISH_RENAME_MAX_ATTEMPTS = 3;
+/** Fixed waits before the second and third attempts (100 ms maximum for one materialization). */
+const PUBLISH_RENAME_RETRY_DELAYS_MS = [25, 75];
+/** Raised when the owned staging directory can no longer be proven to be ours; cleanup must not touch it. */
+class UnownedStagingError extends SyntheticRepositoryMaterializationError {
+}
+function sleepSync(milliseconds) {
+    Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, milliseconds);
+}
+function isTransientWindowsRenameError(error, platform) {
+    return platform === "win32" && typeof error === "object" && error !== null && error.code === "EPERM";
+}
 function withinRealRoot(realRoot, candidate) {
     try {
         relativeWithinRoot(realRoot, realpathSync(candidate));
@@ -63,12 +76,64 @@ function describeExisting(caseDirectory, expectedManifestText) {
         return { reusable: false, reasons: verification.issues };
     return { reusable: true };
 }
+/** Proves, before a rename retry, that the owned staging directory is still ours and the destination is still free. */
+function assertStagingStillPublishable(realRoot, staging, caseDirectory, caseId) {
+    let rootStillVerified = false;
+    try {
+        rootStillVerified = realpathSync(realRoot) === realRoot;
+    }
+    catch {
+        rootStillVerified = false;
+    }
+    if (!rootStillVerified) {
+        throw new UnownedStagingError("path-escape", ["output root no longer resolves to the verified physical root; the staging rename was not retried."]);
+    }
+    let stats;
+    try {
+        stats = lstatSync(staging);
+    }
+    catch {
+        throw new UnownedStagingError("io-failure", ["owned staging directory disappeared before the staging rename retry; nothing was published."]);
+    }
+    if (stats.isSymbolicLink() || !stats.isDirectory() || path.dirname(staging) !== realRoot || !withinRealRoot(realRoot, staging)) {
+        throw new UnownedStagingError("path-escape", ["owned staging directory is no longer a plain directory inside the output root; the staging rename was not retried."]);
+    }
+    if (exists(caseDirectory)) {
+        throw new SyntheticRepositoryMaterializationError("collision", [`output child ${caseId} appeared during materialization; nothing was overwritten.`]);
+    }
+}
+/**
+ * The single atomic publication of a verified staging directory. On Windows an externally induced, brief EPERM from this
+ * one rename is retried up to the fixed budget (3 attempts, 25 ms then 75 ms apart) after the ownership and destination
+ * guards are re-proven; every other error, and every non-Windows platform, gets exactly one attempt. Returns the number
+ * of attempts used.
+ */
+function publishStagingDirectory(options) {
+    for (let attempt = 1;; attempt += 1) {
+        try {
+            options.renameDirectory(options.staging, options.caseDirectory);
+            return attempt;
+        }
+        catch (error) {
+            if (!isTransientWindowsRenameError(error, options.platform))
+                throw error;
+            if (attempt >= PUBLISH_RENAME_MAX_ATTEMPTS) {
+                throw new SyntheticRepositoryMaterializationError("io-failure", [
+                    `final staging directory rename failed with EPERM after ${attempt} attempts (retry budget exhausted); nothing was copied, overwritten or published.`,
+                ]);
+            }
+            options.sleep(PUBLISH_RENAME_RETRY_DELAYS_MS[attempt - 1]);
+            assertStagingStillPublishable(options.realRoot, options.staging, options.caseDirectory, options.caseId);
+        }
+    }
+}
 /**
  * Materializes a frozen Batch 1 plan below `outputRoot/<caseId>/` as `repository/` plus a sibling
  * `synthetic-repository-manifest.json`. The plan is the single semantic source of truth: it is verified, never
  * re-planned or repaired. A deterministic existing directory is reused only when its manifest and every file
  * match the requested generation exactly; any other existing child is a collision. Writes go to an owned staging
- * directory that is verified and then renamed into place; on failure only that staging directory is removed.
+ * directory that is verified and then renamed into place (a brief, externally induced Windows EPERM at that single
+ * rename is retried within a fixed budget, see `publishStagingDirectory`); on failure only that staging directory is removed.
  */
 export function materializeSyntheticRepository(plan, outputRoot, io = {}) {
     try {
@@ -156,15 +221,46 @@ export function materializeSyntheticRepository(plan, outputRoot, io = {}) {
         if (exists(caseDirectory)) {
             throw new SyntheticRepositoryMaterializationError("collision", [`output child ${plan.caseId} appeared during materialization; nothing was overwritten.`]);
         }
-        renameSync(staging, caseDirectory);
+        const attempts = publishStagingDirectory({
+            staging,
+            caseDirectory,
+            caseId: plan.caseId,
+            realRoot,
+            renameDirectory: io.renameDirectory ?? renameSync,
+            sleep: io.sleep ?? sleepSync,
+            platform: io.platform ?? process.platform,
+        });
         staging = undefined;
+        if (attempts > 1) {
+            // The staged tree was verified before the rename; after a retried publication the destination is re-verified too.
+            // It now holds this invocation's output, so a failure here is reported and the destination is left in place.
+            const publishedManifest = path.join(caseDirectory, SYNTHETIC_REPOSITORY_MANIFEST_FILE_NAME);
+            const published = verifySyntheticRepositoryMaterialization({
+                manifestPath: publishedManifest,
+                repositoryRoot: path.join(caseDirectory, SYNTHETIC_REPOSITORY_REPOSITORY_DIRECTORY),
+            });
+            if (!published.ok || readFileSync(publishedManifest, "utf8") !== manifestText) {
+                throw new SyntheticRepositoryMaterializationError("io-failure", [
+                    `published output for ${plan.caseId} failed verification after the staging rename succeeded on attempt ${attempts}; the destination was left in place for inspection.`,
+                    ...published.issues,
+                ]);
+            }
+        }
     }
     catch (error) {
-        if (staging !== undefined)
-            rmSync(staging, { recursive: true, force: true });
-        if (error instanceof SyntheticRepositoryMaterializationError)
-            throw error;
-        throw new SyntheticRepositoryMaterializationError("io-failure", [error instanceof Error ? error.message : String(error)]);
+        let cleanupFailure;
+        if (staging !== undefined && !(error instanceof UnownedStagingError)) {
+            try {
+                rmSync(staging, { recursive: true, force: true });
+            }
+            catch (cleanupError) {
+                cleanupFailure = `cleanup of the owned staging directory failed: ${cleanupError instanceof Error ? cleanupError.message : String(cleanupError)}`;
+            }
+        }
+        const reported = error instanceof SyntheticRepositoryMaterializationError ? error : new SyntheticRepositoryMaterializationError("io-failure", [error instanceof Error ? error.message : String(error)]);
+        if (cleanupFailure === undefined)
+            throw reported;
+        throw new SyntheticRepositoryMaterializationError(reported.code, [...reported.errors, cleanupFailure]);
     }
     return result(false);
 }

package/dist/src/experiments/plugins/incrementalChangeStaleness/comparison.js

@@ -71,6 +71,16 @@ export function compareRequiredFileEvidence(stale, fullRefresh) {
         return "stale-better";
     return "same";
 }
+/**
+ * Reference-adequacy gate over the section 30 rules. Equal evidence only supports "same" when the full-refresh
+ * reference itself observed every required file: two treatments that both lack the required files are equally
+ * insufficient, which is not evidence that the candidate did not regress. That case is `unknown`. A candidate
+ * that observed files the insufficient reference did not is still reported as better; nothing else changes.
+ */
+export function compareRequiredFileEvidenceAgainstReference(candidate, reference) {
+    const relation = compareRequiredFileEvidence(candidate, reference);
+    return relation === "same" && reference.status !== "present" ? "unknown" : relation;
+}
 /**
  * Sections 31-35: the frozen precedence algorithm. Correctness and required-file evidence are the
  * only two inputs; affected-neighborhood evidence and reindexRecommendation are never inputs here.

package/dist/src/experiments/plugins/incrementalChangeStaleness/comparisonV2.js

@@ -1,4 +1,4 @@
-import { compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
+import { compareCorrectness, compareRequiredFileEvidence, compareRequiredFileEvidenceAgainstReference } from "./comparison.js";
 function neutral(relation) {
     if (relation === "stale-worse")
         return "candidate-worse";
@@ -14,6 +14,14 @@ export function compareCandidateCorrectness(candidate, reference) {
 export function compareCandidateRequiredFileEvidence(candidate, reference) {
     return neutral(compareRequiredFileEvidence(candidate, reference));
 }
+/**
+ * Candidate-vs-reference required-file relation with the reference-adequacy gate: equal weakness (both
+ * missing) is `unknown`, not `same`, because an insufficient full-refresh reference cannot support a
+ * no-regression statement. Every other outcome follows `compareCandidateRequiredFileEvidence`.
+ */
+export function compareCandidateRequiredFileEvidenceAgainstReference(candidate, reference) {
+    return neutral(compareRequiredFileEvidenceAgainstReference(candidate, reference));
+}
 /**
  * Frozen precedence. Rule 0 (realization gate) first: a full fallback is never evidence about
  * partial-refresh performance. Then observed regression (either dimension worse), then inconclusive

package/dist/src/experiments/plugins/incrementalChangeStaleness/execution.js

@@ -5,10 +5,10 @@ import { assessAffectedNeighborhood, mapAffectedNeighborhoodSeeds } from "../../
 import { classifyAgentRunOutcome } from "../../../evaluation/classifyAgentRunOutcome.js";
 import { parseAgentAnswer } from "../../../evaluation/parseAgentAnswer.js";
 import { readEvaluationCases } from "../../../evaluation/readEvaluationCases.js";
-import { runMyDevKitRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { isBoundedRetrievalResult, runMyDevKitBoundedRetrievalFromIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
 import { scoreCorrectness } from "../../../evaluation/scoreCorrectness.js";
 import { generatePromptVariants } from "../../../prompts/index.js";
-import { buildRequiredFileEvidence, classifyStaleRisk, compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
+import { buildRequiredFileEvidence, classifyStaleRisk, compareCorrectness, compareRequiredFileEvidenceAgainstReference } from "./comparison.js";
 import { WARM_INDEX_BENCHMARK_CASES_PATH } from "./scenarioCatalog.js";
 // ---------------------------------------------------------------------------
 // v0.6.2 Batch 4 -- treatment retrieval, correctness, and required-file
@@ -134,7 +134,9 @@ export function classifyIncrementalChangeStalenessRetrieval(result) {
     return result.commands.some((command) => !command.ok) ? "failed" : "skipped";
 }
 export async function runIncrementalChangeStalenessTreatmentRetrieval(args) {
-    return runMyDevKitRetrievalFromIndex({
+    // Opt-in bounded multi-source protocol: the staleness experiment must observe multi-file requirements,
+    // which the default first-candidate-only retrieval cannot. Candidates derive from the search output only.
+    return runMyDevKitBoundedRetrievalFromIndex({
         evaluationCase: args.derivedTask,
         kitCommand: args.kitCommand,
         indexDir: args.indexDir,
@@ -142,10 +144,41 @@ export async function runIncrementalChangeStalenessTreatmentRetrieval(args) {
         requireKit: false
     });
 }
+/**
+ * Required-file evidence under the bounded multi-source protocol. "Observed" means a successful `source`
+ * command for that file, never a search match. Fails closed: a protocol that cannot source as many files as
+ * are required, a failed source command, or rejected candidate identities leave unsourced required files
+ * `unknown` instead of `missing`. A `missing` result is relative to the fixed candidate budget and says so.
+ */
+function buildBoundedRequiredFileEvidence(args) {
+    const coverage = args.retrieval.boundedCoverage;
+    const observedFiles = coverage.attempts.filter((attempt) => attempt.sourced).map((attempt) => attempt.file);
+    const unknown = (unknownReason) => buildRequiredFileEvidence({ requiredFiles: args.requiredFiles, observedFiles, observedFilesComplete: false, unknownReason });
+    const required = buildRequiredFileEvidence({ requiredFiles: args.requiredFiles, observedFiles, observedFilesComplete: true });
+    if (required.requiredFiles.length > coverage.maxSourceFiles) {
+        return unknown(`Coverage-limited: the bounded retrieval protocol sources at most ${coverage.maxSourceFiles} candidate file(s) but ${required.requiredFiles.length} are required.`);
+    }
+    if (required.status === "present")
+        return required;
+    const failedAttempts = coverage.attempts.filter((attempt) => !attempt.sourced).length;
+    if (failedAttempts > 0) {
+        return unknown(`${failedAttempts} bounded source command(s) failed; unsourced required files cannot be classified as absent.`);
+    }
+    if (coverage.rejectedCandidateCount > 0) {
+        return unknown(`${coverage.rejectedCandidateCount} search candidate(s) were rejected (unsafe file identity or no node id); unsourced required files cannot be classified as absent.`);
+    }
+    return {
+        ...required,
+        reason: `Required file(s) were not sourced within the bounded retrieval protocol (${observedFiles.length} of up to ${coverage.maxSourceFiles} candidate file(s) sourced; ` +
+            `${coverage.eligibleCandidateFileCount} eligible; search truncated: ${coverage.searchTruncated ? "yes" : "no"}; candidate budget exhausted: ${coverage.candidateBudgetExhausted ? "yes" : "no"}). ` +
+            "This is not an exhaustive source-set assessment."
+    };
+}
 /**
  * Section 27: a `failed` retrieval status means the actual-files-read set cannot be trusted as
  * complete (unknown); `completed` and `skipped` (search ran and legitimately found no candidate)
- * both provide a complete, trustworthy files-read set, possibly empty.
+ * both provide a complete, trustworthy files-read set, possibly empty. A result produced by the opt-in
+ * bounded protocol is evaluated against its own declared coverage instead.
  */
 export function buildIncrementalChangeStalenessRequiredFileEvidence(args) {
     if (!args.retrieval) {
@@ -157,6 +190,9 @@ export function buildIncrementalChangeStalenessRequiredFileEvidence(args) {
         });
     }
     const status = classifyIncrementalChangeStalenessRetrieval(args.retrieval);
+    if (status !== "failed" && isBoundedRetrievalResult(args.retrieval)) {
+        return buildBoundedRequiredFileEvidence({ requiredFiles: args.requiredFiles, retrieval: args.retrieval });
+    }
     return buildRequiredFileEvidence({
         requiredFiles: args.requiredFiles,
         observedFiles: args.retrieval.filesRead,
@@ -412,7 +448,7 @@ export async function executeIncrementalChangeStalenessScenario(options) {
         deps
     });
     const correctnessRelation = compareCorrectness(correctnessComparable(stale.fakeAgent), correctnessComparable(fullRefresh.fakeAgent));
-    const requiredFileEvidenceRelation = compareRequiredFileEvidence(stale.requiredFileEvidence, fullRefresh.requiredFileEvidence);
+    const requiredFileEvidenceRelation = compareRequiredFileEvidenceAgainstReference(stale.requiredFileEvidence, fullRefresh.requiredFileEvidence);
     const classification = classifyStaleRisk(correctnessRelation, requiredFileEvidenceRelation);
     return {
         ...scenarioBase,

package/dist/src/experiments/plugins/incrementalChangeStaleness/executionV2.js

@@ -1,7 +1,7 @@
 import path from "node:path";
 import { resolveWithinRoot } from "../../../core/pathSafety.js";
-import { classifyStaleRisk, compareCorrectness, compareRequiredFileEvidence } from "./comparison.js";
-import { compareCandidateCorrectness, compareCandidateRequiredFileEvidence, classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
+import { classifyStaleRisk, compareCorrectness, compareRequiredFileEvidenceAgainstReference } from "./comparison.js";
+import { compareCandidateCorrectness, compareCandidateRequiredFileEvidenceAgainstReference, classifyPartialRefreshAgainstFull } from "./comparisonV2.js";
 import { INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_IDS, INCREMENTAL_CHANGE_STALENESS_V2_TREATMENT_INTENTS } from "./disposableTarget.js";
 import { assessIncrementalChangeStalenessAffectedNeighborhood, checkIncrementalChangeStalenessAffectedNeighborhoodSymmetry, correctnessComparable, defaultIncrementalChangeStalenessExecutionDeps, executeTreatmentEvaluation, INCONCLUSIVE_NO_EXECUTION, resolveIncrementalChangeStalenessScenarioTask } from "./execution.js";
 /**
@@ -153,7 +153,7 @@ export function buildReferenceComparisons(treatments) {
         throw new Error("Reference comparisons require the stale-index and full-refresh treatment executions.");
     // Historical v0.6.2 semantics, unchanged.
     const staleCorrectness = compareCorrectness(correctnessComparable(stale.fakeAgent), correctnessComparable(reference.fakeAgent));
-    const staleRequired = compareRequiredFileEvidence(stale.requiredFileEvidence, reference.requiredFileEvidence);
+    const staleRequired = compareRequiredFileEvidenceAgainstReference(stale.requiredFileEvidence, reference.requiredFileEvidence);
     const staleRisk = classifyStaleRisk(staleCorrectness, staleRequired);
     const comparisons = [
         {
@@ -174,7 +174,7 @@ export function buildReferenceComparisons(treatments) {
             throw new Error(`${candidateTreatmentId} has no incremental refresh execution to compare.`);
         }
         const correctnessRelation = compareCandidateCorrectness(correctnessComparable(candidate.fakeAgent), correctnessComparable(reference.fakeAgent));
-        const requiredFileEvidenceRelation = compareCandidateRequiredFileEvidence(candidate.requiredFileEvidence, reference.requiredFileEvidence);
+        const requiredFileEvidenceRelation = compareCandidateRequiredFileEvidenceAgainstReference(candidate.requiredFileEvidence, reference.requiredFileEvidence);
         const classified = classifyPartialRefreshAgainstFull({
             realization: candidate.refreshExecution.realization,
             correctnessRelation,

package/dist/src/report/experiments/buildIncrementalChangeStalenessReportV2.js

@@ -14,6 +14,10 @@ export const LIMITATIONS_V2 = [
     "\"no-observed-regression-relative-to-full\" means only that the executed scenario did not show a regression under the frozen correctness and required-file evidence dimensions. It does not prove general equivalence or safety, and does not show that full refresh is unnecessary.",
     "Required-file evidence is a bounded answer-key file-presence check, not retrieval precision/recall.",
     "Correctness remains deterministic fake-agent/harness evidence in v0.6.3.",
+    "Fake-agent correctness is a synthetic score computed from the answer key (the fake agent deliberately reports only two expected facts, so scenarios requiring three or more facts cannot pass). It is not measured my-dev-kit answer correctness and does not discriminate between treatments; a failed synthetic flag is not a my-dev-kit regression and a passing one would not be evidence of Kit correctness.",
+    "Files Read and Observed Files count a file only when the bounded retrieval protocol ran a successful source command for it; a file merely surfaced by search, lookup or slice was not read. Artifacts recorded before v0.9.5 name only the single selected search candidate.",
+    "A Missing required-file result is relative to the fixed bounded candidate budget and is not an exhaustive source-set assessment; evidence the protocol could not observe (coverage limits, failed source commands, rejected candidates) is reported as unknown. Equal required-file evidence supports a same relation only when the full-refresh reference itself observed every required file; two equally insufficient treatments are reported as unknown/inconclusive.",
+    "Command log paths in persisted retrieval telemetry name run-owned files that the mandatory run cleanup removes after the artifact is written; they are not retained and must not be relied on as links.",
     "Lab affected-neighborhood evidence (changed baseline neighborhood relative to the task) is distinct from upstream refresh execution evidence (what my-dev-kit actually refreshed).",
     "\"reindexRecommendation\" remains observational evidence; it did not select or trigger any treatment.",
     "\"changedSymbolCount\" does not mean source-level symbol-diff proof; it retains the released v0.6.1 baseline-symbol-identity definition.",

package/dist/src/report/experiments/incrementalChangeStalenessReportRowsV2.js

@@ -27,7 +27,7 @@ export function treatmentBlock(scenario, treatmentId, label) {
         ["Estimated context tokens", retrieval.status === "not-run" ? "unavailable" : String(retrieval.totalEstimatedTokens)],
         ["Retrieval duration (ms)", retrieval.status === "not-run" ? "unavailable" : String(retrieval.durationMs)],
         ["Fake-agent status", display(treatment.fakeAgent?.status, "not run")],
-        ["Correctness score", correctness && correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable"],
+        ["Synthetic fake-agent correctness score", correctness && correctness.available && correctness.score !== null ? String(correctness.score) : "unavailable"],
         ["Required-file evidence", treatment.requiredFileEvidence.status],
         ["Required-file reason", display(treatment.requiredFileEvidence.reason, "not applicable")],
         ["Affected-neighborhood relationship", treatment.affectedNeighborhood.relationship],