@dailephd/my-dev-kit-lab 0.8.1 → 0.8.2

package/dist/src/experiments/plugins/contextPackGeneration/execution.js

@@ -0,0 +1,361 @@
+import { readFile } from "node:fs/promises";
+import path from "node:path";
+import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { runMeasuredCommand } from "../../../core/runMeasuredCommand.js";
+import { validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { buildMyDevKitIndex, probeMyDevKitVersion } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { groupWarmIndexCases, taskOutputSegment } from "../warmIndexReuse/selection.js";
+import { buildContextPack } from "./buildContextPack.js";
+import { CONTEXT_PACK_GENERATION_ERROR_TEXT } from "./executionTypes.js";
+import { buildContextPackIdentityEvidence, buildRawFullFileIdentityEvidence } from "./identityEvidence.js";
+import { CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "./metadata.js";
+import { contextPackArtifactRelativePath } from "./packArtifact.js";
+import { deriveSymbolSourceRange, findIndexedSymbolLine, interpretSymbolIndex, parseLookup, parseSearchHits, parseSlice, parseSource } from "./packEvidence.js";
+import { GRAPH_DEPTH, MAX_SOURCE_LINES_PER_SLICE, MAX_SOURCE_SLICES, SEARCH_RESULT_LIMIT, selectRelevantSymbols, selectSeedNodes } from "./packSelectionPolicy.js";
+const defaultDependencies = {
+    buildIndex: buildMyDevKitIndex,
+    runCommand: runMeasuredCommand,
+    runRawBaseline: runRawFullFileBaseline,
+    readSymbolIndex: async (indexDir) => JSON.parse(await readFile(path.join(indexDir, "symbol-index.json"), "utf8")),
+    probeVersion: probeMyDevKitVersion
+};
+const pad = (index) => String(index + 1).padStart(2, "0");
+function failedTreatment(treatmentId, code, steps = []) {
+    return {
+        treatmentId,
+        status: "failed",
+        availability: null,
+        availabilityReason: null,
+        size: null,
+        identityEvidence: null,
+        includedFiles: [],
+        steps,
+        sections: null,
+        evidenceNotes: [],
+        packArtifactPath: null,
+        errors: [{ code, message: CONTEXT_PACK_GENERATION_ERROR_TEXT[code] }]
+    };
+}
+function failedCase(evaluationCase, code) {
+    return {
+        evidence: {
+            caseId: evaluationCase.id,
+            caseName: evaluationCase.title,
+            benchmarkProject: evaluationCase.benchmarkProject,
+            taskLocality: evaluationCase.taskLocality ?? null,
+            treatments: CONTEXT_PACK_GENERATION_TREATMENT_IDS.map((treatmentId) => failedTreatment(treatmentId, code))
+        },
+        pack: null
+    };
+}
+const statusOf = (availability) => (availability === "available" ? "completed" : availability === "partial" ? "partial" : "failed");
+async function executeRawTreatment(context) {
+    try {
+        const baseline = await context.dependencies.runRawBaseline(context.evaluationCase);
+        const symbolIndex = context.symbolIndex;
+        const availability = symbolIndex ? "available" : "partial";
+        const reason = symbolIndex ? null : "symbol-index-unavailable";
+        return {
+            treatmentId: "raw-full-file",
+            status: statusOf(availability),
+            availability,
+            availabilityReason: reason,
+            size: { totalChars: baseline.totalChars, totalEstimatedTokens: baseline.totalEstimatedTokens, tokenCountMethod: baseline.tokenCountMethod },
+            identityEvidence: buildRawFullFileIdentityEvidence({
+                includedFiles: baseline.filesIncluded,
+                indexedFiles: symbolIndex?.files ?? [],
+                availability,
+                reason
+            }),
+            includedFiles: [...baseline.filesIncluded],
+            steps: [],
+            sections: null,
+            evidenceNotes: [],
+            packArtifactPath: null,
+            errors: []
+        };
+    }
+    catch {
+        return failedTreatment("raw-full-file", "raw-baseline-failed");
+    }
+}
+async function executePackTreatment(context) {
+    const { evaluationCase, indexDir, symbolIndex } = context;
+    const steps = [];
+    const run = (commandId, extraArgs) => context.dependencies.runCommand({ commandId, commandString: context.kitCommand, cwd: process.cwd(), outDir: context.commandsDir, extraArgs });
+    const fail = (code) => ({ treatment: failedTreatment("context-pack", code, steps), pack: null });
+    // 1. One search, capped, upstream rank preserved.
+    let hits;
+    try {
+        const search = await run("search", ["search", "--index", indexDir, "--query", evaluationCase.query, "--limit", String(SEARCH_RESULT_LIMIT), "--json"]);
+        const parsed = search.ok ? parseSearchHits(search.stdout, SEARCH_RESULT_LIMIT) : null;
+        if (!search.ok || !parsed || !parsed.ok) {
+            steps.push({ kind: "search", nodeId: null, sourceMode: null, succeeded: false, reason: search.ok ? "malformed-output" : "command-failed" });
+            return fail("retrieval-failed");
+        }
+        steps.push({ kind: "search", nodeId: null, sourceMode: null, succeeded: true, reason: null });
+        hits = parsed.value;
+    }
+    catch {
+        steps.push({ kind: "search", nodeId: null, sourceMode: null, succeeded: false, reason: "command-failed" });
+        return fail("retrieval-failed");
+    }
+    const fileCandidates = [];
+    const symbolCandidates = [];
+    const testCandidates = [];
+    const callCandidates = [];
+    const lineByNodeId = new Map();
+    const extraNotes = [];
+    for (const hit of hits) {
+        if (hit.path !== null) {
+            fileCandidates.push({ path: hit.path, origin: "search", rank: hit.rank });
+            testCandidates.push({ path: hit.path, origin: "search-candidate", rank: hit.rank });
+        }
+        if (hit.kind === "symbol") {
+            symbolCandidates.push({ name: hit.label ?? hit.nodeId.slice(hit.nodeId.indexOf("#") + 1), nodeId: hit.nodeId, file: hit.path, line: null, origin: "search", rank: hit.rank });
+        }
+    }
+    // 2. Seeds in upstream order; lookup then depth-1 slice per seed.
+    const seeds = selectSeedNodes(hits.map((hit) => ({ nodeId: hit.nodeId, origin: "search", rank: hit.rank })));
+    extraNotes.push(...seeds.notes);
+    let graphIncomplete = false;
+    for (const [index, seed] of seeds.items.entries()) {
+        try {
+            const lookup = await run(`lookup-${pad(index)}`, ["lookup", "--index", indexDir, "--node", seed.nodeId, "--depth", String(GRAPH_DEPTH), "--json"]);
+            const parsed = lookup.ok ? parseLookup(lookup.stdout) : null;
+            if (parsed && parsed.ok) {
+                steps.push({ kind: "lookup", nodeId: seed.nodeId, sourceMode: null, succeeded: true, reason: null });
+                const node = parsed.value.node;
+                if (node.path !== null)
+                    fileCandidates.push({ path: node.path, origin: "lookup", rank: seed.rank });
+                if (node.kind === "symbol" && node.symbolName !== null) {
+                    symbolCandidates.push({ name: node.symbolName, nodeId: node.id, file: node.path, line: node.line, origin: "lookup", rank: seed.rank });
+                    if (node.line !== null)
+                        lineByNodeId.set(node.id, node.line);
+                }
+            }
+            else {
+                graphIncomplete = true;
+                steps.push({ kind: "lookup", nodeId: seed.nodeId, sourceMode: null, succeeded: false, reason: lookup.ok ? "malformed-output" : "command-failed" });
+            }
+        }
+        catch {
+            graphIncomplete = true;
+            steps.push({ kind: "lookup", nodeId: seed.nodeId, sourceMode: null, succeeded: false, reason: "command-failed" });
+        }
+        try {
+            const slice = await run(`slice-${pad(index)}`, ["slice", "--index", indexDir, "--node", seed.nodeId, "--depth", String(GRAPH_DEPTH), "--direction", "both", "--json"]);
+            const parsed = slice.ok ? parseSlice(slice.stdout) : null;
+            if (parsed && parsed.ok) {
+                steps.push({ kind: "slice", nodeId: seed.nodeId, sourceMode: null, succeeded: true, reason: null });
+                for (const node of parsed.value.nodes) {
+                    if (node.path !== null) {
+                        fileCandidates.push({ path: node.path, origin: "graph", rank: seed.rank });
+                        testCandidates.push({ path: node.path, origin: "graph-neighbor", rank: seed.rank });
+                    }
+                    if (node.kind === "symbol" && node.symbolName !== null) {
+                        symbolCandidates.push({ name: node.symbolName, nodeId: node.id, file: node.path, line: node.line, origin: "graph", rank: seed.rank });
+                        if (node.line !== null && !lineByNodeId.has(node.id))
+                            lineByNodeId.set(node.id, node.line);
+                    }
+                }
+                for (const edge of parsed.value.edges)
+                    callCandidates.push({ fromNodeId: edge.source, toNodeId: edge.target, kind: edge.kind });
+            }
+            else {
+                graphIncomplete = true;
+                steps.push({ kind: "slice", nodeId: seed.nodeId, sourceMode: null, succeeded: false, reason: slice.ok ? "malformed-output" : "command-failed" });
+            }
+        }
+        catch {
+            graphIncomplete = true;
+            steps.push({ kind: "slice", nodeId: seed.nodeId, sourceMode: null, succeeded: false, reason: "command-failed" });
+        }
+    }
+    // 3. Bounded source for the selected symbols, in selection order, at most MAX_SOURCE_SLICES commands.
+    const selectedSymbols = selectRelevantSymbols(symbolCandidates).items;
+    const sliceCandidates = [];
+    let sourceIncomplete = false;
+    let sourceCount = 0;
+    for (const symbol of selectedSymbols) {
+        if (sourceCount >= MAX_SOURCE_SLICES)
+            break;
+        if (symbol.file === null)
+            continue;
+        const line = (symbol.nodeId !== null ? lineByNodeId.get(symbol.nodeId) : undefined) ?? (symbolIndex ? findIndexedSymbolLine(symbolIndex, symbol.file, symbol.name) : null);
+        const range = symbolIndex && line !== null ? deriveSymbolSourceRange(symbolIndex, symbol.file, line) : null;
+        if (!range && symbol.nodeId === null)
+            continue;
+        sourceCount += 1;
+        const sourceMode = range ? "index-range" : "node";
+        const nodeId = symbol.nodeId;
+        try {
+            const args = range
+                ? ["source", "--index", indexDir, "--file", symbol.file, "--start", String(range.startLine), "--end", String(range.endLine), "--json"]
+                : ["source", "--index", indexDir, "--node", nodeId, "--max-lines", String(MAX_SOURCE_LINES_PER_SLICE), "--json"];
+            const source = await run(`source-${pad(sourceCount - 1)}`, args);
+            const parsed = source.ok ? parseSource(source.stdout) : null;
+            if (!parsed || !parsed.ok) {
+                sourceIncomplete = true;
+                steps.push({ kind: "source", nodeId, sourceMode, succeeded: false, reason: source.ok ? "malformed-output" : "command-failed" });
+                continue;
+            }
+            const value = parsed.value;
+            steps.push({ kind: "source", nodeId, sourceMode, succeeded: true, reason: value.boundaryKnown ? null : "symbol-end-unknown" });
+            if (range?.capped)
+                extraNotes.push("source-slice-cap-reached");
+            sliceCandidates.push({
+                file: symbol.file,
+                nodeId,
+                symbolName: symbol.name,
+                startLine: value.startLine,
+                endLine: value.endLine,
+                text: value.content,
+                boundaryKnown: value.boundaryKnown,
+                truncated: range?.capped === true,
+                continuationAvailable: range ? range.capped : !value.eof,
+                rank: symbol.rank
+            });
+        }
+        catch {
+            sourceIncomplete = true;
+            steps.push({ kind: "source", nodeId, sourceMode, succeeded: false, reason: "command-failed" });
+        }
+    }
+    const boundaryUnknown = sliceCandidates.some((candidate) => !candidate.boundaryKnown);
+    // 4. Deterministic section availability decided here (execution), then pure composition.
+    const available = { availability: "available", reason: null };
+    const graphSection = graphIncomplete ? { availability: "partial", reason: "graph-step-incomplete" } : available;
+    const sourceSection = sourceIncomplete
+        ? { availability: "partial", reason: "source-step-incomplete" }
+        : boundaryUnknown
+            ? { availability: "partial", reason: "symbol-end-unknown" }
+            : available;
+    const sectionAvailability = { files: graphSection, symbols: graphSection, sourceSlices: sourceSection, callRelationships: graphSection, tests: graphSection, evidenceNotes: available };
+    const packAvailability = graphIncomplete || sourceIncomplete || boundaryUnknown ? "partial" : "available";
+    const packReason = graphIncomplete ? "graph-step-incomplete" : sourceIncomplete ? "source-step-incomplete" : boundaryUnknown ? "symbol-end-unknown" : null;
+    let pack;
+    try {
+        pack = buildContextPack({
+            caseId: evaluationCase.id,
+            benchmarkProject: evaluationCase.benchmarkProject,
+            taskLocality: evaluationCase.taskLocality ?? null,
+            task: { title: evaluationCase.title, summary: evaluationCase.query },
+            myDevKitVersion: context.myDevKitVersion,
+            fileCandidates,
+            symbolCandidates,
+            sourceSliceCandidates: sliceCandidates,
+            callCandidates,
+            testCandidates,
+            evidenceNotes: extraNotes,
+            sectionAvailability,
+            availability: packAvailability,
+            reason: packReason
+        });
+    }
+    catch {
+        return fail("pack-construction-failed");
+    }
+    // Slice-contained symbols need the symbol index; without it the identity evidence is explicitly partial, never guessed.
+    const identitySource = symbolIndex || pack.availability !== "available" ? pack : { ...pack, availability: "partial", reason: "symbol-index-unavailable" };
+    return {
+        pack,
+        treatment: {
+            treatmentId: "context-pack",
+            status: statusOf(identitySource.availability),
+            availability: identitySource.availability,
+            availabilityReason: identitySource.reason,
+            size: pack.size,
+            identityEvidence: buildContextPackIdentityEvidence(identitySource, symbolIndex?.files ?? []),
+            includedFiles: pack.files.map((file) => file.path),
+            steps,
+            sections: pack.sections.map((section) => ({ ...section })),
+            evidenceNotes: [...pack.evidenceNotes],
+            packArtifactPath: contextPackArtifactRelativePath(evaluationCase.id),
+            errors: []
+        }
+    };
+}
+/**
+ * Matched two-treatment execution (raw-full-file, context-pack). One base index per benchmark project, shared by both
+ * treatments of every case in it; cases run strictly sequentially in corpus order. Builds no scientific analysis and
+ * writes nothing durable: it returns bounded evidence plus the in-memory packs.
+ */
+export async function executeContextPackGeneration(options) {
+    const dependencies = { ...defaultDependencies, ...options.dependencies };
+    const results = new Map();
+    const validCases = [];
+    for (const evaluationCase of options.cases) {
+        if (validateRetrievalPrecisionRecallCase(evaluationCase).length > 0)
+            results.set(evaluationCase.id, failedCase(evaluationCase, "ground-truth-invalid"));
+        else
+            validCases.push(evaluationCase);
+    }
+    let myDevKitVersion = null;
+    for (const group of groupWarmIndexCases(validCases)) {
+        if (group.structuralErrors.length > 0) {
+            for (const evaluationCase of group.cases)
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-group-inconsistent"));
+            continue;
+        }
+        const baseIndexDir = resolveWithinRoot(options.outputRoot, path.join("indexes", group.projectSegment, "base"));
+        const projectCommandsDir = resolveWithinRoot(options.outputRoot, path.join("commands", group.projectSegment));
+        let indexReady = false;
+        try {
+            const build = await dependencies.buildIndex({
+                target: { absoluteTargetRoot: group.targetRoot, sourceRoots: [...group.sourceRoots] },
+                kitCommand: options.kitCommand,
+                indexDir: baseIndexDir,
+                commandsDir: path.join(projectCommandsDir, "index"),
+                requireKit: false,
+                // Context packs carry call relationships, which upstream emits only for call-graph indexes.
+                callGraph: true
+            });
+            indexReady = build.ok;
+        }
+        catch {
+            indexReady = false;
+        }
+        if (!indexReady) {
+            for (const evaluationCase of group.cases)
+                results.set(evaluationCase.id, failedCase(evaluationCase, "project-index-failed"));
+            continue;
+        }
+        if (myDevKitVersion === null) {
+            const tool = await dependencies.probeVersion({ kitCommand: options.kitCommand, commandsDir: resolveWithinRoot(options.outputRoot, path.join("commands", "version")) });
+            myDevKitVersion = tool.version ?? "unavailable";
+        }
+        let symbolIndex = null;
+        try {
+            symbolIndex = interpretSymbolIndex(await dependencies.readSymbolIndex(baseIndexDir));
+        }
+        catch {
+            symbolIndex = null;
+        }
+        for (const evaluationCase of group.cases) {
+            const context = {
+                evaluationCase,
+                kitCommand: options.kitCommand,
+                indexDir: baseIndexDir,
+                commandsDir: resolveWithinRoot(projectCommandsDir, taskOutputSegment(evaluationCase.id)),
+                symbolIndex,
+                myDevKitVersion,
+                dependencies
+            };
+            // Strictly sequential, fixed treatment order.
+            const raw = await executeRawTreatment(context);
+            const pack = await executePackTreatment(context);
+            const evidence = {
+                caseId: evaluationCase.id,
+                caseName: evaluationCase.title,
+                benchmarkProject: evaluationCase.benchmarkProject,
+                taskLocality: evaluationCase.taskLocality ?? null,
+                treatments: [raw, pack.treatment]
+            };
+            results.set(evaluationCase.id, { evidence, pack: pack.pack });
+        }
+    }
+    // Selection order is the corpus order after filtering, regardless of how projects were grouped.
+    return options.cases.map((evaluationCase) => results.get(evaluationCase.id));
+}

package/dist/src/experiments/plugins/contextPackGeneration/executionArtifact.js

@@ -0,0 +1,17 @@
+import { CONTEXT_PACK_GENERATION_TREATMENT_IDS } from "./metadata.js";
+import { CONTEXT_PACK_SELECTION_POLICY_ID } from "./packSelectionPolicy.js";
+export const CONTEXT_PACK_GENERATION_EXECUTION_ARTIFACT_FILE = "context-pack-generation-execution.json";
+export const CONTEXT_PACK_GENERATION_EXECUTION_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-generation-execution-v1";
+export function buildContextPackGenerationExecutionArtifact(args) {
+    return {
+        schemaVersion: CONTEXT_PACK_GENERATION_EXECUTION_SCHEMA_VERSION,
+        runId: args.runId,
+        pluginId: args.pluginId,
+        pluginSchemaVersion: args.pluginSchemaVersion,
+        startedAt: args.startedAt,
+        completedAt: args.completedAt,
+        treatmentOrder: [...CONTEXT_PACK_GENERATION_TREATMENT_IDS],
+        selectionPolicyId: CONTEXT_PACK_SELECTION_POLICY_ID,
+        cases: structuredClone(args.cases)
+    };
+}

package/dist/src/experiments/plugins/contextPackGeneration/executionTypes.js

@@ -0,0 +1,11 @@
+/** Plugin-owned text per code; caught error messages and process stderr are never copied into durable evidence. */
+export const CONTEXT_PACK_GENERATION_ERROR_TEXT = {
+    "ground-truth-invalid": "The case ground truth was invalid.",
+    "project-group-inconsistent": "The case source configuration was inconsistent.",
+    "project-index-failed": "The my-dev-kit index could not be prepared.",
+    "raw-baseline-failed": "The raw full-file baseline could not be built.",
+    "retrieval-failed": "The context-pack retrieval did not complete.",
+    "pack-construction-failed": "The context pack could not be constructed."
+};
+/** Fixed schema of the external-local identity-redaction marker. */
+export const CONTEXT_PACK_IDENTITY_REDACTION_SCHEMA_VERSION = "my-dev-kit-lab-context-pack-identity-redaction-v1";

package/dist/src/experiments/plugins/contextPackGeneration/identityEvidence.js

@@ -0,0 +1,71 @@
+import { compareCodeUnits } from "../../../evaluation/indexSnapshot.js";
+import { normalizeRetrievedRepositoryPath } from "../../../evaluation/retrievalQuality/buildRetrievalEvidence.js";
+function canonicalSymbols(identities) {
+    const unique = new Map();
+    for (const identity of identities)
+        unique.set(`${identity.file ?? ""}\0${identity.name}`, identity);
+    return [...unique.values()].sort((a, b) => compareCodeUnits(a.file ?? "", b.file ?? "") || compareCodeUnits(a.name, b.name));
+}
+/** Raw-full-file rule: every indexed symbol defined in a file the raw baseline actually included. */
+export function collectRawFullFileSymbols(indexedFiles, includedFiles) {
+    const included = new Set();
+    for (const file of includedFiles) {
+        const normalized = normalizeRetrievedRepositoryPath(file);
+        if (normalized !== null)
+            included.add(normalized);
+    }
+    const found = [];
+    for (const entry of indexedFiles) {
+        const path = normalizeRetrievedRepositoryPath(entry.path);
+        if (path === null || !included.has(path))
+            continue;
+        for (const symbol of entry.symbols)
+            found.push({ name: symbol.name, file: path });
+    }
+    return canonicalSymbols(found);
+}
+/**
+ * Context-pack rule: the union of the explicitly selected symbols and the indexed symbols whose definition line falls
+ * inside a selected (post-cap) source slice range of the same file.
+ */
+export function collectContextPackSymbols(pack, indexedFiles) {
+    const found = pack.symbols.map((symbol) => (symbol.file === null ? { name: symbol.name } : { name: symbol.name, file: symbol.file }));
+    for (const entry of indexedFiles) {
+        const path = normalizeRetrievedRepositoryPath(entry.path);
+        if (path === null)
+            continue;
+        const slices = pack.sourceSlices.filter((slice) => slice.file === path);
+        if (slices.length === 0)
+            continue;
+        for (const symbol of entry.symbols) {
+            if (slices.some((slice) => symbol.location.line >= slice.startLine && symbol.location.line <= slice.endLine)) {
+                found.push({ name: symbol.name, file: path });
+            }
+        }
+    }
+    return canonicalSymbols(found);
+}
+const withReason = (input) => input.reason === null ? { availability: input.availability } : { availability: input.availability, availabilityReason: input.reason };
+/** Context-pack identity evidence for the existing retrieval-quality calculator. Tests stay descriptive and are excluded. */
+export function buildContextPackIdentityEvidence(pack, indexedFiles) {
+    const files = new Set([...pack.files.map((file) => file.path), ...pack.sourceSlices.map((slice) => slice.file)]);
+    return {
+        ...withReason({ availability: pack.availability, reason: pack.reason }),
+        files: [...files].sort(compareCodeUnits),
+        symbols: collectContextPackSymbols(pack, indexedFiles)
+    };
+}
+/** Raw-full-file identity evidence; availability is supplied by the caller, never inferred from an empty list. */
+export function buildRawFullFileIdentityEvidence(input) {
+    const files = new Set();
+    for (const file of input.includedFiles) {
+        const normalized = normalizeRetrievedRepositoryPath(file);
+        if (normalized !== null)
+            files.add(normalized);
+    }
+    return {
+        ...withReason(input),
+        files: [...files].sort(compareCodeUnits),
+        symbols: collectRawFullFileSymbols(input.indexedFiles, input.includedFiles)
+    };
+}

package/dist/src/experiments/plugins/contextPackGeneration/index.js

@@ -0,0 +1,19 @@
+export * from "./analysis.js";
+export * from "./analysisArtifact.js";
+export * from "./analysisTypes.js";
+export * from "./buildContextPack.js";
+export * from "./config.js";
+export * from "./execution.js";
+export * from "./executionArtifact.js";
+export * from "./executionTypes.js";
+export * from "./identityEvidence.js";
+export * from "./localSubjectExecution.js";
+export * from "./localSubjectPrivacy.js";
+export * from "./metadata.js";
+export * from "./metrics.js";
+export * from "./packArtifact.js";
+export * from "./packEvidence.js";
+export * from "./packSelectionPolicy.js";
+export * from "./plugin.js";
+export * from "./renderContextPack.js";
+export * from "./types.js";

package/dist/src/experiments/plugins/contextPackGeneration/localSubjectExecution.js

@@ -0,0 +1,146 @@
+import path from "node:path";
+import { validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { buildMyDevKitIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { deriveGuidedIndexExclusions } from "../contextWindowScaling/localSubjectExclusions.js";
+import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch } from "../contextWindowScaling/localSubjectScratch.js";
+import { executeContextPackGeneration } from "./execution.js";
+/** Repository path carried by a `file:<path>` or `symbol:<path>#<name>` node ID; null when the ID has no such shape. */
+export function repositoryPathOfNodeId(nodeId) {
+    const match = /^(?:file|symbol):([^#]+)(?:#.*)?$/.exec(nodeId);
+    return match ? match[1] : null;
+}
+/** Every real repository path identity one case result exposes through either treatment or the in-memory pack. */
+export function collectContextPackFileIdentities(result) {
+    const identities = new Set();
+    for (const treatment of result.evidence.treatments) {
+        for (const file of treatment.includedFiles)
+            identities.add(file);
+        for (const file of treatment.identityEvidence?.files ?? [])
+            identities.add(file);
+        for (const symbol of treatment.identityEvidence?.symbols ?? []) {
+            if (symbol.file !== undefined)
+                identities.add(symbol.file);
+        }
+    }
+    const pack = result.pack;
+    if (pack !== null) {
+        for (const file of pack.files)
+            identities.add(file.path);
+        for (const symbol of pack.symbols) {
+            if (symbol.file !== null)
+                identities.add(symbol.file);
+        }
+        for (const slice of pack.sourceSlices)
+            identities.add(slice.file);
+        for (const test of pack.tests)
+            identities.add(test.path);
+        for (const relationship of pack.callRelationships) {
+            for (const endpoint of [relationship.fromNodeId, relationship.toNodeId]) {
+                const endpointPath = repositoryPathOfNodeId(endpoint);
+                if (endpointPath !== null)
+                    identities.add(endpointPath);
+            }
+        }
+    }
+    return identities;
+}
+/** Counts distinct file identities outside the eligible universe. Only the count leaves this function. */
+export function countContextPackFilesOutsideEligibleUniverse(result, eligibleFiles) {
+    return [...collectContextPackFileIdentities(result)].filter((identity) => !eligibleFiles.has(identity)).length;
+}
+const FATAL_CASE_ERROR_CODES = new Set(["ground-truth-invalid", "project-group-inconsistent", "project-index-failed"]);
+/**
+ * Runs a loaded local subject through the two-treatment context-pack experiment. One private base index per configured
+ * case (each case keeps its own exact source roots), a private scratch outside the target, guided exclusions so the index
+ * never sees ineligible files, the raw baseline restricted to the eligible universe, and pre/post target-immutability
+ * enforcement. It performs NO durable writes; success is returned only when every safety gate passed. A mutated target
+ * is never reverted. Issue messages are fixed text and counts.
+ */
+export async function executeLocalRepositorySubjectContextPackGeneration(args) {
+    const { subject } = args;
+    await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
+    const guidedExclusions = deriveGuidedIndexExclusions({
+        eligibleFiles: subject.eligibleFiles,
+        excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles]
+    });
+    if (subject.evaluationCases.some((evaluationCase) => validateRetrievalPrecisionRecallCase(evaluationCase).length > 0)) {
+        throw new LocalSubjectExecutionError([
+            { code: "GROUND_TRUTH_INVALID", message: "context-pack ground truth is incomplete for configured case(s)." }
+        ]);
+    }
+    const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
+    const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
+    const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!before.ok) {
+        throw new LocalSubjectExecutionError([{ code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` }]);
+    }
+    const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
+    const issues = [];
+    const results = [];
+    const eligible = new Set(subject.eligibleFiles);
+    const baseBuildIndex = args.dependencies?.buildIndex;
+    const baseRawBaseline = args.dependencies?.runRawBaseline ?? runRawFullFileBaseline;
+    for (const [position, evaluationCase] of subject.evaluationCases.entries()) {
+        try {
+            // The experiment owner writes its indexes and command output below the case's own scratch directory (short fixed segment).
+            const [caseResult] = await executeContextPackGeneration({
+                cases: [evaluationCase],
+                kitCommand: args.kitCommand,
+                outputRoot: path.join(scratch.path, `p${position + 1}`),
+                dependencies: {
+                    ...args.dependencies,
+                    // Guided exclusions keep ignored and oversized files out of the index; the raw baseline reads only eligible files.
+                    buildIndex: async (options) => {
+                        return (baseBuildIndex ?? buildMyDevKitIndex)({ ...options, excludePaths: guidedExclusions });
+                    },
+                    runRawBaseline: (rawCase) => baseRawBaseline(rawCase, { eligibleFiles: subject.eligibleFiles })
+                }
+            });
+            if (caseResult.evidence.treatments.some((treatment) => treatment.errors.some((error) => FATAL_CASE_ERROR_CODES.has(error.code)))) {
+                issues.push({ code: "EXECUTION_FAILED", message: "context-pack index preparation failed for a configured case." });
+                break;
+            }
+            const outside = countContextPackFilesOutsideEligibleUniverse(caseResult, eligible);
+            if (outside > 0) {
+                issues.push({
+                    code: "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE",
+                    message: `context-pack generation exposed ${outside} file identit${outside === 1 ? "y" : "ies"} outside the eligible subject universe.`
+                });
+                break;
+            }
+            results.push(caseResult);
+        }
+        catch {
+            issues.push({ code: "EXECUTION_FAILED", message: "context-pack execution failed for a configured case." });
+            break;
+        }
+    }
+    // Attempted even after a primary failure so a target mutation is never masked.
+    let comparison = null;
+    const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!after.ok) {
+        issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
+    }
+    else {
+        comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
+        if (comparison.status === "mutated") {
+            issues.push({ code: "TARGET_MUTATED", message: "the target changed during execution; it was not restored." });
+        }
+    }
+    const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
+    if (cleanupFailure) {
+        issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: "the private scratch could not be removed." });
+    }
+    if (issues.length > 0 || !comparison) {
+        throw new LocalSubjectExecutionError(issues.length > 0 ? issues : [{ code: "EXECUTION_FAILED", message: "execution did not complete." }], {
+            immutability: comparison
+        });
+    }
+    return {
+        results,
+        immutability: { status: "unchanged", preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount, newMutationCount: 0 }
+    };
+}