@dailephd/my-dev-kit-lab 0.8.0 → 0.8.2

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/localSubjectExecution.js

@@ -0,0 +1,124 @@
+import path from "node:path";
+import { validateRetrievalPrecisionRecallCase } from "../../../evaluation/retrievalQuality/index.js";
+import { buildMyDevKitIndex } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
+import { LocalSubjectExecutionError } from "../contextWindowScaling/localSubjectErrors.js";
+import { deriveGuidedIndexExclusions } from "../contextWindowScaling/localSubjectExclusions.js";
+import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch } from "../contextWindowScaling/localSubjectScratch.js";
+import { executeRetrievalQueryStrategyCaseFromPreparedIndex } from "./execution.js";
+/**
+ * Counts distinct repository-relative file identities exposed by the seven strategy-neutral evidence objects that lie
+ * outside the eligible subject universe. It inspects every non-null evidence file path and evidence symbol file, never
+ * source text, command output or node IDs. Only the count leaves this function; an offending identity is never named.
+ */
+export function countRetrievalQueryStrategyFilesOutsideEligibleUniverse(caseEvidence, eligibleFiles) {
+    const identities = new Set();
+    for (const treatment of caseEvidence.treatments) {
+        if (treatment.evidence === null)
+            continue;
+        for (const file of treatment.evidence.files)
+            identities.add(file.path);
+        for (const symbol of treatment.evidence.symbols) {
+            if (symbol.file !== null)
+                identities.add(symbol.file);
+        }
+    }
+    return [...identities].filter((identity) => !eligibleFiles.has(identity)).length;
+}
+/**
+ * Runs a loaded local subject through the seven-strategy comparison. One private base index per configured case (each
+ * case keeps its own exact source roots), all seven treatments from that one base, a private scratch outside the
+ * target, and pre/post target-immutability enforcement. It performs NO durable artifact writes; success is returned
+ * only when every safety gate passed. A mutated target is never reverted. Issue messages are fixed text and counts.
+ */
+export async function executeLocalRepositorySubjectRetrievalQueryStrategyComparison(args) {
+    const { subject } = args;
+    const buildIndex = args.dependencies?.buildIndex ?? buildMyDevKitIndex;
+    const executeCase = args.dependencies?.executeCase ?? executeRetrievalQueryStrategyCaseFromPreparedIndex;
+    await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
+    const guidedExclusions = deriveGuidedIndexExclusions({
+        eligibleFiles: subject.eligibleFiles,
+        excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles]
+    });
+    if (subject.evaluationCases.some((evaluationCase) => validateRetrievalPrecisionRecallCase(evaluationCase).length > 0)) {
+        throw new LocalSubjectExecutionError([
+            { code: "GROUND_TRUTH_INVALID", message: "retrieval comparison ground truth is incomplete for configured case(s)." }
+        ]);
+    }
+    const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
+    const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
+    const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!before.ok) {
+        throw new LocalSubjectExecutionError([{ code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` }]);
+    }
+    const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
+    const issues = [];
+    const caseEvidence = [];
+    const eligible = new Set(subject.eligibleFiles);
+    for (const [position, evaluationCase] of subject.evaluationCases.entries()) {
+        // Short, fixed segments: private case or repository names never become directory names.
+        const caseIndexRoot = path.join(scratch.path, `i${position + 1}`);
+        const baseIndexDir = path.join(caseIndexRoot, "base");
+        const commandsRoot = path.join(scratch.path, `c${position + 1}`);
+        try {
+            // requireKit: false keeps a failed index as a result we can classify instead of an arbitrary thrown message.
+            const build = await buildIndex({
+                target: { absoluteTargetRoot: evaluationCase.absoluteTargetRoot, sourceRoots: [...evaluationCase.sourceRoots] },
+                kitCommand: args.kitCommand,
+                indexDir: baseIndexDir,
+                commandsDir: path.join(commandsRoot, "index"),
+                requireKit: false,
+                excludePaths: guidedExclusions
+            });
+            if (!build.ok) {
+                issues.push({ code: "EXECUTION_FAILED", message: "retrieval comparison index preparation failed for a configured case." });
+                break;
+            }
+            const evidence = await executeCase({
+                evaluationCase,
+                kitCommand: args.kitCommand,
+                baseIndexDir,
+                commandsDir: path.join(commandsRoot, "retrieval"),
+                semanticIndexesRoot: path.join(caseIndexRoot, "strategies")
+            });
+            const outside = countRetrievalQueryStrategyFilesOutsideEligibleUniverse(evidence, eligible);
+            if (outside > 0) {
+                issues.push({
+                    code: "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE",
+                    message: `retrieval comparison exposed ${outside} file identit${outside === 1 ? "y" : "ies"} outside the eligible subject universe.`
+                });
+                break;
+            }
+            caseEvidence.push(evidence);
+        }
+        catch {
+            issues.push({ code: "EXECUTION_FAILED", message: "retrieval comparison execution failed for a configured case." });
+            break;
+        }
+    }
+    // Attempted even after a primary failure so a target mutation is never masked.
+    let comparison = null;
+    const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!after.ok) {
+        issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
+    }
+    else {
+        comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
+        if (comparison.status === "mutated") {
+            issues.push({ code: "TARGET_MUTATED", message: "the target changed during execution; it was not restored." });
+        }
+    }
+    const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
+    if (cleanupFailure) {
+        issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: "the private scratch could not be removed." });
+    }
+    if (issues.length > 0 || !comparison) {
+        throw new LocalSubjectExecutionError(issues.length > 0 ? issues : [{ code: "EXECUTION_FAILED", message: "execution did not complete." }], {
+            immutability: comparison
+        });
+    }
+    return {
+        caseEvidence,
+        immutability: { status: "unchanged", preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount, newMutationCount: 0 }
+    };
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/localSubjectPrivacy.js

@@ -0,0 +1,176 @@
+import { compareCodeUnits } from "../../../evaluation/indexSnapshot.js";
+import { redactedFileList } from "../contextWindowScaling/localSubjectPrivacy.js";
+/** Single fixed placeholder for a withheld case title. */
+export const RETRIEVAL_QUERY_STRATEGY_REDACTED_CASE_TITLE = "<redacted case title>";
+/** The one V1 marker stamped on every externally projected case. */
+export const RETRIEVAL_QUERY_STRATEGY_IDENTITY_REDACTION = {
+    fileIdentities: "redacted",
+    symbolIdentities: "redacted",
+    factIdentities: "redacted",
+    warningText: "redacted",
+    caseTitle: "redacted",
+    semanticNodeIds: "redacted"
+};
+export const RETRIEVAL_QUERY_STRATEGY_PRIVACY_FAILURE_MESSAGE = "External-local retrieval-query-strategy-comparison privacy projection failed; no durable output was written.";
+export const RETRIEVAL_QUERY_STRATEGY_UNEXPECTED_EXTERNAL_FAILURE_MESSAGE = "Local subject retrieval-query-strategy comparison failed (EXECUTION_FAILED): execution failed before completing (details withheld).";
+/** One opaque placeholder per identity: counts stay exact; null stays null; an empty list stays empty. */
+function placeholders(kind, count) {
+    return Array.from({ length: count }, (_, index) => `<redacted ${kind} ${index + 1}>`);
+}
+const filePlaceholders = (list) => (list === null ? null : redactedFileList(list.length));
+const symbolPlaceholders = (list) => (list === null ? null : placeholders("symbol", list.length));
+const factPlaceholders = (list) => (list === null ? null : placeholders("fact", list.length));
+/** Plugin-owned text per error code; a caught Error.message is never copied into durable external evidence. */
+const FIXED_COMPARISON_ERROR_TEXT = {
+    "ground-truth-invalid": "The case ground truth was invalid.",
+    "project-group-inconsistent": "The case source configuration was inconsistent.",
+    "project-index-failed": "The my-dev-kit index could not be prepared.",
+    "strategy-index-copy-failed": "The isolated semantic-strategy index could not be prepared.",
+    "retrieval-failed": "The retrieval strategy did not complete.",
+    "strategy-index-cleanup-failed": "The isolated semantic-strategy index could not be removed."
+};
+function projectEvidence(evidence) {
+    // One deterministic mapping per treatment over every file identity the evidence exposes.
+    const identities = new Set();
+    for (const file of evidence.files)
+        identities.add(file.path);
+    for (const symbol of evidence.symbols) {
+        if (symbol.file !== null)
+            identities.add(symbol.file);
+    }
+    const mapping = new Map();
+    [...identities].sort(compareCodeUnits).forEach((identity, index) => mapping.set(identity, `<redacted file ${index + 1}>`));
+    return {
+        ...evidence,
+        files: evidence.files.map((file) => ({ path: mapping.get(file.path) })),
+        symbols: evidence.symbols.map((symbol, index) => ({
+            name: `<redacted symbol ${index + 1}>`,
+            nodeId: null,
+            file: symbol.file === null ? null : mapping.get(symbol.file)
+        })),
+        steps: evidence.steps.map((step) => ({ ...step }))
+    };
+}
+function projectTreatment(treatment) {
+    return {
+        ...treatment,
+        retrieval: treatment.retrieval === null
+            ? null
+            : {
+                ...treatment.retrieval,
+                warnings: placeholders("warning", treatment.retrieval.warnings.length),
+                steps: treatment.retrieval.steps.map((step) => ({ ...step }))
+            },
+        evidence: treatment.evidence === null ? null : projectEvidence(treatment.evidence),
+        errors: treatment.errors.map((error) => ({ code: error.code, message: FIXED_COMPARISON_ERROR_TEXT[error.code] }))
+    };
+}
+/**
+ * Projects identity-bearing seven-treatment execution evidence into privacy-safe durable evidence. It performs no
+ * metric calculation and changes no count or availability.
+ */
+export function projectRetrievalQueryStrategyExecutionForExternalLocalPersistence(evidence) {
+    return evidence.map((entry) => {
+        const cloned = structuredClone(entry);
+        return {
+            caseId: cloned.caseId,
+            caseName: RETRIEVAL_QUERY_STRATEGY_REDACTED_CASE_TITLE,
+            benchmarkProject: cloned.benchmarkProject,
+            taskLocality: cloned.taskLocality,
+            treatments: cloned.treatments.map(projectTreatment),
+            identityRedaction: { ...RETRIEVAL_QUERY_STRATEGY_IDENTITY_REDACTION }
+        };
+    });
+}
+/** Only identity lists change; every number, availability and fixed-vocabulary reason is carried through untouched. */
+function projectRetrievalQualityMetricsForExternalPersistence(quality) {
+    return {
+        ...quality,
+        file: {
+            ...quality.file,
+            relevantRetrievedFiles: filePlaceholders(quality.file.relevantRetrievedFiles),
+            irrelevantRetrievedFiles: filePlaceholders(quality.file.irrelevantRetrievedFiles),
+            missedFiles: filePlaceholders(quality.file.missedFiles)
+        },
+        symbol: {
+            ...quality.symbol,
+            relevantRetrievedSymbols: symbolPlaceholders(quality.symbol.relevantRetrievedSymbols),
+            irrelevantRetrievedSymbols: symbolPlaceholders(quality.symbol.irrelevantRetrievedSymbols),
+            missedSymbols: symbolPlaceholders(quality.symbol.missedSymbols)
+        },
+        fact: {
+            ...quality.fact,
+            coveredFactIds: factPlaceholders(quality.fact.coveredFactIds),
+            uncoveredFactIds: factPlaceholders(quality.fact.uncoveredFactIds)
+        }
+    };
+}
+/**
+ * Preserves every numeric scientific value and the strategy interpretation; replaces only the private identity lists
+ * inside per-treatment quality. F1, means, Pareto fronts and the best strategy are never recalculated.
+ */
+export function projectRetrievalQueryStrategyAnalysisForExternalLocalPersistence(analysis) {
+    const cloned = structuredClone(analysis);
+    return {
+        cases: cloned.cases.map((entry) => ({
+            ...entry,
+            treatments: entry.treatments.map((treatment) => ({
+                ...treatment,
+                quality: treatment.quality === null ? null : projectRetrievalQualityMetricsForExternalPersistence(treatment.quality)
+            }))
+        })),
+        scopes: cloned.scopes
+    };
+}
+/**
+ * Defense in depth: fails closed if a known private file path, case title, warning, or root survived projection. It
+ * deliberately does not substring-search short symbol or fact identifiers, which would collide with ordinary words.
+ */
+export function assertRetrievalQueryStrategyExternalProjectionIsPrivate(projected, privateValues) {
+    const serialized = JSON.stringify(projected);
+    const variants = (value) => {
+        const forward = value.replace(/\\/g, "/");
+        const backward = value.replace(/\//g, "\\");
+        return [value, forward, backward].flatMap((entry) => [entry, JSON.stringify(entry).slice(1, -1)]);
+    };
+    const candidates = [...privateValues.filePaths, ...privateValues.caseTitles, ...privateValues.warnings, ...privateValues.privateRoots].filter((value) => value.length > 0);
+    if (candidates.some((value) => variants(value).some((variant) => serialized.includes(variant)))) {
+        throw new Error(RETRIEVAL_QUERY_STRATEGY_PRIVACY_FAILURE_MESSAGE);
+    }
+}
+const GENERIC_ISSUE_TEXT = {
+    WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
+    BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
+    AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
+    SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
+    GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
+    GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly"
+};
+/**
+ * Safe, persistable description of an external-local failure: issue codes, counts and mutation kinds only. Execution
+ * issue messages are fixed text by construction; nothing else (no path, identity, title, or caught text) can appear.
+ */
+export function describeRetrievalQueryStrategyLocalSubjectFailureForPersistence(error) {
+    const parts = [];
+    const seen = new Set();
+    for (const issue of error.issues) {
+        if (issue.code === "EXECUTION_FAILED" || issue.code === "GROUND_TRUTH_INVALID" || issue.code === "RETRIEVAL_OUTSIDE_ELIGIBLE_UNIVERSE") {
+            if (!seen.has(issue.message)) {
+                seen.add(issue.message);
+                parts.push(issue.message.replace(/\.$/, ""));
+            }
+            continue;
+        }
+        if (seen.has(issue.code))
+            continue;
+        seen.add(issue.code);
+        if (issue.code === "TARGET_MUTATED") {
+            const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
+            parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
+        }
+        else {
+            parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
+        }
+    }
+    return `Local subject retrieval-query-strategy comparison failed (${error.code}): ${parts.join("; ")}.`;
+}

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/metadata.js

@@ -0,0 +1,51 @@
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID = "retrieval-query-strategy-comparison";
+/** The v0.8.1 comparison reuses the frozen v0.8.0 corpus lineage; there is no second benchmark corpus. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_CASES_RESOURCE = "benchmarks/contracts/warm-index-benchmark-cases.json";
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_PROJECT_PROFILES_RESOURCE = "benchmarks/contracts/benchmark-project-profiles.json";
+export const retrievalQueryStrategyComparisonMetadata = {
+    id: RETRIEVAL_QUERY_STRATEGY_COMPARISON_PLUGIN_ID,
+    name: "Retrieval Query Strategy Comparison",
+    description: "Compare seven deterministic my-dev-kit retrieval query strategies on matched bundled benchmark cases without coding agents.",
+    schemaVersion: "1.0.0",
+    status: "experimental",
+    supportedTargets: ["self", "external-local"],
+    supportedOutputs: ["json", "html", "text", "artifact"]
+};
+/** Exactly the seven treatment levels, in the frozen strategy order. */
+export const RETRIEVAL_QUERY_STRATEGY_COMPARISON_VARIANTS = [
+    {
+        id: "keyword-search",
+        name: "Keyword search",
+        description: "Use the task query with my-dev-kit search and expose only the search result context."
+    },
+    {
+        id: "symbol-lookup",
+        name: "Symbol lookup",
+        description: "Search for the task query, select the top candidate, and inspect that exact graph node with lookup."
+    },
+    {
+        id: "graph-neighborhood",
+        name: "Graph neighborhood",
+        description: "Search for the task query, select the top candidate, and retrieve its bounded graph neighborhood with slice."
+    },
+    {
+        id: "source-slice",
+        name: "Source slice",
+        description: "Search for the task query, select the top candidate, and retrieve bounded numbered source for that node."
+    },
+    {
+        id: "data-model-graph",
+        name: "Data-model graph",
+        description: "Match task-query tokens against data-model graph labels and inspect the best deterministic entity or field."
+    },
+    {
+        id: "model-view-lineage",
+        name: "Model-view lineage",
+        description: "Match the task query to a data-model entity or field and retrieve its conservative static model-to-view lineage."
+    },
+    {
+        id: "combined-graph-guided",
+        name: "Combined graph-guided",
+        description: "Use the v0.8.0 compatibility workflow: search, lookup, slice, then bounded source for the top candidate."
+    }
+];

package/dist/src/experiments/plugins/retrievalQueryStrategyComparison/metrics.js

@@ -0,0 +1,60 @@
+import { RETRIEVAL_QUERY_STRATEGY_IDS } from "../../../evaluation/retrievalQueryStrategies.js";
+/** Projects already-calculated scientific values onto generic metrics. Owns no formula. */
+export function toRetrievalQueryStrategyOutcomeMetrics(treatment, caseId) {
+    const base = { variantId: treatment.strategyId, caseId };
+    const quality = treatment.quality;
+    const ratio = (id, name, metric) => ({
+        ...base,
+        id,
+        name,
+        value: metric?.value ?? null,
+        unit: "ratio"
+    });
+    const count = (id, name, value) => ({
+        ...base,
+        id,
+        name,
+        value: value ?? null,
+        unit: "count"
+    });
+    return [
+        ratio("file-precision", "File precision", quality?.file.precision),
+        ratio("file-recall", "File recall", quality?.file.recall),
+        ratio("file-f1", "File F1", treatment.fileF1),
+        ratio("symbol-precision", "Symbol precision", quality?.symbol.precision),
+        ratio("symbol-recall", "Symbol recall", quality?.symbol.recall),
+        ratio("symbol-f1", "Symbol F1", treatment.symbolF1),
+        ratio("fact-coverage", "Fact coverage", quality?.fact.coverage),
+        ratio("irrelevant-context-ratio", "Irrelevant context ratio", quality?.irrelevantContextRatio),
+        { ...base, id: "retrieved-token-count", name: "Retrieved token count", value: quality?.retrievedTokenCount ?? null, unit: "tokens" },
+        count("missed-file-count", "Missed file count", quality?.file.missedFileCount),
+        count("missed-symbol-count", "Missed symbol count", quality?.symbol.missedSymbolCount),
+        count("uncovered-fact-count", "Uncovered fact count", quality?.fact.uncoveredFactCount),
+        count("irrelevant-retrieved-file-count", "Irrelevant retrieved file count", quality?.file.irrelevantRetrievedFiles?.length),
+        count("irrelevant-retrieved-symbol-count", "Irrelevant retrieved symbol count", quality?.symbol.irrelevantRetrievedSymbols?.length)
+    ];
+}
+/** Per scope: one matched-case count, then four objective means per strategy in canonical order. No best/rank/score metric. */
+export function toRetrievalQueryStrategyRunMetrics(analysis) {
+    const metrics = [];
+    for (const scope of analysis.scopes) {
+        const { scopeId } = scope;
+        metrics.push({
+            id: `${scopeId}-comparison-case-count`,
+            name: `${scopeId} matched comparison cases`,
+            value: scope.comparisonCaseCount,
+            unit: "count"
+        });
+        for (const strategyId of RETRIEVAL_QUERY_STRATEGY_IDS) {
+            const objectives = scope.strategySummaries.find((summary) => summary.strategyId === strategyId)?.objectives ?? null;
+            metrics.push({ id: `${scopeId}-mean-file-f1`, name: `${scopeId} mean file F1`, value: objectives?.meanFileF1 ?? null, unit: "ratio", variantId: strategyId }, { id: `${scopeId}-mean-symbol-f1`, name: `${scopeId} mean symbol F1`, value: objectives?.meanSymbolF1 ?? null, unit: "ratio", variantId: strategyId }, { id: `${scopeId}-mean-fact-coverage`, name: `${scopeId} mean fact coverage`, value: objectives?.meanFactCoverage ?? null, unit: "ratio", variantId: strategyId }, {
+                id: `${scopeId}-mean-retrieved-token-count`,
+                name: `${scopeId} mean retrieved token count`,
+                value: objectives?.meanRetrievedTokenCount ?? null,
+                unit: "tokens",
+                variantId: strategyId
+            });
+        }
+    }
+    return metrics;
+}