@dailephd/my-dev-kit-lab 0.7.1 → 0.8.0

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectExecution.js

@@ -0,0 +1,136 @@
+import path from "node:path";
+import { relativeWithinRoot } from "../../../core/pathSafety.js";
+import { runMyDevKitRetrieval } from "../../../evaluation/runMyDevKitRetrieval.js";
+import { runRawFullFileBaseline } from "../../../evaluation/runRawFullFileBaseline.js";
+import { captureTargetSnapshot, compareTargetSnapshots } from "../../../evaluation/targetImmutability/index.js";
+import { executeContextWindowScalingCases, } from "./execution.js";
+import { LocalSubjectExecutionError, redactPrivatePaths, } from "./localSubjectErrors.js";
+import { deriveGuidedIndexExclusions } from "./localSubjectExclusions.js";
+import { assertWorkRootOutsideTarget, createPrivateScratch, defaultPrivateScratchIo, removePrivateScratch, } from "./localSubjectScratch.js";
+import { resolveScalingProjectProfiles } from "./projectProfile.js";
+function toRepositoryRelative(repositoryRoot, file) {
+    const slashed = file.replace(/\\/g, "/");
+    if (path.isAbsolute(file)) {
+        try {
+            return relativeWithinRoot(repositoryRoot, file);
+        }
+        catch {
+            return slashed;
+        }
+    }
+    return slashed.replace(/^\.\//, "");
+}
+/** Safe local constructors: raw context from eligible files only; guided retrieval over the original root. */
+function createSafeLocalDependencies(subject, guidedExclusions) {
+    const eligible = new Set(subject.eligibleFiles);
+    return {
+        constructRawContext: (evaluationCase) => runRawFullFileBaseline(evaluationCase, { eligibleFiles: subject.eligibleFiles }),
+        constructGuidedContext: async ({ evaluationCase, kitCommand, outputDir }) => {
+            const result = await runMyDevKitRetrieval({
+                evaluationCase,
+                kitCommand,
+                outputDir,
+                requireKit: false,
+                excludePaths: guidedExclusions,
+            });
+            const returned = [...result.filesRead, ...(result.selectedFile ? [result.selectedFile] : [])];
+            const outside = returned.filter((file) => !eligible.has(toRepositoryRelative(subject.repositoryRoot, file)));
+            if (outside.length > 0) {
+                throw new Error(`Guided retrieval returned ${outside.length} file(s) outside the eligible subject universe: ${outside
+                    .map((file) => toRepositoryRelative(subject.repositoryRoot, file))
+                    .join(", ")}`);
+            }
+            return result;
+        },
+    };
+}
+/**
+ * Internal programmatic seam: runs a loaded Batch 1 local subject through the existing context-window core with
+ * safe raw and guided constructors, a private scratch outside the target, and pre/post target-immutability
+ * enforcement. Order: validate work root, derive exclusions, capture before, create scratch, execute, capture after,
+ * compare, remove scratch, then decide. Success is returned only when execution, immutability and cleanup all pass.
+ * A mutated target is never reverted.
+ */
+export async function executeLocalRepositorySubjectContextWindowScaling(args) {
+    const { subject } = args;
+    await assertWorkRootOutsideTarget(args.workRoot, subject.repositoryRoot);
+    const guidedExclusions = deriveGuidedIndexExclusions({
+        eligibleFiles: subject.eligibleFiles,
+        excludedFiles: [...subject.runtimeSafetyExclusions.gitIgnoredFiles, ...subject.runtimeSafetyExclusions.oversizedFiles],
+    });
+    const snapshotConfig = { targetRootPath: subject.repositoryRoot, relativeFilePaths: subject.eligibleFiles };
+    const snapshotOptions = { externalLocalSafe: { maxHashedFileBytes: subject.manifest.safetyPolicy.maxFileBytes } };
+    const before = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!before.ok) {
+        throw new LocalSubjectExecutionError([
+            { code: "BEFORE_SNAPSHOT_FAILED", message: `target snapshot before execution failed (${before.code}).` },
+        ]);
+    }
+    const scratch = await createPrivateScratch(args.workRoot, subject.repositoryRoot);
+    const privatePaths = [subject.repositoryRoot, scratch.path, path.resolve(args.workRoot)];
+    const redact = (message) => redactPrivatePaths(message, privatePaths);
+    const issues = [];
+    let caseEvidence = null;
+    try {
+        const projectProfiles = resolveScalingProjectProfiles(subject.evaluationCases, [], { eligibleFiles: subject.eligibleFiles });
+        caseEvidence = await executeContextWindowScalingCases({
+            cases: subject.evaluationCases,
+            contextBudgets: args.contextBudgets,
+            kitCommand: args.kitCommand,
+            outputRoot: scratch.path,
+            projectProfiles,
+            cwd: scratch.path,
+            env: args.env,
+            dependencies: { ...createSafeLocalDependencies(subject, guidedExclusions), ...args.dependencies },
+        });
+    }
+    catch (error) {
+        issues.push({
+            code: "EXECUTION_FAILED",
+            message: redact(`execution threw: ${error instanceof Error ? error.message : String(error)}`),
+        });
+    }
+    if (caseEvidence) {
+        for (const caseResult of caseEvidence) {
+            for (const treatment of caseResult.treatments) {
+                if (treatment.status === "failed" || treatment.errors.length > 0) {
+                    issues.push({
+                        code: "EXECUTION_FAILED",
+                        message: redact(`case ${caseResult.caseId} treatment ${treatment.variantId} failed: ${treatment.errors
+                            .map((entry) => `${entry.code}: ${entry.message}`)
+                            .join("; ")}`),
+                    });
+                }
+            }
+        }
+    }
+    let comparison = null;
+    const after = await captureTargetSnapshot(snapshotConfig, snapshotOptions);
+    if (!after.ok) {
+        issues.push({ code: "AFTER_SNAPSHOT_FAILED", message: `target snapshot after execution failed (${after.code}).` });
+    }
+    else {
+        comparison = compareTargetSnapshots(before.snapshot, after.snapshot);
+        if (comparison.status === "mutated") {
+            issues.push({
+                code: "TARGET_MUTATED",
+                message: `the target changed during execution (${comparison.mutations.map((mutation) => mutation.id).join(", ")}); it was not restored.`,
+            });
+        }
+    }
+    const cleanupFailure = await removePrivateScratch(scratch, args.scratchIo ?? defaultPrivateScratchIo);
+    if (cleanupFailure) {
+        issues.push({ code: "SCRATCH_CLEANUP_FAILED", message: redact(cleanupFailure) });
+    }
+    if (issues.length > 0 || !caseEvidence || !comparison) {
+        throw new LocalSubjectExecutionError(issues, { caseEvidence, immutability: comparison });
+    }
+    return {
+        caseEvidence,
+        immutability: {
+            status: "unchanged",
+            preExistingGitStatusEntryCount: comparison.preExistingGitStatusEntryCount,
+            newMutationCount: 0,
+        },
+    };
+}

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectPrivacy.js

@@ -0,0 +1,128 @@
+import path from "node:path";
+import { redactPrivatePaths } from "./localSubjectErrors.js";
+/** Value used wherever a durable external-local artifact would otherwise show a machine-local path. */
+export const EXTERNAL_LOCAL_REDACTED_VALUE = "[redacted]";
+const REDACTED_FILE_TEXT = "<redacted file>";
+/** One opaque placeholder per file: counts stay exact and an empty list still means "no files". */
+export function redactedFileList(count) {
+    return Array.from({ length: count }, (_, index) => `<redacted file ${index + 1}>`);
+}
+/** Replaces known repository-relative paths (longest first) and private roots inside free text. */
+export function redactKnownPaths(text, knownFiles, privateRoots = []) {
+    let redacted = redactPrivatePaths(text, privateRoots);
+    for (const file of [...knownFiles].sort((left, right) => right.length - left.length)) {
+        if (file.length === 0)
+            continue;
+        // Native tool errors on Windows may spell a repository-relative path with backslashes.
+        for (const variant of new Set([file, file.replace(/\//g, "\\")]))
+            redacted = redacted.split(variant).join(REDACTED_FILE_TEXT);
+    }
+    return redacted;
+}
+function redactStrings(value, redact) {
+    if (typeof value === "string")
+        return redact(value);
+    if (Array.isArray(value))
+        return value.map((entry) => redactStrings(entry, redact));
+    if (value && typeof value === "object") {
+        return Object.fromEntries(Object.entries(value).map(([key, nested]) => [key, redactStrings(nested, redact)]));
+    }
+    return value;
+}
+/**
+ * Projects execution evidence for durable persistence of an external-local run. File identity lists become
+ * placeholder lists of the same length (with an explicit per-treatment flag) and every free-text field has known
+ * repository paths and private roots replaced. Scientific numbers, statuses and identifiers are untouched.
+ */
+export function projectEvidenceForExternalLocalPersistence(evidence, privacy) {
+    const redact = (text) => redactKnownPaths(text, privacy.knownFiles, privacy.privateRoots);
+    return evidence.map((caseEvidence) => {
+        const cloned = structuredClone(caseEvidence);
+        return {
+            ...redactStrings(cloned, redact),
+            treatments: cloned.treatments.map((treatment) => {
+                const safe = redactStrings(treatment, redact);
+                return {
+                    ...safe,
+                    context: {
+                        ...safe.context,
+                        observedFiles: treatment.context.observedFiles === null ? null : redactedFileList(treatment.context.observedFiles.length),
+                    },
+                    relevantFileEvidence: {
+                        ...safe.relevantFileEvidence,
+                        expectedRelevantFiles: redactedFileList(treatment.relevantFileEvidence.expectedRelevantFiles.length),
+                        omittedRelevantFiles: redactedFileList(treatment.relevantFileEvidence.omittedRelevantFiles.length),
+                    },
+                    fileIdentityRedaction: "redacted",
+                };
+            }),
+        };
+    });
+}
+/** Durable target identity for an external-local run: logical identity and Git identity, no filesystem facts. */
+export function projectExternalLocalTarget(manifest) {
+    return {
+        kind: "external-local",
+        targetRoot: manifest.logicalTargetRoot,
+        toolRoot: EXTERNAL_LOCAL_REDACTED_VALUE,
+        packageName: null,
+        packageVersion: null,
+        hasPackageJson: false,
+        hasLockfile: false,
+        branch: manifest.repository.branch,
+        commit: manifest.repository.commit,
+        hasGit: true,
+        isSelf: false,
+        privacyProjection: "external-local-redacted",
+    };
+}
+/** Replaces target and machine-local metadata of a run before any report is serialized. */
+export function projectRunForExternalLocalPersistence(run, target) {
+    const metadata = { ...(run.metadata ?? {}) };
+    if ("outputRoot" in metadata)
+        metadata.outputRoot = EXTERNAL_LOCAL_REDACTED_VALUE;
+    if (typeof metadata.executionArtifactPath === "string")
+        metadata.executionArtifactPath = path.basename(metadata.executionArtifactPath);
+    return { ...run, target, metadata: metadata };
+}
+const GENERIC_ISSUE_TEXT = {
+    WORK_ROOT_INSIDE_TARGET: "the output directory must be outside the local subject repository",
+    BEFORE_SNAPSHOT_FAILED: "the target snapshot before execution could not be captured",
+    AFTER_SNAPSHOT_FAILED: "the target snapshot after execution could not be captured",
+    SCRATCH_CLEANUP_FAILED: "the private scratch could not be removed",
+    GUIDED_EXCLUSION_LIMIT: "the guided-index exclusions exceeded the safe limit",
+    GUIDED_EXCLUSION_UNREPRESENTABLE: "the guided-index exclusions could not be expressed exactly",
+};
+/**
+ * Safe, persistable description of a local-subject execution failure: issue codes, case and treatment identifiers,
+ * error codes and mutation kinds only. Repository file names, mutation identifiers and the full comparison are
+ * never included.
+ */
+export function describeLocalSubjectFailureForPersistence(error) {
+    const parts = [];
+    const seen = new Set();
+    for (const issue of error.issues) {
+        if (seen.has(issue.code))
+            continue;
+        seen.add(issue.code);
+        if (issue.code === "EXECUTION_FAILED") {
+            const details = [];
+            for (const caseEvidence of error.caseEvidence ?? []) {
+                for (const treatment of caseEvidence.treatments) {
+                    if (treatment.status === "failed" || treatment.errors.length > 0) {
+                        details.push(`case ${caseEvidence.caseId} treatment ${treatment.variantId}: ${treatment.errors.map((entry) => entry.code).join(", ") || "failed"}`);
+                    }
+                }
+            }
+            parts.push(details.length > 0 ? `execution failed (${details.join("; ")})` : "execution failed before completing (details withheld)");
+        }
+        else if (issue.code === "TARGET_MUTATED") {
+            const kinds = [...new Set((error.immutability?.mutations ?? []).map((mutation) => mutation.kind))].sort();
+            parts.push(`the target changed during execution${kinds.length > 0 ? ` (${kinds.join(", ")})` : ""}; it was not restored`);
+        }
+        else {
+            parts.push(GENERIC_ISSUE_TEXT[issue.code] ?? `issue ${issue.code}`);
+        }
+    }
+    return `Local subject execution failed (${error.code}): ${parts.join("; ")}.`;
+}

package/dist/src/experiments/plugins/contextWindowScaling/localSubjectScratch.js

@@ -0,0 +1,85 @@
+import { realpath as nativeRealpath } from "node:fs";
+import { lstat, mkdir, mkdtemp, rm } from "node:fs/promises";
+import path from "node:path";
+import { promisify } from "node:util";
+import { relativeWithinRoot } from "../../../core/pathSafety.js";
+import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
+const realpath = promisify(nativeRealpath.native);
+export const defaultPrivateScratchIo = {
+    removeDirectory: (directory) => rm(directory, { recursive: true, force: true }),
+};
+/** Physical path of a path that may not exist yet: realpath of the deepest existing ancestor plus the remainder. */
+async function resolvePhysicalPath(candidate) {
+    const resolved = path.resolve(candidate);
+    const remainder = [];
+    let current = resolved;
+    for (;;) {
+        try {
+            const physical = await realpath(current);
+            return remainder.length === 0 ? physical : path.join(physical, ...remainder.reverse());
+        }
+        catch (error) {
+            const code = error.code;
+            const parent = path.dirname(current);
+            if ((code !== "ENOENT" && code !== "ENOTDIR") || parent === current)
+                return resolved;
+            remainder.push(path.basename(current));
+            current = parent;
+        }
+    }
+}
+function isSameOrInside(root, candidate) {
+    try {
+        relativeWithinRoot(root, candidate);
+        return true;
+    }
+    catch {
+        return false;
+    }
+}
+/** True when both paths resolve to the same physical directory (follows links; case-insensitive on Windows). */
+export async function isSamePhysicalDirectory(left, right) {
+    const [physicalLeft, physicalRight] = await Promise.all([resolvePhysicalPath(left), resolvePhysicalPath(right)]);
+    return path.relative(physicalLeft, physicalRight) === "";
+}
+/** Rejects a Lab-owned work root that equals or lies inside the target. Performs no writes. */
+export async function assertWorkRootOutsideTarget(workRoot, targetRoot) {
+    const physicalTarget = await resolvePhysicalPath(targetRoot);
+    const physicalWorkRoot = await resolvePhysicalPath(workRoot);
+    if (isSameOrInside(physicalTarget, physicalWorkRoot)) {
+        throw new LocalSubjectExecutionError([
+            { code: "WORK_ROOT_INSIDE_TARGET", message: "the Lab work root must be outside the local subject repository." },
+        ]);
+    }
+}
+/** Creates a short run-owned private scratch directory under the work root and re-verifies it is outside the target. */
+export async function createPrivateScratch(workRoot, targetRoot) {
+    await mkdir(path.resolve(workRoot), { recursive: true });
+    const scratchPath = await mkdtemp(path.join(await realpath(path.resolve(workRoot)), "s-"));
+    if (isSameOrInside(await resolvePhysicalPath(targetRoot), await resolvePhysicalPath(scratchPath))) {
+        await rm(scratchPath, { recursive: true, force: true });
+        throw new LocalSubjectExecutionError([
+            { code: "WORK_ROOT_INSIDE_TARGET", message: "the private scratch directory resolved inside the local subject repository." },
+        ]);
+    }
+    return { path: scratchPath };
+}
+/**
+ * Removes the scratch and verifies it is gone. Returns a failure description instead of throwing so the caller can
+ * keep the primary error; never retries and never sleeps.
+ */
+export async function removePrivateScratch(scratch, io = defaultPrivateScratchIo) {
+    try {
+        await io.removeDirectory(scratch.path);
+    }
+    catch (error) {
+        return `private scratch could not be removed: ${error instanceof Error ? error.message : String(error)}`;
+    }
+    try {
+        await lstat(scratch.path);
+        return "private scratch still exists after removal.";
+    }
+    catch (error) {
+        return error.code === "ENOENT" ? null : "private scratch state could not be verified after removal.";
+    }
+}

package/dist/src/experiments/plugins/contextWindowScaling/metadata.js

@@ -4,9 +4,9 @@ export const CONTEXT_WINDOW_SCALING_TREATMENT_IDS = ["raw-full-file", "my-dev-ki
 export const contextWindowScalingMetadata = {
     id: CONTEXT_WINDOW_SCALING_PLUGIN_ID,
     name: "Context Window Scaling",
-    description: "Measure raw-full-file and my-dev-kit-guided strategies under increasing deterministic estimated-context-token budgets, over the bundled fixed scaling corpus or caller-supplied deterministic synthetic repositories (--synthetic-config).",
+    description: "Measure raw-full-file and my-dev-kit-guided strategies under increasing deterministic estimated-context-token budgets, over the bundled fixed scaling corpus, caller-supplied deterministic synthetic repositories (--synthetic-config), or an explicitly selected local Git repository (--target with --local-subject-config). Operational context-fit measurement only; it does not measure retrieval precision, recall, or ranking quality.",
     schemaVersion: "1.0.0",
     status: "experimental",
-    supportedTargets: ["self"],
+    supportedTargets: ["self", "external-local"],
     supportedOutputs: ["json", "text", "html", "plot"],
 };

package/dist/src/experiments/plugins/contextWindowScaling/plugin.js

@@ -2,6 +2,7 @@ import { mkdir, writeFile } from "node:fs/promises";
 import path from "node:path";
 import { tokenCountMethod } from "../../../core/countTokens.js";
 import { resolveWithinRoot } from "../../../core/pathSafety.js";
+import { serializeLocalRepositorySubjectManifest, } from "../../../evaluation/localRepositorySubject/index.js";
 import { summarizeExperimentRun } from "../../results.js";
 import { aggregateStatus } from "../warmIndexReuse/execution.js";
 import { contextWindowScalingConfigDefinition, defaultContextWindowScalingConfig, validateContextWindowScalingConfig, } from "./config.js";
@@ -9,7 +10,13 @@ import { executeContextWindowScalingCases, } from "./execution.js";
 import { buildContextWindowScalingExecutionArtifact, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE, } from "./executionArtifact.js";
 import { CONTEXT_WINDOW_SCALING_TREATMENT_IDS, contextWindowScalingMetadata } from "./metadata.js";
 import { aggregateContextWindowScaling, toAggregateExperimentMetrics, toOutcomeBudgetMetrics, } from "./metrics.js";
+import { LocalSubjectExecutionError } from "./localSubjectErrors.js";
+import { executeLocalRepositorySubjectContextWindowScaling } from "./localSubjectExecution.js";
+import { describeLocalSubjectFailureForPersistence, projectEvidenceForExternalLocalPersistence, projectExternalLocalTarget, redactKnownPaths, } from "./localSubjectPrivacy.js";
+import { isSamePhysicalDirectory } from "./localSubjectScratch.js";
 import { resolveScalingProjectProfiles } from "./projectProfile.js";
+/** Privacy-safe Batch 1 manifest persisted beside the execution artifact for external-local runs. */
+export const LOCAL_REPOSITORY_SUBJECT_MANIFEST_FILE = "local-repository-subject-manifest.json";
 const VARIANTS = [
     {
         id: "raw-full-file",
@@ -31,6 +38,14 @@ export const contextWindowScalingPlugin = {
     validateConfig: validateContextWindowScalingConfig,
     async run(context) {
         const startedAt = context.startedAt.toISOString();
+        const localSubject = readLocalSubjectInput(context.inputs);
+        if (localSubject)
+            return runLocalSubject(context, localSubject, startedAt);
+        if (context.target.kind === "external-local") {
+            // Never fall back to the legacy raw/guided path against an external repository: it is neither
+            // ignore-aware nor symlink-aware and writes artifacts it must not.
+            throw new Error("External-local context-window-scaling targets require a loaded local repository subject (--local-subject-config).");
+        }
         const cases = readCasesInput(context.inputs);
         const projectProfiles = resolveScalingProjectProfiles(cases, readProjectProfilesInput(context.inputs));
         const outDir = context.outputRoot ?? path.resolve(context.toolRoot, "lab-output", "context-window-scaling");
@@ -72,6 +87,84 @@ export const contextWindowScalingPlugin = {
         return result.summary ?? summarizeExperimentRun(result);
     },
 };
+function readLocalSubjectInput(inputs) {
+    const value = inputs?.localSubject;
+    return value && typeof value === "object" ? value : undefined;
+}
+/**
+ * External-local path: the Batch 2 safe seam owns execution, immutability and scratch cleanup; this function only
+ * projects the privacy-lean result for persistence and feeds the same artifact and run builders as the legacy path.
+ */
+async function runLocalSubject(context, subject, startedAt) {
+    if (context.target.isSelf || context.target.kind !== "external-local") {
+        throw new Error("Local-repository subject mode requires an external-local target.");
+    }
+    if (!(await isSamePhysicalDirectory(context.target.targetRoot, subject.repositoryRoot))) {
+        throw new Error("The selected --target is not the repository the local subject was loaded from.");
+    }
+    const outDir = context.outputRoot;
+    if (!outDir)
+        throw new Error("Local-repository subject mode requires an experiment output directory.");
+    const knownFiles = [
+        ...subject.eligibleFiles,
+        ...subject.runtimeSafetyExclusions.gitIgnoredFiles,
+        ...subject.runtimeSafetyExclusions.oversizedFiles,
+    ];
+    const privateRoots = [subject.repositoryRoot, context.target.targetRoot, outDir, context.toolRoot];
+    let localResult;
+    try {
+        localResult = await executeLocalRepositorySubjectContextWindowScaling({
+            subject,
+            contextBudgets: context.config.contextBudgets,
+            kitCommand: context.config.kitCommand,
+            workRoot: outDir,
+            env: readEnvInput(context.inputs),
+            dependencies: readDependenciesInput(context.inputs),
+        });
+    }
+    catch (error) {
+        if (error instanceof LocalSubjectExecutionError)
+            throw new Error(describeLocalSubjectFailureForPersistence(error));
+        throw new Error(redactKnownPaths(error instanceof Error ? error.message : String(error), knownFiles, privateRoots));
+    }
+    const startedAtIso = startedAt;
+    const evidence = projectEvidenceForExternalLocalPersistence(localResult.caseEvidence, { knownFiles, privateRoots });
+    await mkdir(outDir, { recursive: true });
+    const completedAt = new Date().toISOString();
+    const artifact = buildContextWindowScalingExecutionArtifact({
+        runId: context.runId,
+        pluginId: contextWindowScalingMetadata.id,
+        pluginSchemaVersion: contextWindowScalingMetadata.schemaVersion,
+        startedAt: startedAtIso,
+        completedAt,
+        tokenCountMethod,
+        contextBudgets: context.config.contextBudgets,
+        cases: evidence,
+    });
+    const artifactPath = resolveWithinRoot(outDir, CONTEXT_WINDOW_SCALING_EXECUTION_ARTIFACT_FILE);
+    await writeFile(artifactPath, `${JSON.stringify(artifact, null, 2)}
+`, "utf8");
+    const manifestPath = resolveWithinRoot(outDir, LOCAL_REPOSITORY_SUBJECT_MANIFEST_FILE);
+    await writeFile(manifestPath, serializeLocalRepositorySubjectManifest(subject.manifest), "utf8");
+    const run = mapExecutionToRun({
+        runId: context.runId,
+        startedAt: startedAtIso,
+        completedAt,
+        target: projectExternalLocalTarget(subject.manifest),
+        contextBudgets: context.config.contextBudgets,
+        executionEvidence: evidence,
+        artifactPath,
+    });
+    run.artifacts.push({
+        id: "local-repository-subject-manifest",
+        label: "Privacy-safe local repository subject manifest",
+        path: manifestPath,
+        kind: "artifact",
+        mimeType: "application/json",
+        description: "Subject identity, Git commit, safety policy and aggregate inventory counts; no paths, source text or file lists.",
+    });
+    return run;
+}
 export function mapExecutionToRun(args) {
     const cases = args.executionEvidence.map((caseEvidence) => ({
         id: caseEvidence.caseId,

package/dist/src/experiments/plugins/contextWindowScaling/projectProfile.js

@@ -1,12 +1,7 @@
 import path from "node:path";
 import { calculateProjectComplexityScore, computeProjectComplexityMetrics, PROJECT_COMPLEXITY_FORMULA, } from "../../../evaluation/projectComplexity.js";
 import { buildProjectFileTree } from "../../../evaluation/projectFileTree.js";
-/**
- * The fixed scaling project has no entry in the shared benchmark-project-profiles contract. The
- * existing prompt owner requires a profile per case, so a missing one is derived in memory from
- * the existing file-tree and complexity owners. Supplied profiles always win; nothing is written.
- */
-export function resolveScalingProjectProfiles(cases, supplied) {
+export function resolveScalingProjectProfiles(cases, supplied, options = {}) {
     const profiles = [...supplied];
     const known = new Set(profiles.map((profile) => profile.projectId));
     for (const evaluationCase of cases) {
@@ -14,13 +9,13 @@ export function resolveScalingProjectProfiles(cases, supplied) {
         if (known.has(profileId))
             continue;
         known.add(profileId);
-        profiles.push(deriveProfile(profileId, evaluationCase, cases));
+        profiles.push(deriveProfile(profileId, evaluationCase, cases, options));
     }
     return profiles;
 }
-function deriveProfile(profileId, evaluationCase, cases) {
+function deriveProfile(profileId, evaluationCase, cases, options) {
     const siblings = cases.filter((candidate) => (candidate.projectProfileRef ?? candidate.benchmarkProject) === profileId);
-    const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot);
+    const fileTree = buildProjectFileTree(evaluationCase.absoluteTargetRoot, options.eligibleFiles === undefined ? {} : { allowedRelativeFiles: options.eligibleFiles });
     const average = (values) => (values.length === 0 ? 0 : values.reduce((a, b) => a + b, 0) / values.length);
     const complexityMetrics = computeProjectComplexityMetrics(evaluationCase.absoluteTargetRoot, fileTree, {
         taskCount: siblings.length,

package/dist/src/experiments/plugins/index.js

@@ -2,3 +2,4 @@ export * from "./contextStrategyComparison/index.js";
 export * from "./contextWindowScaling/index.js";
 export * from "./incrementalChangeStaleness/index.js";
 export * from "./warmIndexReuse/index.js";
+export * from "./retrievalPrecisionRecall/index.js";

package/dist/src/experiments/plugins/retrievalPrecisionRecall/config.js

@@ -0,0 +1,39 @@
+import { invalidExperimentConfig, isPlainObject, mergeConfig, validExperimentConfig } from "../../config.js";
+export const defaultRetrievalPrecisionRecallConfig = {
+    outDir: "lab-output/retrieval-precision-recall",
+    kitCommand: "npx @dailephd/my-dev-kit@latest"
+};
+const SUPPORTED_FIELDS = ["outDir", "kitCommand", "caseIds", "benchmarkProjects"];
+export const retrievalPrecisionRecallConfigDefinition = {
+    fields: [
+        { name: "outDir", type: "string", required: true, description: "Experiment output root." },
+        { name: "kitCommand", type: "string", description: "my-dev-kit command used for index and retrieval." },
+        { name: "caseIds", type: "array", description: "Bundled evaluation case IDs to include (corpus order is preserved)." },
+        { name: "benchmarkProjects", type: "array", description: "Bundled benchmark project IDs to include (corpus order is preserved)." }
+    ]
+};
+export function validateRetrievalPrecisionRecallConfig(config) {
+    if (config !== undefined && !isPlainObject(config)) {
+        return invalidExperimentConfig(["retrieval precision/recall config must be an object."]);
+    }
+    const errors = [];
+    const unsupported = Object.keys(config ?? {}).filter((key) => !SUPPORTED_FIELDS.includes(key));
+    if (unsupported.length > 0) {
+        errors.push(`Unsupported retrieval-precision-recall config field(s): ${unsupported.sort().join(", ")}.`);
+    }
+    const normalized = mergeConfig(defaultRetrievalPrecisionRecallConfig, config);
+    for (const field of ["outDir", "kitCommand"]) {
+        const value = normalized[field];
+        if (typeof value !== "string" || !value.trim())
+            errors.push(`${field} must be a non-empty string.`);
+    }
+    for (const field of ["caseIds", "benchmarkProjects"]) {
+        const value = normalized[field];
+        if (value === undefined)
+            continue;
+        if (!Array.isArray(value) || value.length === 0 || !value.every((item) => typeof item === "string" && item.trim())) {
+            errors.push(`${field} must be a non-empty array of non-empty strings when provided.`);
+        }
+    }
+    return errors.length > 0 ? invalidExperimentConfig(errors) : validExperimentConfig(normalized);
+}