@dailephd/my-dev-kit-lab 0.6.3 → 0.7.1

package/dist/src/commands/runExperimentRunCommand.js

@@ -1,8 +1,10 @@
+import { readFile } from "node:fs/promises";
 import path from "node:path";
 import { parseAgentCommandTemplate } from "../agents/index.js";
 import { parseAgentId } from "../agents/agentRegistry.js";
 import { readBenchmarkProjectProfiles, readEvaluationCases } from "../evaluation/index.js";
-import { contextStrategyComparisonPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
+import { contextStrategyComparisonPlugin, contextWindowScalingPlugin, createDefaultExperimentPluginRegistry, getWarmIndexCampaignPreset, incrementalChangeStalenessPlugin, parseContextBudgetsCliValue, parseWarmIndexCampaignPresetId, resolveExperimentTarget, runExperiment, warmIndexReusePlugin } from "../experiments/index.js";
+import { prepareSyntheticContextWindowScalingInputs } from "../experiments/plugins/contextWindowScaling/index.js";
 import { buildDefaultExperimentOutputRoot } from "../experiments/outputPaths.js";
 import { parsePromptComplexityLevel, parsePromptStrategy } from "../prompts/index.js";
 import { writePluginExperimentReports } from "../report/index.js";
@@ -32,27 +34,42 @@ import { runWarmIndexCampaignPresentation } from "./runWarmIndexCampaignPresenta
 // ---------------------------------------------------------------------------
 const DEFAULT_CASES_RESOURCE = "examples/token-savings-cases.json";
 const DEFAULT_PROJECT_PROFILES_RESOURCE = "benchmarks/contracts/benchmark-project-profiles.json";
+// context-window-scaling owns one frozen bundled case catalog; its fixed project is referenced by the
+// catalog's package-relative targetRoot, so both resolve from the package root, never the cwd.
+const CONTEXT_WINDOW_SCALING_CASES_RESOURCE = "benchmarks/contracts/context-window-scaling-cases.json";
+// Options the context-window-scaling plugin accepts; every other flag is rejected, not ignored.
+const CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS = ["--experiment", "--out", "--case", "--synthetic-config", "--context-budgets", "--kit-command"];
 export async function runExperimentRunCommandFromArgs(argv, options = {}) {
     try {
         const args = parseRunExperimentArgs(argv);
         const context = options.context ?? createLabExecutionContext();
         const toolRoot = context.packageRoot;
         const registry = createDefaultExperimentPluginRegistry();
-        const inputs = await loadPluginInputs(args, toolRoot, context);
         const runId = generateExperimentRunId(args.experimentId);
-        const outputRoot = resolveOutputRoot(args, {
+        let outputRoot = resolveOutputRoot(args, {
             toolRoot,
             invocationCwd: context.invocationCwd,
             installedDefaultOutputRoot: options.installedDefaultOutputRoot,
             experimentId: args.experimentId,
             runId
         });
+        if (args.syntheticConfigPath !== undefined && outputRoot === undefined) {
+            // Synthetic generation needs the concrete root before the runner runs; use the exact default the
+            // runner itself would compute for this self-targeted invocation.
+            outputRoot = buildDefaultExperimentOutputRoot({
+                toolRoot,
+                pluginId: args.experimentId,
+                target: resolveExperimentTarget(args.targetPath, toolRoot),
+                runId
+            });
+        }
+        const inputs = await loadPluginInputs(args, toolRoot, context, outputRoot);
         const result = await runExperiment({
             pluginId: args.experimentId,
             registry,
             targetPath: args.targetPath,
             outputRoot,
-            config: args.config,
+            config: pluginConfigFor(args),
             inputs,
             toolRoot,
             runId
@@ -142,9 +159,14 @@ export function parseRunExperimentArgs(argv) {
     let includeRealAgents;
     let kitCommand;
     let campaignPreset;
+    let contextBudgets;
+    let syntheticConfigPath;
+    const seenFlags = [];
     const commandTemplates = {};
     for (let index = 0; index < argv.length; index += 1) {
         const arg = argv[index];
+        if (arg !== undefined && arg.startsWith("--"))
+            seenFlags.push(arg);
         if (arg === "--experiment") {
             experimentId = readRequiredValue(argv, ++index, "--experiment");
         }
@@ -160,6 +182,12 @@ export function parseRunExperimentArgs(argv) {
         else if (arg === "--project-profiles") {
             projectProfilesPath = readRequiredValue(argv, ++index, "--project-profiles");
         }
+        else if (arg === "--synthetic-config") {
+            if (syntheticConfigPath !== undefined) {
+                throw new Error("--synthetic-config may be supplied only once.");
+            }
+            syntheticConfigPath = readRequiredValue(argv, ++index, "--synthetic-config");
+        }
         else if (arg === "--case") {
             caseIds.push(...splitList(readRequiredValue(argv, ++index, "--case")));
         }
@@ -202,6 +230,9 @@ export function parseRunExperimentArgs(argv) {
         else if (arg === "--kit-command") {
             kitCommand = readRequiredValue(argv, ++index, "--kit-command");
         }
+        else if (arg === "--context-budgets") {
+            contextBudgets = parseContextBudgetsCliValue(readRequiredValue(argv, ++index, "--context-budgets"));
+        }
         else if (arg === "--campaign") {
             campaignPreset = parseWarmIndexCampaignPresetId(readRequiredValue(argv, ++index, "--campaign"));
         }
@@ -216,7 +247,29 @@ export function parseRunExperimentArgs(argv) {
     if (!experimentId) {
         throw new Error("Usage: --experiment <id> [--target <path>] [--out <directory>]");
     }
-    const KIT_COMMAND_PLUGIN_IDS = [warmIndexReusePlugin.metadata.id, incrementalChangeStalenessPlugin.metadata.id];
+    const KIT_COMMAND_PLUGIN_IDS = [
+        warmIndexReusePlugin.metadata.id,
+        incrementalChangeStalenessPlugin.metadata.id,
+        contextWindowScalingPlugin.metadata.id
+    ];
+    if (contextBudgets !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
+        throw new Error(`--context-budgets is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
+    }
+    if (syntheticConfigPath !== undefined && experimentId !== contextWindowScalingPlugin.metadata.id) {
+        throw new Error(`--synthetic-config is only supported for --experiment ${contextWindowScalingPlugin.metadata.id}.`);
+    }
+    if (experimentId === contextWindowScalingPlugin.metadata.id) {
+        const unsupported = [...new Set(seenFlags.filter((flag) => !CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS.includes(flag)))];
+        if (unsupported.length > 0) {
+            throw new Error(`${unsupported.join(", ")} ${unsupported.length === 1 ? "is" : "are"} not supported for --experiment ${experimentId}; supported options: ${CONTEXT_WINDOW_SCALING_ALLOWED_FLAGS.join(", ")}.`);
+        }
+        if (seenFlags.includes("--case") && caseIds.length === 0) {
+            throw new Error("--case must list at least one case id.");
+        }
+        if (syntheticConfigPath !== undefined && seenFlags.includes("--case")) {
+            throw new Error("--case and --synthetic-config are mutually exclusive; the synthetic config owns the generated case set.");
+        }
+    }
     if (kitCommand !== undefined && !KIT_COMMAND_PLUGIN_IDS.includes(experimentId)) {
         throw new Error(`--kit-command is only supported for --experiment ${KIT_COMMAND_PLUGIN_IDS.join(" or ")}.`);
     }
@@ -238,6 +291,7 @@ export function parseRunExperimentArgs(argv) {
         experimentId,
         targetPath,
         outDir,
+        syntheticConfigPath,
         config: withoutUndefined({
             casesPath,
             projectProfilesPath,
@@ -253,7 +307,8 @@ export function parseRunExperimentArgs(argv) {
             includeRealAgents,
             commandTemplates: Object.keys(commandTemplates).length > 0 ? commandTemplates : undefined,
             kitCommand,
-            campaignPreset
+            campaignPreset,
+            contextBudgets
         })
     };
 }
@@ -279,7 +334,7 @@ function generateExperimentRunId(pluginId) {
     const timestamp = new Date().toISOString().replace(/[:.]/g, "-");
     return `${pluginId}-${timestamp}`;
 }
-async function loadPluginInputs(args, toolRoot, context) {
+async function loadPluginInputs(args, toolRoot, context, outputRoot) {
     if (args.experimentId === contextStrategyComparisonPlugin.metadata.id) {
         const validation = contextStrategyComparisonPlugin.validateConfig(args.config);
         if (!validation.valid || !validation.config) {
@@ -297,8 +352,64 @@ async function loadPluginInputs(args, toolRoot, context) {
         }
         return loadCasesAndProjectProfiles(args, toolRoot, context);
     }
+    if (args.experimentId === contextWindowScalingPlugin.metadata.id) {
+        return loadContextWindowScalingInputs(args, toolRoot, context, outputRoot);
+    }
     return undefined;
 }
+// --case is a selection over the frozen bundled catalog (catalog order, not argument order);
+// the rest of the CLI config is the plugin's own ContextWindowScalingConfig.
+function pluginConfigFor(args) {
+    if (args.experimentId !== contextWindowScalingPlugin.metadata.id)
+        return args.config;
+    const { caseIds: _caseIds, ...config } = args.config;
+    return config;
+}
+async function loadContextWindowScalingInputs(args, toolRoot, context, outputRoot) {
+    const validation = contextWindowScalingPlugin.validateConfig(pluginConfigFor(args));
+    if (!validation.valid || !validation.config) {
+        throw new Error(`Invalid context window scaling config: ${validation.errors.join("; ")}`);
+    }
+    if (args.syntheticConfigPath !== undefined) {
+        if (outputRoot === undefined) {
+            throw new Error("Internal error: synthetic generation requires a resolved experiment output root.");
+        }
+        const configPath = path.resolve(context.invocationCwd, args.syntheticConfigPath);
+        const syntheticRepositoryConfig = await readSyntheticConfigJson(configPath, args.syntheticConfigPath);
+        const prepared = prepareSyntheticContextWindowScalingInputs({ syntheticRepositoryConfig, outputRoot });
+        return { cases: prepared.cases, projectProfiles: prepared.projectProfiles, env: process.env };
+    }
+    const catalog = await readEvaluationCases(resolvePackageResource(context, CONTEXT_WINDOW_SCALING_CASES_RESOURCE), toolRoot);
+    const requested = args.config.caseIds;
+    if (requested === undefined)
+        return { cases: catalog, env: process.env };
+    const duplicates = [...new Set(requested.filter((id, position) => requested.indexOf(id) !== position))];
+    if (duplicates.length > 0) {
+        throw new Error(`--case contains duplicate case id(s): ${duplicates.join(", ")}.`);
+    }
+    const known = new Set(catalog.map((evaluationCase) => evaluationCase.id));
+    const unknown = requested.filter((id) => !known.has(id));
+    if (unknown.length > 0) {
+        throw new Error(`--case contains unknown case id(s): ${unknown.join(", ")}. Expected one of ${catalog.map((evaluationCase) => evaluationCase.id).join(", ")}.`);
+    }
+    return { cases: catalog.filter((evaluationCase) => requested.includes(evaluationCase.id)), env: process.env };
+}
+// The command only reads and parses the caller-supplied file; schema validation belongs to the synthetic planner.
+async function readSyntheticConfigJson(resolvedPath, displayPath) {
+    let text;
+    try {
+        text = await readFile(resolvedPath, "utf8");
+    }
+    catch (error) {
+        throw new Error(`Cannot read --synthetic-config ${displayPath}: ${error instanceof Error ? error.message : String(error)}`);
+    }
+    try {
+        return JSON.parse(text);
+    }
+    catch (error) {
+        throw new Error(`--synthetic-config ${displayPath} is not valid JSON: ${error instanceof Error ? error.message : String(error)}`);
+    }
+}
 // Shared by both registered plugins: explicit paths resolve against toolRoot, defaults come from
 // bundled package resources so they do not depend on the invocation cwd.
 async function loadCasesAndProjectProfiles(args, toolRoot, context) {

package/dist/src/evaluation/syntheticRepository/config.js

@@ -0,0 +1,304 @@
+import { TASK_LOCALITIES } from "../types.js";
+import { SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION, SYNTHETIC_REPOSITORY_LANGUAGES, SyntheticRepositoryConfigError, } from "./types.js";
+export const SYNTHETIC_REPOSITORY_LIMITS = {
+    maxCases: 64,
+    caseIdMaxLength: 64,
+    seedMaxLength: 256,
+    sourceFileCount: { min: 1, max: 10000 },
+    moduleDepth: { min: 1, max: 64 },
+    internalImportCount: { min: 0, max: 100000 },
+    symbolCount: { min: 1, max: 100000 },
+    testFileCount: { min: 0, max: 10000 },
+    repeatedPatternCount: { min: 0, max: 100000 },
+    broadChangeMinSourceFiles: 4,
+    crossModuleMinSourceFiles: 2,
+};
+/** Fixed field order of one case; also the canonical serialization order. */
+export const SYNTHETIC_REPOSITORY_CASE_FIELDS = [
+    "id",
+    "language",
+    "seed",
+    "sourceFileCount",
+    "moduleDepth",
+    "internalImportCount",
+    "symbolCount",
+    "testFileCount",
+    "taskLocality",
+    "repeatedPatternCount",
+];
+const COUNT_FIELDS = [
+    "sourceFileCount",
+    "moduleDepth",
+    "internalImportCount",
+    "symbolCount",
+    "testFileCount",
+    "repeatedPatternCount",
+];
+const CASE_ID_PATTERN = /^[A-Za-z0-9][A-Za-z0-9._-]*$/;
+const WINDOWS_DEVICE_NAME_PATTERN = /^(con|prn|aux|nul|com[1-9]|lpt[1-9])$/i;
+/** Locale-independent ordering by UTF-16 code unit. */
+export function compareCodeUnits(left, right) {
+    if (left < right)
+        return -1;
+    if (left > right)
+        return 1;
+    return 0;
+}
+/**
+ * Maximum number of unique directed acyclic import edges among `sourceFileCount` modules whose longest
+ * dependency chain has exactly `moduleDepth` modules: the edge count of the balanced complete
+ * multipartite arrangement, (N^2 - sum of squared layer sizes) / 2.
+ */
+export function maxImportEdges(sourceFileCount, moduleDepth) {
+    const base = Math.floor(sourceFileCount / moduleDepth);
+    const larger = sourceFileCount % moduleDepth;
+    const sumOfSquares = larger * (base + 1) * (base + 1) + (moduleDepth - larger) * base * base;
+    return (sourceFileCount * sourceFileCount - sumOfSquares) / 2;
+}
+/** Returns the case in canonical field order. Assumes the spec is valid. */
+export function orderCaseSpec(spec) {
+    return {
+        id: spec.id,
+        language: spec.language,
+        seed: spec.seed,
+        sourceFileCount: spec.sourceFileCount,
+        moduleDepth: spec.moduleDepth,
+        internalImportCount: spec.internalImportCount,
+        symbolCount: spec.symbolCount,
+        testFileCount: spec.testFileCount,
+        taskLocality: spec.taskLocality,
+        repeatedPatternCount: spec.repeatedPatternCount,
+    };
+}
+function isPlainObject(value) {
+    if (typeof value !== "object" || value === null || Array.isArray(value))
+        return false;
+    const prototype = Object.getPrototypeOf(value);
+    return prototype === Object.prototype || prototype === null;
+}
+function hasOwn(value, key) {
+    return Object.prototype.hasOwnProperty.call(value, key);
+}
+function describeValue(value) {
+    if (typeof value === "string") {
+        return JSON.stringify(value.length > 40 ? `${value.slice(0, 40)}...` : value);
+    }
+    if (typeof value === "number" || typeof value === "boolean")
+        return String(value);
+    if (value === null)
+        return "null";
+    if (Array.isArray(value))
+        return "array";
+    return typeof value;
+}
+function hasLoneSurrogate(value) {
+    for (let index = 0; index < value.length; index += 1) {
+        const unit = value.charCodeAt(index);
+        if (unit >= 0xd800 && unit <= 0xdbff) {
+            const next = value.charCodeAt(index + 1);
+            if (next >= 0xdc00 && next <= 0xdfff) {
+                index += 1;
+                continue;
+            }
+            return true;
+        }
+        if (unit >= 0xdc00 && unit <= 0xdfff)
+            return true;
+    }
+    return false;
+}
+function validateCaseId(value, label, errors) {
+    if (typeof value !== "string") {
+        errors.push(`${label}.id: must be a string (received ${describeValue(value)}).`);
+        return false;
+    }
+    const before = errors.length;
+    if (value.length < 1 || value.length > SYNTHETIC_REPOSITORY_LIMITS.caseIdMaxLength) {
+        errors.push(`${label}.id: length must be between 1 and ${SYNTHETIC_REPOSITORY_LIMITS.caseIdMaxLength}.`);
+    }
+    else if (!CASE_ID_PATTERN.test(value)) {
+        errors.push(`${label}.id: must begin with an ASCII letter or digit and contain only ASCII letters, digits, ".", "_" and "-" (received ${describeValue(value)}).`);
+    }
+    else {
+        if (value.endsWith(".")) {
+            errors.push(`${label}.id: must not end with ".".`);
+        }
+        if (WINDOWS_DEVICE_NAME_PATTERN.test(value.split(".")[0])) {
+            errors.push(`${label}.id: must not use a reserved device name (received ${describeValue(value)}).`);
+        }
+    }
+    return errors.length === before;
+}
+function validateSeed(value, label, errors) {
+    if (typeof value !== "string") {
+        errors.push(`${label}.seed: must be a string (received ${describeValue(value)}).`);
+        return false;
+    }
+    const before = errors.length;
+    if (!/\S/.test(value)) {
+        errors.push(`${label}.seed: must contain at least one non-whitespace character.`);
+    }
+    if (value.length > SYNTHETIC_REPOSITORY_LIMITS.seedMaxLength) {
+        errors.push(`${label}.seed: must be at most ${SYNTHETIC_REPOSITORY_LIMITS.seedMaxLength} UTF-16 code units.`);
+    }
+    if (hasLoneSurrogate(value)) {
+        errors.push(`${label}.seed: must be well-formed Unicode (lone surrogate found).`);
+    }
+    return errors.length === before;
+}
+function validateCount(value, field, label, errors) {
+    const { min, max } = SYNTHETIC_REPOSITORY_LIMITS[field];
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value < min || value > max) {
+        errors.push(`${label}.${field}: must be an integer between ${min} and ${max} (received ${describeValue(value)}).`);
+        return false;
+    }
+    return true;
+}
+function validateCase(value, label, errors) {
+    if (!isPlainObject(value)) {
+        errors.push(`${label}: must be an object (received ${describeValue(value)}).`);
+        return undefined;
+    }
+    const before = errors.length;
+    const allowed = new Set(SYNTHETIC_REPOSITORY_CASE_FIELDS);
+    for (const key of Object.keys(value).sort(compareCodeUnits)) {
+        if (!allowed.has(key))
+            errors.push(`${label}: unknown field ${JSON.stringify(key)}.`);
+    }
+    for (const field of SYNTHETIC_REPOSITORY_CASE_FIELDS) {
+        if (!hasOwn(value, field))
+            errors.push(`${label}: missing required field ${field}.`);
+    }
+    const idOk = hasOwn(value, "id") && validateCaseId(value.id, label, errors);
+    const languageOk = hasOwn(value, "language") &&
+        SYNTHETIC_REPOSITORY_LANGUAGES.includes(value.language);
+    if (hasOwn(value, "language") && !languageOk) {
+        errors.push(`${label}.language: must be one of ${SYNTHETIC_REPOSITORY_LANGUAGES.join(", ")} (received ${describeValue(value.language)}).`);
+    }
+    const seedOk = hasOwn(value, "seed") && validateSeed(value.seed, label, errors);
+    const countOk = new Map();
+    for (const field of COUNT_FIELDS) {
+        countOk.set(field, hasOwn(value, field) && validateCount(value[field], field, label, errors));
+    }
+    const localityOk = hasOwn(value, "taskLocality") && TASK_LOCALITIES.includes(value.taskLocality);
+    if (hasOwn(value, "taskLocality") && !localityOk) {
+        errors.push(`${label}.taskLocality: must be one of ${TASK_LOCALITIES.join(", ")} (received ${describeValue(value.taskLocality)}).`);
+    }
+    const sourceFileCount = value.sourceFileCount;
+    const moduleDepth = value.moduleDepth;
+    const internalImportCount = value.internalImportCount;
+    const symbolCount = value.symbolCount;
+    const taskLocality = value.taskLocality;
+    const sourceOk = countOk.get("sourceFileCount") === true;
+    const depthOk = countOk.get("moduleDepth") === true;
+    const importsOk = countOk.get("internalImportCount") === true;
+    const symbolsOk = countOk.get("symbolCount") === true;
+    if (sourceOk && depthOk) {
+        if (moduleDepth > sourceFileCount) {
+            errors.push(`${label}.moduleDepth: must not exceed sourceFileCount (${moduleDepth} > ${sourceFileCount}).`);
+        }
+        else if (importsOk) {
+            if (moduleDepth > 1 && internalImportCount < moduleDepth - 1) {
+                errors.push(`${label}.internalImportCount: must be at least moduleDepth - 1 (${moduleDepth - 1}) to realize moduleDepth ${moduleDepth}.`);
+            }
+            const capacity = maxImportEdges(sourceFileCount, moduleDepth);
+            if (internalImportCount > capacity) {
+                errors.push(`${label}.internalImportCount: exceeds the ${capacity} unique acyclic edges possible for sourceFileCount ${sourceFileCount} with exact moduleDepth ${moduleDepth}.`);
+            }
+        }
+    }
+    if (sourceOk && symbolsOk && symbolCount < sourceFileCount) {
+        errors.push(`${label}.symbolCount: must be at least sourceFileCount (${symbolCount} < ${sourceFileCount}).`);
+    }
+    if (sourceOk && localityOk) {
+        if (taskLocality === "cross-module") {
+            if (sourceFileCount < SYNTHETIC_REPOSITORY_LIMITS.crossModuleMinSourceFiles) {
+                errors.push(`${label}.taskLocality: cross-module requires sourceFileCount >= 2.`);
+            }
+            if (importsOk && internalImportCount < 1) {
+                errors.push(`${label}.taskLocality: cross-module requires internalImportCount >= 1.`);
+            }
+        }
+        else if (taskLocality === "broad-change" && sourceFileCount < SYNTHETIC_REPOSITORY_LIMITS.broadChangeMinSourceFiles) {
+            errors.push(`${label}.taskLocality: broad-change requires sourceFileCount >= 4.`);
+        }
+    }
+    if (errors.length !== before || !idOk || !languageOk || !seedOk || !localityOk)
+        return undefined;
+    return orderCaseSpec(value);
+}
+/** Validates one case spec and returns it in canonical field order. Never throws, never repairs. */
+export function validateSyntheticRepositoryCaseSpec(input) {
+    const errors = [];
+    const spec = validateCase(input, "case", errors);
+    if (errors.length > 0 || spec === undefined)
+        return { ok: false, errors };
+    return { ok: true, errors: [], spec };
+}
+/**
+ * Strictly validates an unknown value as SyntheticRepositoryConfigV1. On success the returned config is
+ * canonical: cases sorted ascending by id (code-unit order), fixed field order, values untouched.
+ */
+export function validateSyntheticRepositoryConfig(input) {
+    const errors = [];
+    if (!isPlainObject(input)) {
+        return { ok: false, errors: [`config: must be an object (received ${describeValue(input)}).`] };
+    }
+    for (const key of Object.keys(input).sort(compareCodeUnits)) {
+        if (key !== "schemaVersion" && key !== "cases")
+            errors.push(`config: unknown field ${JSON.stringify(key)}.`);
+    }
+    if (!hasOwn(input, "schemaVersion")) {
+        errors.push("config: missing required field schemaVersion.");
+    }
+    else if (input.schemaVersion !== SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION) {
+        errors.push(`config.schemaVersion: must be ${JSON.stringify(SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION)} (received ${describeValue(input.schemaVersion)}).`);
+    }
+    if (!hasOwn(input, "cases")) {
+        errors.push("config: missing required field cases.");
+        return { ok: false, errors };
+    }
+    const cases = input.cases;
+    if (!Array.isArray(cases)) {
+        errors.push(`config.cases: must be an array (received ${describeValue(cases)}).`);
+        return { ok: false, errors };
+    }
+    if (cases.length < 1 || cases.length > SYNTHETIC_REPOSITORY_LIMITS.maxCases) {
+        errors.push(`config.cases: must contain between 1 and ${SYNTHETIC_REPOSITORY_LIMITS.maxCases} cases (received ${cases.length}).`);
+        return { ok: false, errors };
+    }
+    const valid = [];
+    const firstIndexById = new Map();
+    cases.forEach((candidate, index) => {
+        const label = `cases[${index}]`;
+        const spec = validateCase(candidate, label, errors);
+        if (spec)
+            valid.push(spec);
+        const id = isPlainObject(candidate) ? candidate.id : undefined;
+        if (typeof id === "string") {
+            const key = id.toLowerCase();
+            const first = firstIndexById.get(key);
+            if (first === undefined) {
+                firstIndexById.set(key, index);
+            }
+            else {
+                errors.push(`${label}.id: duplicates cases[${first}].id (ids are compared without regard to letter case).`);
+            }
+        }
+    });
+    if (errors.length > 0)
+        return { ok: false, errors };
+    const sorted = [...valid].sort((left, right) => compareCodeUnits(left.id, right.id));
+    return {
+        ok: true,
+        errors: [],
+        config: { schemaVersion: SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION, cases: sorted },
+    };
+}
+/** Validates then returns the canonical configuration; throws SyntheticRepositoryConfigError when invalid. */
+export function normalizeSyntheticRepositoryConfig(input) {
+    const result = validateSyntheticRepositoryConfig(input);
+    if (!result.ok)
+        throw new SyntheticRepositoryConfigError(result.errors);
+    return result.config;
+}

package/dist/src/evaluation/syntheticRepository/evaluationCase.js

@@ -0,0 +1,43 @@
+import { describeSyntheticTask } from "./layout.js";
+/** The plan's answer key expressed in the existing benchmark answer-key vocabulary (no second correctness contract). */
+export function buildSyntheticAnswerKey(plan) {
+    const { factTextById } = describeSyntheticTask(plan);
+    return {
+        expectedFiles: [...plan.answerKey.expectedFiles],
+        expectedSymbols: [...plan.answerKey.expectedSymbols],
+        expectedFacts: plan.answerKey.facts.map((fact) => ({
+            id: fact.factId,
+            text: factTextById.get(fact.factId),
+            weight: fact.weight,
+            required: fact.required,
+        })),
+        expectedContextTargets: plan.answerKey.expectedContextTargets.map((target) => ({
+            file: target.file,
+            symbols: [...target.symbols],
+            required: target.required,
+        })),
+        minimumCorrectFacts: plan.answerKey.minimumCorrectFacts,
+    };
+}
+/**
+ * Builds the existing EvaluationCase directly in memory (never through readEvaluationCases, which owns
+ * repository-contained JSON cases). `targetRoot` is the stable logical root from the manifest; only
+ * `absoluteTargetRoot` carries the physical generated repository directory.
+ */
+export function buildSyntheticEvaluationCase(plan, manifest, repositoryRoot) {
+    const text = describeSyntheticTask(plan);
+    return {
+        id: plan.task.taskId,
+        title: text.title,
+        benchmarkProject: plan.logicalProjectId,
+        targetRoot: manifest.logicalTargetRoot,
+        sourceRoots: [...manifest.sourceRoots],
+        query: text.query,
+        expectedFiles: [...plan.answerKey.expectedFiles],
+        expectedSymbols: [...plan.answerKey.expectedSymbols],
+        rawIncludeGlobs: [...manifest.rawIncludeGlobs],
+        answerKey: buildSyntheticAnswerKey(plan),
+        taskLocality: plan.task.locality,
+        absoluteTargetRoot: repositoryRoot,
+    };
+}

package/dist/src/evaluation/syntheticRepository/identity.js

@@ -0,0 +1,41 @@
+import { createHash } from "node:crypto";
+import { normalizeSyntheticRepositoryConfig, orderCaseSpec, validateSyntheticRepositoryCaseSpec } from "./config.js";
+import { SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_ID, SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION, SYNTHETIC_REPOSITORY_IDENTITY_ALGORITHM_VERSION, SyntheticRepositoryConfigError, } from "./types.js";
+/** Canonical JSON text of one valid case: fixed key order, exact values. */
+export function canonicalSyntheticRepositoryCaseText(spec) {
+    return JSON.stringify(orderCaseSpec(spec));
+}
+/** Canonical JSON text of a normalized configuration (cases already sorted by id). */
+export function canonicalSyntheticRepositoryConfigText(config) {
+    return JSON.stringify({
+        schemaVersion: config.schemaVersion,
+        cases: config.cases.map(orderCaseSpec),
+    });
+}
+/**
+ * SHA-256 over the UTF-8 bytes of five newline-joined lines: schema id, schema version, identity
+ * algorithm version, a domain tag ("case" or "config") and the canonical JSON payload. JSON escapes every
+ * newline, so the payload cannot forge an earlier line.
+ */
+function hashDomain(domain, payload) {
+    const text = [
+        SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_ID,
+        SYNTHETIC_REPOSITORY_CONFIG_SCHEMA_VERSION,
+        SYNTHETIC_REPOSITORY_IDENTITY_ALGORITHM_VERSION,
+        domain,
+        payload,
+    ].join("\n");
+    return createHash("sha256").update(Buffer.from(text, "utf8")).digest("hex");
+}
+/** Per-case generation identity (lowercase 64-character SHA-256 hex). Throws on an invalid case. */
+export function computeSyntheticRepositoryCaseIdentity(input) {
+    const result = validateSyntheticRepositoryCaseSpec(input);
+    if (!result.ok)
+        throw new SyntheticRepositoryConfigError(result.errors);
+    return hashDomain("case", canonicalSyntheticRepositoryCaseText(result.spec));
+}
+/** Whole-configuration identity over the normalized form; input case order never matters. */
+export function computeSyntheticRepositoryConfigIdentity(input) {
+    const config = normalizeSyntheticRepositoryConfig(input);
+    return hashDomain("config", canonicalSyntheticRepositoryConfigText(config));
+}

package/dist/src/evaluation/syntheticRepository/index.js

@@ -0,0 +1,16 @@
+export * from "./types.js";
+export { SYNTHETIC_REPOSITORY_CASE_FIELDS, SYNTHETIC_REPOSITORY_LIMITS, compareCodeUnits, maxImportEdges, normalizeSyntheticRepositoryConfig, validateSyntheticRepositoryCaseSpec, validateSyntheticRepositoryConfig, } from "./config.js";
+export { canonicalSyntheticRepositoryCaseText, canonicalSyntheticRepositoryConfigText, computeSyntheticRepositoryCaseIdentity, computeSyntheticRepositoryConfigIdentity, } from "./identity.js";
+export { createSyntheticRepositoryPrng, syntheticRepositoryPrngInitialState } from "./prng.js";
+export { planTopology } from "./topology.js";
+export { planSyntheticRepositories, planSyntheticRepositoryCase, verifySyntheticRepositoryPlan } from "./planning.js";
+export { SyntheticRepositoryRenderError } from "./renderShared.js";
+export { renderTypeScriptRepository } from "./renderTypeScript.js";
+export { pythonModuleName, renderPythonRepository } from "./renderPython.js";
+export { renderSyntheticRepository } from "./renderRepository.js";
+export { inspectRepositoryFiles } from "./inspection.js";
+export { deriveSyntheticRepositoryLayout, describeSyntheticTask } from "./layout.js";
+export { SYNTHETIC_REPOSITORY_MANIFEST_FILE_NAME, SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_ID, SYNTHETIC_REPOSITORY_MANIFEST_SCHEMA_VERSION, SYNTHETIC_REPOSITORY_REPOSITORY_DIRECTORY, buildSyntheticRepositoryManifest, computeRepositoryContentIdentity, serializeSyntheticRepositoryManifest, validateSyntheticRepositoryManifest, } from "./manifest.js";
+export { verifySyntheticRepositoryMaterialization } from "./manifestVerification.js";
+export { buildSyntheticAnswerKey, buildSyntheticEvaluationCase } from "./evaluationCase.js";
+export { SyntheticRepositoryMaterializationError, materializeSyntheticRepository } from "./materialize.js";