@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment-identity.js

@@ -0,0 +1,257 @@
+import { readFile } from "node:fs/promises";
+import { isAbsolute, relative, resolve, sep } from "node:path";
+import { bindSubjectRoot, contentHash, valueHash } from "../core/index.js";
+import { ReflectiveOptimizerConfigSchema } from "../optimization/reflective.js";
+import { authoredDependencyGraphHash } from "./authored-dependencies.js";
+import { assertValidationArgumentsBound } from "./candidates.js";
+import { selectCases } from "./discover.js";
+import { evaluatorImplementationHash } from "./evaluator-identity.js";
+import { profileOrThrow } from "./experiment-context.js";
+import { caseRubricCliReferences, selectorsBySplit } from "./experiment-corpus.js";
+import { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes, LUX_VERSION, RuntimeCliReferenceSchema, referencedAnswererCli, referencedCli, runtimeCliIdentityHashForReferences, } from "./experiment-runtime-identity.js";
+import { loadEvalCase } from "./loader.js";
+import { boundExperimentDriverConfig, resolveAnswererReference, resolveDriverReference, } from "./orchestrator.js";
+import { runtimeIdentity } from "./provenance.js";
+import { validatePluginConfig } from "./validate-plugin-config.js";
+export const resolvedProfileSnapshot = async (execution) => {
+    const behaviorHashes = new Map();
+    const configuredSourceHashes = new Map();
+    const behaviorHash = (root) => {
+        const existing = behaviorHashes.get(root);
+        if (existing)
+            return existing;
+        const created = claudeUserBehaviorHash(root);
+        behaviorHashes.set(root, created);
+        return created;
+    };
+    return Object.fromEntries(await Promise.all(execution.loaded.campaign.profiles.map(async (profileId) => {
+        const profile = profileOrThrow(execution.config, profileId);
+        // Offline grader campaigns never launch a subject driver. Keep their
+        // identity independent of irrelevant local agent settings/plugins
+        // while still snapshotting the harness that performs regrading.
+        const driver = execution.loaded.campaign.evaluation.kind === "agent"
+            ? (profile.driver ?? execution.config.defaultDriver)
+            : undefined;
+        const driverCli = referencedCli(driver);
+        const harness = profile.harness ?? execution.config.harness;
+        const answerer = execution.loaded.campaign.evaluation.kind === "agent"
+            ? (profile.answerer ?? execution.config.defaultAnswerer)
+            : undefined;
+        const answererCli = referencedAnswererCli(answerer);
+        const driverUserBehaviorHash = driverCli?.name === "claude-code" ? await behaviorHash(claudeConfigRoot(driver)) : null;
+        const driverConfiguredSources = await configuredClaudeSourceHashes(driver, execution.config.rootDir, configuredSourceHashes);
+        let harnessCommand = harness?.command;
+        if (!harnessCommand) {
+            harnessCommand = harness?.provider === "codex" ? "codex" : "claude";
+        }
+        const capturesHarnessIdentity = Boolean(harness) || execution.loaded.campaign.evaluation.kind === "grader";
+        const harnessCliIdentity = capturesHarnessIdentity
+            ? await cliIdentity(harnessCommand, execution.config.rootDir)
+            : null;
+        return [
+            profileId,
+            {
+                id: profileId,
+                driver: driver ?? null,
+                driverCliIdentity: driverCli
+                    ? await cliIdentity(driverCli.command, execution.config.rootDir)
+                    : null,
+                ...(driverCli?.name === "claude-code"
+                    ? {
+                        driverClaudeConfigRoot: claudeConfigRoot(driver),
+                        driverUserBehaviorHash,
+                        driverConfiguredSources,
+                    }
+                    : {}),
+                answerer,
+                answererCliIdentity: answererCli
+                    ? await cliIdentity(answererCli.command, execution.config.rootDir)
+                    : null,
+                harness: harness ?? null,
+                harnessCliIdentity,
+            },
+        ];
+    })));
+};
+const trustedExecutableEntrypoints = async (execution, corpus) => {
+    const paths = new Set();
+    const add = (command) => {
+        if (!command || !isAbsolute(command))
+            return;
+        const fromRoot = relative(execution.config.rootDir, resolve(command));
+        if (fromRoot !== "" &&
+            fromRoot !== ".." &&
+            !fromRoot.startsWith(`..${sep}`) &&
+            !isAbsolute(fromRoot)) {
+            paths.add(fromRoot);
+        }
+    };
+    for (const validation of execution.loaded.campaign.subject.validate)
+        add(validation.command);
+    if (execution.loaded.campaign.optimizer) {
+        const optimizer = ReflectiveOptimizerConfigSchema.parse(execution.loaded.campaign.optimizer.config ?? {});
+        add(optimizer.command);
+    }
+    for (const profileId of execution.loaded.campaign.profiles) {
+        const profile = profileOrThrow(execution.config, profileId);
+        add(referencedCli(profile.driver ?? execution.config.defaultDriver)?.command);
+        add(referencedAnswererCli(profile.answerer ?? execution.config.defaultAnswerer)?.command);
+        add((profile.harness ?? execution.config.harness)?.command);
+    }
+    if (corpus.kind === "agent") {
+        const selected = [
+            ...new Map(selectorsBySplit(execution.loaded.campaign)
+                .flatMap((split) => selectCases(corpus.cases, split.selector))
+                .map((evalCase) => [evalCase.path, evalCase])).values(),
+        ];
+        for (const evalCase of selected) {
+            const authored = bindSubjectRoot(await loadEvalCase(evalCase.path), execution.loaded.subjectRoot);
+            add(referencedCli(authored.driver)?.command);
+            add(referencedAnswererCli(authored.answerer)?.command);
+            for (const reference of caseRubricCliReferences(execution, authored)) {
+                add(reference.command);
+            }
+        }
+    }
+    return paths;
+};
+export const runtimeCliReferences = async (execution, corpus, mode) => {
+    const references = new Map();
+    const add = (command, baseRoot) => {
+        if (!command)
+            return;
+        const reference = RuntimeCliReferenceSchema.parse({ command, baseRoot: resolve(baseRoot) });
+        references.set(`${reference.command}\u0000${reference.baseRoot}`, reference);
+    };
+    for (const validation of execution.loaded.campaign.subject.validate) {
+        add(validation.command, execution.loaded.subjectRoot);
+    }
+    if (mode === "optimize" && execution.loaded.campaign.optimizer) {
+        const optimizer = ReflectiveOptimizerConfigSchema.parse(execution.loaded.campaign.optimizer.config ?? {});
+        add(optimizer.command ?? (optimizer.provider === "codex" ? "codex" : "claude"), execution.loaded.directory);
+    }
+    if (corpus.kind === "grader") {
+        for (const profileId of execution.loaded.campaign.profiles) {
+            const profile = profileOrThrow(execution.config, profileId);
+            const harness = profile.harness ?? execution.config.harness;
+            add(harness?.command ?? (harness?.provider === "codex" ? "codex" : "claude"), execution.config.rootDir);
+        }
+    }
+    if (corpus.kind === "agent") {
+        const selected = [
+            ...new Map(selectorsBySplit(execution.loaded.campaign)
+                .flatMap((split) => selectCases(corpus.cases, split.selector))
+                .map((evalCase) => [evalCase.path, evalCase])).values(),
+        ];
+        for (const evalCase of selected) {
+            const authored = bindSubjectRoot(await loadEvalCase(evalCase.path), execution.loaded.subjectRoot);
+            for (const profileId of execution.loaded.campaign.profiles) {
+                const profile = profileOrThrow(execution.config, profileId);
+                const harness = profile.harness ?? execution.config.harness;
+                const driver = resolveDriverReference(authored, profile.driver ?? execution.config.defaultDriver);
+                const answerer = resolveAnswererReference(authored, profile.answerer ?? execution.config.defaultAnswerer, harness);
+                add(referencedCli(driver)?.command, execution.config.rootDir);
+                add(referencedAnswererCli(answerer)?.command, execution.config.rootDir);
+            }
+            for (const reference of caseRubricCliReferences(execution, authored)) {
+                add(reference.command, execution.config.rootDir);
+            }
+        }
+    }
+    return [...references.values()].sort((left, right) => {
+        const commandOrder = left.command.localeCompare(right.command);
+        return commandOrder || left.baseRoot.localeCompare(right.baseRoot);
+    });
+};
+const runtimeCliIdentityHash = async (execution, corpus, mode) => runtimeCliIdentityHashForReferences(await runtimeCliReferences(execution, corpus, mode));
+const authoredConfigDependencyHash = async (execution, corpus) => {
+    const configRoot = resolve(execution.config.rootDir);
+    const runsRoot = resolve(execution.runsRoot ?? execution.config.runsRoot);
+    const runsFromConfig = relative(configRoot, runsRoot);
+    if (runsFromConfig === "") {
+        throw new Error("experiment runsRoot cannot be the Lux config root");
+    }
+    const entrypoints = await trustedExecutableEntrypoints(execution, corpus);
+    const add = (path) => {
+        if (path)
+            entrypoints.add(relative(configRoot, resolve(path)));
+    };
+    add(execution.config.configPath);
+    add(execution.loaded.path);
+    if (corpus.kind === "agent") {
+        for (const split of selectorsBySplit(execution.loaded.campaign)) {
+            for (const evalCase of selectCases(corpus.cases, split.selector))
+                add(evalCase.path);
+        }
+    }
+    else {
+        add(execution.loaded.graderAnnotations?.path);
+    }
+    return authoredDependencyGraphHash(configRoot, [...entrypoints]);
+};
+const validateAgentCaseProfileDrivers = async (execution, corpus) => {
+    const selected = [
+        ...new Map(selectorsBySplit(execution.loaded.campaign)
+            .flatMap((split) => selectCases(corpus.cases, split.selector))
+            .map((evalCase) => [evalCase.path, evalCase])).values(),
+    ];
+    for (const evalCase of selected) {
+        const authored = bindSubjectRoot(await loadEvalCase(evalCase.path), execution.loaded.subjectRoot);
+        for (const profileId of execution.loaded.campaign.profiles) {
+            const profile = profileOrThrow(execution.config, profileId);
+            const reference = resolveDriverReference(authored, profile.driver ?? execution.config.defaultDriver);
+            const name = typeof reference === "string" ? reference : reference.name;
+            const rawConfig = typeof reference === "object" ? reference.config : undefined;
+            boundExperimentDriverConfig(name, rawConfig, "identity-check");
+            const plugin = execution.registry.getDriver(name);
+            if (!plugin) {
+                throw new Error(`case '${evalCase.id}' profile '${profileId}' references unknown driver '${name}'`);
+            }
+            const validated = validatePluginConfig(plugin.configSchema, rawConfig, `case '${evalCase.id}' profile '${profileId}' driver '${name}'`);
+            boundExperimentDriverConfig(name, validated, "identity-check");
+        }
+    }
+};
+export const evaluationIdentity = async (execution, corpus, mode, boundRuntimeCliIdentityHash) => {
+    const profiles = await resolvedProfileSnapshot(execution);
+    if (corpus.kind === "agent") {
+        await validateAgentCaseProfileDrivers(execution, corpus);
+    }
+    const configSourceHash = execution.config.configPath
+        ? contentHash(await readFile(execution.config.configPath))
+        : null;
+    const configDependencyHash = await authoredConfigDependencyHash(execution, corpus);
+    const optimizerConfig = mode === "optimize" && execution.loaded.campaign.optimizer
+        ? ReflectiveOptimizerConfigSchema.parse(execution.loaded.campaign.optimizer.config ?? {})
+        : undefined;
+    let optimizerCommand;
+    if (optimizerConfig) {
+        optimizerCommand = optimizerConfig.command;
+        if (!optimizerCommand) {
+            optimizerCommand = optimizerConfig.provider === "codex" ? "codex" : "claude";
+        }
+    }
+    return valueHash({
+        evaluationKind: execution.loaded.campaign.evaluation.kind,
+        evaluatorVersion: 1,
+        luxVersion: LUX_VERSION,
+        evaluatorImplementationHash: await evaluatorImplementationHash(),
+        runtime: runtimeIdentity(),
+        corpusHash: corpus.hash,
+        profiles,
+        configSourceHash,
+        configDependencyHash,
+        validationCliIdentities: await Promise.all(execution.loaded.campaign.subject.validate.map(async (validation) => {
+            assertValidationArgumentsBound(execution.loaded.subjectRoot, validation.args);
+            return {
+                args: validation.args,
+                cli: await cliIdentity(validation.command, execution.loaded.subjectRoot),
+            };
+        })),
+        optimizerCliIdentity: optimizerCommand
+            ? await cliIdentity(optimizerCommand, execution.loaded.directory)
+            : null,
+        runtimeCliIdentityHash: boundRuntimeCliIdentityHash ?? (await runtimeCliIdentityHash(execution, corpus, mode)),
+    });
+};
+//# sourceMappingURL=experiment-identity.js.map

package/lib/orchestrator/experiment-identity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-identity.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-identity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AAC/D,OAAO,EAAE,eAAe,EAAE,WAAW,EAAE,SAAS,EAAE,MAAM,kBAAkB,CAAC;AAC3E,OAAO,EAAE,+BAA+B,EAAE,MAAM,+BAA+B,CAAC;AAChF,OAAO,EAAE,2BAA2B,EAAE,MAAM,4BAA4B,CAAC;AACzE,OAAO,EAAE,8BAA8B,EAAE,MAAM,iBAAiB,CAAC;AACjE,OAAO,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAC5C,OAAO,EAAE,2BAA2B,EAAE,MAAM,yBAAyB,CAAC;AAEtE,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EAAE,uBAAuB,EAAE,gBAAgB,EAAE,MAAM,wBAAwB,CAAC;AACnF,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,WAAW,EACX,4BAA4B,EAC5B,WAAW,EAEX,yBAAyB,EACzB,qBAAqB,EACrB,aAAa,EACb,mCAAmC,GACpC,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EACL,2BAA2B,EAC3B,wBAAwB,EACxB,sBAAsB,GACvB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AAClD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AAEnE,MAAM,CAAC,MAAM,uBAAuB,GAAG,KAAK,EAC1C,SAAmC,EACD,EAAE;IACpC,MAAM,cAAc,GAAG,IAAI,GAAG,EAAkC,CAAC;IACjE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClE,MAAM,YAAY,GAAG,CAAC,IAAY,EAA0B,EAAE;QAC5D,MAAM,QAAQ,GAAG,cAAc,CAAC,GAAG,CAAC,IAAI,CAAC,CAAC;QAC1C,IAAI,QAAQ;YAAE,OAAO,QAAQ,CAAC;QAC9B,MAAM,OAAO,GAAG,sBAAsB,CAAC,IAAI,CAAC,CAAC;QAC7C,cAAc,CAAC,GAAG,CAAC,IAAI,EAAE,OAAO,CAAC,CAAC;QAClC,OAAO,OAAO,CAAC;IACjB,CAAC,CAAC;IACF,OAAO,MAAM,CAAC,WAAW,CACvB,MAAM,OAAO,CAAC,GAAG,CACf,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;QACzD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC5D,qEAAqE;QACrE,kEAAkE;QAClE,gEAAgE;QAChE,MAAM,MAAM,GACV,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO;YACnD,CAAC,CAAC,CAAC,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC;YACpD,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;QACxC,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;QAC5D,MAAM,QAAQ,GACZ,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,OAAO;YACnD,CAAC,CAAC,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC;YACxD,CAAC,CAAC,SAAS,CAAC;QAChB,MAAM,WAAW,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;QACpD,MAAM,sBAAsB,GAC1B,SAAS,EAAE,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,MAAM,YAAY,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;QAC1F,MAAM,uBAAuB,GAAG,MAAM,4BAA4B,CAChE,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,OAAO,EACxB,sBAAsB,CACvB,CAAC;QACF,IAAI,cAAc,GAAG,OAAO,EAAE,OAAO,CAAC;QACtC,IAAI,CAAC,cAAc,EAAE,CAAC;YACpB,cAAc,GAAG,OAAO,EAAE,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QACtE,CAAC;QACD,MAAM,uBAAuB,GAC3B,OAAO,CAAC,OAAO,CAAC,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,CAAC;QAC7E,MAAM,kBAAkB,GAAG,uBAAuB;YAChD,CAAC,CAAC,MAAM,WAAW,CAAC,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YAC7D,CAAC,CAAC,IAAI,CAAC;QACT,OAAO;YACL,SAAS;YACT;gBACE,EAAE,EAAE,SAAS;gBACb,MAAM,EAAE,MAAM,IAAI,IAAI;gBACtB,iBAAiB,EAAE,SAAS;oBAC1B,CAAC,CAAC,MAAM,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAChE,CAAC,CAAC,IAAI;gBACR,GAAG,CAAC,SAAS,EAAE,IAAI,KAAK,aAAa;oBACnC,CAAC,CAAC;wBACE,sBAAsB,EAAE,gBAAgB,CAAC,MAAM,CAAC;wBAChD,sBAAsB;wBACtB,uBAAuB;qBACxB;oBACH,CAAC,CAAC,EAAE,CAAC;gBACP,QAAQ;gBACR,mBAAmB,EAAE,WAAW;oBAC9B,CAAC,CAAC,MAAM,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAClE,CAAC,CAAC,IAAI;gBACR,OAAO,EAAE,OAAO,IAAI,IAAI;gBACxB,kBAAkB;aACnB;SACF,CAAC;IACJ,CAAC,CAAC,CACH,CACF,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,4BAA4B,GAAG,KAAK,EACxC,SAAmC,EACnC,MAAwB,EACF,EAAE;IACxB,MAAM,KAAK,GAAG,IAAI,GAAG,EAAU,CAAC;IAChC,MAAM,GAAG,GAAG,CAAC,OAA2B,EAAQ,EAAE;QAChD,IAAI,CAAC,OAAO,IAAI,CAAC,UAAU,CAAC,OAAO,CAAC;YAAE,OAAO;QAC7C,MAAM,QAAQ,GAAG,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,OAAO,CAAC,CAAC,CAAC;QACtE,IACE,QAAQ,KAAK,EAAE;YACf,QAAQ,KAAK,IAAI;YACjB,CAAC,QAAQ,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC;YAChC,CAAC,UAAU,CAAC,QAAQ,CAAC,EACrB,CAAC;YACD,KAAK,CAAC,GAAG,CAAC,QAAQ,CAAC,CAAC;QACtB,CAAC;IACH,CAAC,CAAC;IACF,KAAK,MAAM,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ;QAAE,GAAG,CAAC,UAAU,CAAC,OAAO,CAAC,CAAC;IAC7F,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QACxC,MAAM,SAAS,GAAG,+BAA+B,CAAC,KAAK,CACrD,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CACjD,CAAC;QACF,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;IACzB,CAAC;IACD,KAAK,MAAM,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;QAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC5D,GAAG,CAAC,aAAa,CAAC,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CAAC,EAAE,OAAO,CAAC,CAAC;QAC9E,GAAG,CAAC,qBAAqB,CAAC,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,CAAC,EAAE,OAAO,CAAC,CAAC;QAC1F,GAAG,CAAC,CAAC,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,EAAE,OAAO,CAAC,CAAC;IAC9D,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG;YACf,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;iBACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;iBAC7D,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;SACX,CAAC;QACF,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,eAAe,CAC9B,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EACjC,SAAS,CAAC,MAAM,CAAC,WAAW,CAC7B,CAAC;YACF,GAAG,CAAC,aAAa,CAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,OAAO,CAAC,CAAC;YAC7C,GAAG,CAAC,qBAAqB,CAAC,QAAQ,CAAC,QAAQ,CAAC,EAAE,OAAO,CAAC,CAAC;YACvD,KAAK,MAAM,SAAS,IAAI,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACrE,GAAG,CAAC,SAAS,CAAC,OAAO,CAAC,CAAC;YACzB,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,KAAK,CAAC;AACf,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,oBAAoB,GAAG,KAAK,EACvC,SAAmC,EACnC,MAAwB,EACxB,IAA+B,EACC,EAAE;IAClC,MAAM,UAAU,GAAG,IAAI,GAAG,EAA+B,CAAC;IAC1D,MAAM,GAAG,GAAG,CAAC,OAA2B,EAAE,QAAgB,EAAQ,EAAE;QAClE,IAAI,CAAC,OAAO;YAAE,OAAO;QACrB,MAAM,SAAS,GAAG,yBAAyB,CAAC,KAAK,CAAC,EAAE,OAAO,EAAE,QAAQ,EAAE,OAAO,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;QAC5F,UAAU,CAAC,GAAG,CAAC,GAAG,SAAS,CAAC,OAAO,SAAS,SAAS,CAAC,QAAQ,EAAE,EAAE,SAAS,CAAC,CAAC;IAC/E,CAAC,CAAC;IAEF,KAAK,MAAM,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,EAAE,CAAC;QACpE,GAAG,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;IACxD,CAAC;IACD,IAAI,IAAI,KAAK,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC/D,MAAM,SAAS,GAAG,+BAA+B,CAAC,KAAK,CACrD,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CACjD,CAAC;QACF,GAAG,CACD,SAAS,CAAC,OAAO,IAAI,CAAC,SAAS,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,EAC1E,SAAS,CAAC,MAAM,CAAC,SAAS,CAC3B,CAAC;IACJ,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC7B,KAAK,MAAM,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5D,GAAG,CACD,OAAO,EAAE,OAAO,IAAI,CAAC,OAAO,EAAE,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC,EACxE,SAAS,CAAC,MAAM,CAAC,OAAO,CACzB,CAAC;QACJ,CAAC;IACH,CAAC;IACD,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,QAAQ,GAAG;YACf,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;iBACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;iBAC7D,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;SACX,CAAC;QACF,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,QAAQ,GAAG,eAAe,CAC9B,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EACjC,SAAS,CAAC,MAAM,CAAC,WAAW,CAC7B,CAAC;YACF,KAAK,MAAM,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;gBAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;gBAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;gBAC5D,MAAM,MAAM,GAAG,sBAAsB,CACnC,QAAQ,EACR,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CACjD,CAAC;gBACF,MAAM,QAAQ,GAAG,wBAAwB,CACvC,QAAQ,EACR,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,EACpD,OAAO,CACR,CAAC;gBACF,GAAG,CAAC,aAAa,CAAC,MAAM,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;gBAC9D,GAAG,CAAC,qBAAqB,CAAC,QAAQ,CAAC,EAAE,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YAC1E,CAAC;YACD,KAAK,MAAM,SAAS,IAAI,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,EAAE,CAAC;gBACrE,GAAG,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;YACnD,CAAC;QACH,CAAC;IACH,CAAC;IACD,OAAO,CAAC,GAAG,UAAU,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE;QACnD,MAAM,YAAY,GAAG,IAAI,CAAC,OAAO,CAAC,aAAa,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC;QAC/D,OAAO,YAAY,IAAI,IAAI,CAAC,QAAQ,CAAC,aAAa,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC;IACrE,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,sBAAsB,GAAG,KAAK,EAClC,SAAmC,EACnC,MAAwB,EACxB,IAA+B,EACd,EAAE,CACnB,mCAAmC,CAAC,MAAM,oBAAoB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC,CAAC;AAE3F,MAAM,4BAA4B,GAAG,KAAK,EACxC,SAAmC,EACnC,MAAwB,EACP,EAAE;IACnB,MAAM,UAAU,GAAG,OAAO,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC;IACrD,MAAM,QAAQ,GAAG,OAAO,CAAC,SAAS,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC;IAC1E,MAAM,cAAc,GAAG,QAAQ,CAAC,UAAU,EAAE,QAAQ,CAAC,CAAC;IACtD,IAAI,cAAc,KAAK,EAAE,EAAE,CAAC;QAC1B,MAAM,IAAI,KAAK,CAAC,mDAAmD,CAAC,CAAC;IACvE,CAAC;IACD,MAAM,WAAW,GAAG,MAAM,4BAA4B,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC1E,MAAM,GAAG,GAAG,CAAC,IAA+B,EAAQ,EAAE;QACpD,IAAI,IAAI;YAAE,WAAW,CAAC,GAAG,CAAC,QAAQ,CAAC,UAAU,EAAE,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IACjE,CAAC,CAAC;IACF,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;IACjC,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC;IAC3B,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,KAAK,MAAM,KAAK,IAAI,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,EAAE,CAAC;YAChE,KAAK,MAAM,QAAQ,IAAI,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC;gBAAE,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACvF,CAAC;IACH,CAAC;SAAM,CAAC;QACN,GAAG,CAAC,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,IAAI,CAAC,CAAC;IAChD,CAAC;IACD,OAAO,2BAA2B,CAAC,UAAU,EAAE,CAAC,GAAG,WAAW,CAAC,CAAC,CAAC;AACnE,CAAC,CAAC;AACF,MAAM,+BAA+B,GAAG,KAAK,EAC3C,SAAmC,EACnC,MAAoD,EACrC,EAAE;IACjB,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,MAAM,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC7D,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;QAChC,MAAM,QAAQ,GAAG,eAAe,CAC9B,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,EACjC,SAAS,CAAC,MAAM,CAAC,WAAW,CAC7B,CAAC;QACF,KAAK,MAAM,SAAS,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,EAAE,CAAC;YAC3D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5D,MAAM,SAAS,GAAG,sBAAsB,CACtC,QAAQ,EACR,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CACjD,CAAC;YACF,MAAM,IAAI,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,SAAS,CAAC,IAAI,CAAC;YACxE,MAAM,SAAS,GAAG,OAAO,SAAS,KAAK,QAAQ,CAAC,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,SAAS,CAAC;YAC/E,2BAA2B,CAAC,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;YAC/D,MAAM,MAAM,GAAG,SAAS,CAAC,QAAQ,CAAC,SAAS,CAAC,IAAI,CAAC,CAAC;YAClD,IAAI,CAAC,MAAM,EAAE,CAAC;gBACZ,MAAM,IAAI,KAAK,CACb,SAAS,QAAQ,CAAC,EAAE,cAAc,SAAS,gCAAgC,IAAI,GAAG,CACnF,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,oBAAoB,CACpC,MAAM,CAAC,YAAY,EACnB,SAAS,EACT,SAAS,QAAQ,CAAC,EAAE,cAAc,SAAS,aAAa,IAAI,GAAG,CAChE,CAAC;YACF,2BAA2B,CAAC,IAAI,EAAE,SAAS,EAAE,gBAAgB,CAAC,CAAC;QACjE,CAAC;IACH,CAAC;AACH,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,kBAAkB,GAAG,KAAK,EACrC,SAAmC,EACnC,MAAwB,EACxB,IAA+B,EAC/B,2BAAoC,EACnB,EAAE;IACnB,MAAM,QAAQ,GAAG,MAAM,uBAAuB,CAAC,SAAS,CAAC,CAAC;IAC1D,IAAI,MAAM,CAAC,IAAI,KAAK,OAAO,EAAE,CAAC;QAC5B,MAAM,+BAA+B,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IAC3D,CAAC;IACD,MAAM,gBAAgB,GAAG,SAAS,CAAC,MAAM,CAAC,UAAU;QAClD,CAAC,CAAC,WAAW,CAAC,MAAM,QAAQ,CAAC,SAAS,CAAC,MAAM,CAAC,UAAU,CAAC,CAAC;QAC1D,CAAC,CAAC,IAAI,CAAC;IACT,MAAM,oBAAoB,GAAG,MAAM,4BAA4B,CAAC,SAAS,EAAE,MAAM,CAAC,CAAC;IACnF,MAAM,eAAe,GACnB,IAAI,KAAK,UAAU,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS;QACxD,CAAC,CAAC,+BAA+B,CAAC,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,SAAS,CAAC,MAAM,IAAI,EAAE,CAAC;QACzF,CAAC,CAAC,SAAS,CAAC;IAChB,IAAI,gBAAoC,CAAC;IACzC,IAAI,eAAe,EAAE,CAAC;QACpB,gBAAgB,GAAG,eAAe,CAAC,OAAO,CAAC;QAC3C,IAAI,CAAC,gBAAgB,EAAE,CAAC;YACtB,gBAAgB,GAAG,eAAe,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;QAC/E,CAAC;IACH,CAAC;IACD,OAAO,SAAS,CAAC;QACf,cAAc,EAAE,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI;QACzD,gBAAgB,EAAE,CAAC;QACnB,UAAU,EAAE,WAAW;QACvB,2BAA2B,EAAE,MAAM,2BAA2B,EAAE;QAChE,OAAO,EAAE,eAAe,EAAE;QAC1B,UAAU,EAAE,MAAM,CAAC,IAAI;QACvB,QAAQ;QACR,gBAAgB;QAChB,oBAAoB;QACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,GAAG,CACxC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;YAClE,8BAA8B,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,IAAI,CAAC,CAAC;YAC9E,OAAO;gBACL,IAAI,EAAE,UAAU,CAAC,IAAI;gBACrB,GAAG,EAAE,MAAM,WAAW,CAAC,UAAU,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC;aACzE,CAAC;QACJ,CAAC,CAAC,CACH;QACD,oBAAoB,EAAE,gBAAgB;YACpC,CAAC,CAAC,MAAM,WAAW,CAAC,gBAAgB,EAAE,SAAS,CAAC,MAAM,CAAC,SAAS,CAAC;YACjE,CAAC,CAAC,IAAI;QACR,sBAAsB,EACpB,2BAA2B,IAAI,CAAC,MAAM,sBAAsB,CAAC,SAAS,EAAE,MAAM,EAAE,IAAI,CAAC,CAAC;KACzF,CAAC,CAAC;AACL,CAAC,CAAC"}

package/lib/orchestrator/experiment-loader.d.ts

@@ -0,0 +1,175 @@
+import { z } from "zod";
+import { type ExperimentCampaign } from "../core/index.js";
+export declare const LoadedExperimentCampaignSchema: z.ZodObject<{
+    path: z.ZodString;
+    directory: z.ZodString;
+    campaign: z.ZodObject<{
+        id: z.ZodString;
+        subject: z.ZodObject<{
+            root: z.ZodString;
+            mutable: z.ZodArray<z.ZodString>;
+            exclude: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            validate: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                command: z.ZodString;
+                args: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                timeoutMs: z.ZodDefault<z.ZodNumber>;
+                unsafeHostExecution: z.ZodLiteral<true>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>;
+        evaluation: z.ZodDefault<z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"agent">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"grader">;
+            annotations: z.ZodString;
+            allowedAssertionAdditions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodArray<z.ZodString>>>;
+            scoreTolerance: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strict>], "kind">>;
+        profiles: z.ZodArray<z.ZodString>;
+        splits: z.ZodObject<{
+            train: z.ZodOptional<z.ZodObject<{
+                patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            }, z.core.$strict>>;
+            validation: z.ZodObject<{
+                patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            }, z.core.$strict>;
+            test: z.ZodOptional<z.ZodObject<{
+                patterns: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            }, z.core.$strict>>;
+        }, z.core.$strict>;
+        repetitions: z.ZodDefault<z.ZodObject<{
+            search: z.ZodDefault<z.ZodNumber>;
+            promotion: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strict>>;
+        variants: z.ZodDefault<z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            name: z.ZodOptional<z.ZodString>;
+            parentIds: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            components: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodString>>;
+        }, z.core.$strict>>>;
+        objectives: z.ZodDefault<z.ZodObject<{
+            constraints: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                metric: z.ZodString;
+                min: z.ZodOptional<z.ZodNumber>;
+                max: z.ZodOptional<z.ZodNumber>;
+                profile: z.ZodOptional<z.ZodString>;
+                split: z.ZodDefault<z.ZodEnum<{
+                    test: "test";
+                    train: "train";
+                    validation: "validation";
+                }>>;
+                confidence: z.ZodOptional<z.ZodObject<{
+                    method: z.ZodEnum<{
+                        hoeffding: "hoeffding";
+                        "percentile-bootstrap": "percentile-bootstrap";
+                        wilson: "wilson";
+                    }>;
+                    level: z.ZodDefault<z.ZodNumber>;
+                    resamples: z.ZodDefault<z.ZodNumber>;
+                    seed: z.ZodDefault<z.ZodNumber>;
+                    minSamples: z.ZodDefault<z.ZodNumber>;
+                }, z.core.$strict>>;
+            }, z.core.$strict>>>;
+            minimize: z.ZodDefault<z.ZodArray<z.ZodObject<{
+                metric: z.ZodString;
+                profile: z.ZodOptional<z.ZodString>;
+                split: z.ZodDefault<z.ZodEnum<{
+                    test: "test";
+                    train: "train";
+                    validation: "validation";
+                }>>;
+            }, z.core.$strict>>>;
+        }, z.core.$strict>>;
+        optimizer: z.ZodOptional<z.ZodObject<{
+            algorithm: z.ZodLiteral<"reflective">;
+            config: z.ZodOptional<z.ZodUnknown>;
+            candidateSelection: z.ZodDefault<z.ZodEnum<{
+                "current-best": "current-best";
+                pareto: "pareto";
+            }>>;
+            mergeFrontier: z.ZodDefault<z.ZodBoolean>;
+            promotionTopK: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strict>>;
+        budget: z.ZodDefault<z.ZodObject<{
+            maxCandidates: z.ZodDefault<z.ZodNumber>;
+            patience: z.ZodDefault<z.ZodNumber>;
+            maxMetricCalls: z.ZodDefault<z.ZodNumber>;
+            maxJudgeCalls: z.ZodDefault<z.ZodNumber>;
+            maxProposalCalls: z.ZodDefault<z.ZodNumber>;
+            maxTokens: z.ZodDefault<z.ZodNumber>;
+            maxCostUsd: z.ZodDefault<z.ZodNumber>;
+            maxUnpricedModelCalls: z.ZodDefault<z.ZodNumber>;
+            maxWallTimeMs: z.ZodDefault<z.ZodNumber>;
+        }, z.core.$strict>>;
+        meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, z.core.$strict>;
+    subjectRoot: z.ZodString;
+    campaignHash: z.ZodString;
+    graderAnnotations: z.ZodOptional<z.ZodObject<{
+        path: z.ZodString;
+        directory: z.ZodString;
+        dataset: z.ZodObject<{
+            schemaVersion: z.ZodLiteral<1>;
+            annotations: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                runDir: z.ZodString;
+                rubricPath: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+                tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+                expected: z.ZodObject<{
+                    casePassed: z.ZodOptional<z.ZodBoolean>;
+                    caseScore: z.ZodOptional<z.ZodNumber>;
+                    assertions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+                        passed: z.ZodBoolean;
+                        score: z.ZodOptional<z.ZodNumber>;
+                    }, z.core.$strict>>>;
+                }, z.core.$strict>;
+                source: z.ZodEnum<{
+                    human: "human";
+                    synthetic: "synthetic";
+                }>;
+                annotator: z.ZodOptional<z.ZodString>;
+                labeledAt: z.ZodOptional<z.ZodString>;
+                reviewer: z.ZodOptional<z.ZodString>;
+                feedback: z.ZodOptional<z.ZodString>;
+                meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+            }, z.core.$strict>>;
+            meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>;
+        annotations: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            runDir: z.ZodString;
+            rubricPath: z.ZodPipe<z.ZodString, z.ZodTransform<string, string>>;
+            tags: z.ZodDefault<z.ZodArray<z.ZodString>>;
+            expected: z.ZodObject<{
+                casePassed: z.ZodOptional<z.ZodBoolean>;
+                caseScore: z.ZodOptional<z.ZodNumber>;
+                assertions: z.ZodDefault<z.ZodRecord<z.ZodString, z.ZodObject<{
+                    passed: z.ZodBoolean;
+                    score: z.ZodOptional<z.ZodNumber>;
+                }, z.core.$strict>>>;
+            }, z.core.$strict>;
+            source: z.ZodEnum<{
+                human: "human";
+                synthetic: "synthetic";
+            }>;
+            annotator: z.ZodOptional<z.ZodString>;
+            labeledAt: z.ZodOptional<z.ZodString>;
+            reviewer: z.ZodOptional<z.ZodString>;
+            feedback: z.ZodOptional<z.ZodString>;
+            meta: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        }, z.core.$strict>>;
+        hash: z.ZodString;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export type LoadedExperimentCampaign = z.infer<typeof LoadedExperimentCampaignSchema>;
+export declare const ExperimentCommandModeSchema: z.ZodEnum<{
+    experiment: "experiment";
+    optimize: "optimize";
+}>;
+export type ExperimentCommandMode = z.infer<typeof ExperimentCommandModeSchema>;
+export declare const experimentCampaignMode: (path: string, campaign: ExperimentCampaign, requestedMode?: ExperimentCommandMode) => ExperimentCommandMode;
+export declare const resolveExperimentCampaignPath: (projectRoot: string, filePath: string) => Promise<string>;
+export declare const loadExperimentCampaign: (filePath: string) => Promise<LoadedExperimentCampaign>;
+//# sourceMappingURL=experiment-loader.d.ts.map

package/lib/orchestrator/experiment-loader.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-loader.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-loader.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,kBAAkB,CAAC;AAG1B,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAShC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AAEtF,eAAO,MAAM,2BAA2B;;;EAAqC,CAAC;AAC9E,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC;AAEhF,eAAO,MAAM,sBAAsB,SAC3B,MAAM,YACF,kBAAkB,kBACZ,qBAAqB,KACpC,qBAqBF,CAAC;AAEF,eAAO,MAAM,6BAA6B,gBAC3B,MAAM,YACT,MAAM,KACf,OAAO,CAAC,MAAM,CAgBhB,CAAC;AAEF,eAAO,MAAM,sBAAsB,aACvB,MAAM,KACf,OAAO,CAAC,wBAAwB,CA2BlC,CAAC"}

package/lib/orchestrator/experiment-loader.js

@@ -0,0 +1,77 @@
+import { lstat, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, relative, resolve, sep } from "node:path";
+import { pathToFileURL } from "node:url";
+import { z } from "zod";
+import { ExperimentCampaignSchema, formatZodIssues, valueHash, } from "../core/index.js";
+import { LoadedGraderAnnotationsSchema, loadGraderAnnotations } from "./annotation-loader.js";
+export const LoadedExperimentCampaignSchema = z
+    .object({
+    path: z.string(),
+    directory: z.string(),
+    campaign: ExperimentCampaignSchema,
+    subjectRoot: z.string(),
+    campaignHash: z.string(),
+    graderAnnotations: LoadedGraderAnnotationsSchema.optional(),
+})
+    .strict();
+export const ExperimentCommandModeSchema = z.enum(["experiment", "optimize"]);
+export const experimentCampaignMode = (path, campaign, requestedMode) => {
+    const experimentSuffix = path.endsWith(".experiment.ts");
+    const optimizeSuffix = path.endsWith(".optimize.ts");
+    if (experimentSuffix && campaign.optimizer) {
+        throw new Error("a .experiment.ts campaign must not define an optimizer; rename it to .optimize.ts or remove the optimizer");
+    }
+    if (optimizeSuffix && !campaign.optimizer) {
+        throw new Error("a .optimize.ts campaign must define an optimizer; rename it to .experiment.ts or add one");
+    }
+    let authoredMode = "experiment";
+    if (campaign.optimizer)
+        authoredMode = "optimize";
+    if (requestedMode && requestedMode !== authoredMode) {
+        throw new Error(`lux ${requestedMode} cannot run this ${authoredMode} campaign; use \`lux ${authoredMode}\` or change its optimizer`);
+    }
+    return authoredMode;
+};
+export const resolveExperimentCampaignPath = async (projectRoot, filePath) => {
+    const root = resolve(projectRoot);
+    const path = resolve(root, filePath);
+    const fromRoot = relative(root, path);
+    if (isAbsolute(fromRoot) || fromRoot === ".." || fromRoot.startsWith(`..${sep}`)) {
+        throw new Error(`experiment campaign must be contained by the Lux project: ${path}`);
+    }
+    const info = await lstat(path);
+    if (info.isSymbolicLink() || !info.isFile()) {
+        throw new Error(`experiment campaign must be a regular, non-symlink file: ${path}`);
+    }
+    const [canonicalRoot, canonicalPath] = await Promise.all([realpath(root), realpath(path)]);
+    if (canonicalPath !== resolve(canonicalRoot, fromRoot)) {
+        throw new Error(`experiment campaign path must not contain symlinks: ${path}`);
+    }
+    return canonicalPath;
+};
+export const loadExperimentCampaign = async (filePath) => {
+    const path = resolve(filePath);
+    const mod = (await import(pathToFileURL(path).href));
+    if (mod.default === undefined) {
+        throw new Error(`Lux experiment at ${path} has no default export. Use \`export default defineExperiment({ ... })\`.`);
+    }
+    const parsed = ExperimentCampaignSchema.safeParse(mod.default);
+    if (!parsed.success) {
+        throw new Error(`Invalid Lux experiment at ${path}: ${formatZodIssues(parsed.error)}`);
+    }
+    const directory = dirname(path);
+    const graderAnnotations = parsed.data.evaluation.kind === "grader"
+        ? await loadGraderAnnotations(resolve(directory, parsed.data.evaluation.annotations))
+        : undefined;
+    return LoadedExperimentCampaignSchema.parse({
+        path,
+        directory,
+        campaign: parsed.data,
+        subjectRoot: resolve(directory, parsed.data.subject.root),
+        campaignHash: graderAnnotations
+            ? valueHash({ campaign: parsed.data, annotationsHash: graderAnnotations.hash })
+            : valueHash(parsed.data),
+        ...(graderAnnotations ? { graderAnnotations } : {}),
+    });
+};
+//# sourceMappingURL=experiment-loader.js.map

package/lib/orchestrator/experiment-loader.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-loader.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-loader.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAE,OAAO,EAAE,UAAU,EAAE,QAAQ,EAAE,OAAO,EAAE,GAAG,EAAE,MAAM,WAAW,CAAC;AACxE,OAAO,EAAE,aAAa,EAAE,MAAM,UAAU,CAAC;AACzC,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AACxB,OAAO,EAEL,wBAAwB,EACxB,eAAe,EACf,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,6BAA6B,EAAE,qBAAqB,EAAE,MAAM,wBAAwB,CAAC;AAE9F,MAAM,CAAC,MAAM,8BAA8B,GAAG,CAAC;KAC5C,MAAM,CAAC;IACN,IAAI,EAAE,CAAC,CAAC,MAAM,EAAE;IAChB,SAAS,EAAE,CAAC,CAAC,MAAM,EAAE;IACrB,QAAQ,EAAE,wBAAwB;IAClC,WAAW,EAAE,CAAC,CAAC,MAAM,EAAE;IACvB,YAAY,EAAE,CAAC,CAAC,MAAM,EAAE;IACxB,iBAAiB,EAAE,6BAA6B,CAAC,QAAQ,EAAE;CAC5D,CAAC;KACD,MAAM,EAAE,CAAC;AAGZ,MAAM,CAAC,MAAM,2BAA2B,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,YAAY,EAAE,UAAU,CAAC,CAAC,CAAC;AAG9E,MAAM,CAAC,MAAM,sBAAsB,GAAG,CACpC,IAAY,EACZ,QAA4B,EAC5B,aAAqC,EACd,EAAE;IACzB,MAAM,gBAAgB,GAAG,IAAI,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACzD,MAAM,cAAc,GAAG,IAAI,CAAC,QAAQ,CAAC,cAAc,CAAC,CAAC;IACrD,IAAI,gBAAgB,IAAI,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CACb,2GAA2G,CAC5G,CAAC;IACJ,CAAC;IACD,IAAI,cAAc,IAAI,CAAC,QAAQ,CAAC,SAAS,EAAE,CAAC;QAC1C,MAAM,IAAI,KAAK,CACb,0FAA0F,CAC3F,CAAC;IACJ,CAAC;IACD,IAAI,YAAY,GAA0B,YAAY,CAAC;IACvD,IAAI,QAAQ,CAAC,SAAS;QAAE,YAAY,GAAG,UAAU,CAAC;IAClD,IAAI,aAAa,IAAI,aAAa,KAAK,YAAY,EAAE,CAAC;QACpD,MAAM,IAAI,KAAK,CACb,OAAO,aAAa,oBAAoB,YAAY,wBAAwB,YAAY,4BAA4B,CACrH,CAAC;IACJ,CAAC;IACD,OAAO,YAAY,CAAC;AACtB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,6BAA6B,GAAG,KAAK,EAChD,WAAmB,EACnB,QAAgB,EACC,EAAE;IACnB,MAAM,IAAI,GAAG,OAAO,CAAC,WAAW,CAAC,CAAC;IAClC,MAAM,IAAI,GAAG,OAAO,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC;IACrC,MAAM,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,IAAI,CAAC,CAAC;IACtC,IAAI,UAAU,CAAC,QAAQ,CAAC,IAAI,QAAQ,KAAK,IAAI,IAAI,QAAQ,CAAC,UAAU,CAAC,KAAK,GAAG,EAAE,CAAC,EAAE,CAAC;QACjF,MAAM,IAAI,KAAK,CAAC,6DAA6D,IAAI,EAAE,CAAC,CAAC;IACvF,CAAC;IACD,MAAM,IAAI,GAAG,MAAM,KAAK,CAAC,IAAI,CAAC,CAAC;IAC/B,IAAI,IAAI,CAAC,cAAc,EAAE,IAAI,CAAC,IAAI,CAAC,MAAM,EAAE,EAAE,CAAC;QAC5C,MAAM,IAAI,KAAK,CAAC,4DAA4D,IAAI,EAAE,CAAC,CAAC;IACtF,CAAC;IACD,MAAM,CAAC,aAAa,EAAE,aAAa,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,QAAQ,CAAC,IAAI,CAAC,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;IAC3F,IAAI,aAAa,KAAK,OAAO,CAAC,aAAa,EAAE,QAAQ,CAAC,EAAE,CAAC;QACvD,MAAM,IAAI,KAAK,CAAC,uDAAuD,IAAI,EAAE,CAAC,CAAC;IACjF,CAAC;IACD,OAAO,aAAa,CAAC;AACvB,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,sBAAsB,GAAG,KAAK,EACzC,QAAgB,EACmB,EAAE;IACrC,MAAM,IAAI,GAAG,OAAO,CAAC,QAAQ,CAAC,CAAC;IAC/B,MAAM,GAAG,GAAG,CAAC,MAAM,MAAM,CAAC,aAAa,CAAC,IAAI,CAAC,CAAC,IAAI,CAAC,CAA0B,CAAC;IAC9E,IAAI,GAAG,CAAC,OAAO,KAAK,SAAS,EAAE,CAAC;QAC9B,MAAM,IAAI,KAAK,CACb,qBAAqB,IAAI,2EAA2E,CACrG,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,wBAAwB,CAAC,SAAS,CAAC,GAAG,CAAC,OAAO,CAAC,CAAC;IAC/D,IAAI,CAAC,MAAM,CAAC,OAAO,EAAE,CAAC;QACpB,MAAM,IAAI,KAAK,CAAC,6BAA6B,IAAI,KAAK,eAAe,CAAC,MAAM,CAAC,KAAK,CAAC,EAAE,CAAC,CAAC;IACzF,CAAC;IACD,MAAM,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAAC;IAChC,MAAM,iBAAiB,GACrB,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;QACtC,CAAC,CAAC,MAAM,qBAAqB,CAAC,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,WAAW,CAAC,CAAC;QACrF,CAAC,CAAC,SAAS,CAAC;IAChB,OAAO,8BAA8B,CAAC,KAAK,CAAC;QAC1C,IAAI;QACJ,SAAS;QACT,QAAQ,EAAE,MAAM,CAAC,IAAI;QACrB,WAAW,EAAE,OAAO,CAAC,SAAS,EAAE,MAAM,CAAC,IAAI,CAAC,OAAO,CAAC,IAAI,CAAC;QACzD,YAAY,EAAE,iBAAiB;YAC7B,CAAC,CAAC,SAAS,CAAC,EAAE,QAAQ,EAAE,MAAM,CAAC,IAAI,EAAE,eAAe,EAAE,iBAAiB,CAAC,IAAI,EAAE,CAAC;YAC/E,CAAC,CAAC,SAAS,CAAC,MAAM,CAAC,IAAI,CAAC;QAC1B,GAAG,CAAC,iBAAiB,CAAC,CAAC,CAAC,EAAE,iBAAiB,EAAE,CAAC,CAAC,CAAC,EAAE,CAAC;KACpD,CAAC,CAAC;AACL,CAAC,CAAC"}