@prismatic-io/lux 0.0.1

package/lib/orchestrator/experiment-corpus.js

@@ -0,0 +1,276 @@
+import { readFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { RubricConfigSchema } from "../assertions/rubric/index.js";
+import { bindSubjectRoot, contentHash, valueHash, } from "../core/index.js";
+import { assertDisjointAnnotationSplits, selectGraderAnnotations } from "./annotation-loader.js";
+import { discoverCases, selectCases } from "./discover.js";
+import { profileOrThrow } from "./experiment-context.js";
+import { claudeConfigRoot, claudeUserBehaviorHash, cliIdentity, configuredClaudeSourceHashes, referencedAnswererCli, referencedCli, } from "./experiment-runtime-identity.js";
+import { fixtureIdentity } from "./fixture-identity.js";
+import { loadEvalCase } from "./loader.js";
+import { resolveAnswererReference, resolveDriverReference } from "./orchestrator.js";
+import { loadFrozenRegradableRun, releaseFrozenRegradableRun } from "./regrade.js";
+import { loadRubricDefinition } from "./rubric-loader.js";
+export const selectorsBySplit = (campaign) => [
+    ...(campaign.splits.train ? [{ name: "train", selector: campaign.splits.train }] : []),
+    { name: "validation", selector: campaign.splits.validation },
+    ...(campaign.splits.test ? [{ name: "test", selector: campaign.splits.test }] : []),
+];
+const fixtureEvidenceHash = async (execution, casePath, fixtures) => {
+    if (!fixtures)
+        return null;
+    return (await fixtureIdentity(fixtures, execution.config.fixturesRoot ?? dirname(casePath))).hash;
+};
+const agentCaseEvidenceHash = async (execution, casePath) => {
+    const authored = await loadEvalCase(casePath);
+    const { id: _caseId, meta: _meta, fixtures, assertions = [], ...behavior } = authored;
+    const normalizedAssertions = assertions.map(({ id: _assertionId, name: _assertionName, ...assertion }) => assertion);
+    return valueHash({
+        case: { ...behavior, assertions: normalizedAssertions },
+        fixtures: await fixtureEvidenceHash(execution, casePath, fixtures),
+    });
+};
+export const caseRubricCliReferences = (execution, authored) => {
+    const rubrics = (authored.assertions ?? []).flatMap((assertion, index) => assertion.type === "rubric" ? [{ index, config: RubricConfigSchema.parse(assertion) }] : []);
+    return execution.loaded.campaign.profiles.flatMap((profileId) => {
+        const profile = profileOrThrow(execution.config, profileId);
+        const harness = profile.harness ?? execution.config.harness;
+        return rubrics.map(({ index, config }) => {
+            const provider = config.provider ?? harness?.provider ?? "claude-code";
+            const inheritsHarness = config.provider === undefined || config.provider === harness?.provider;
+            return {
+                assertionId: config.id ?? `assertion-${index}`,
+                profileId,
+                provider,
+                command: config.command ??
+                    (inheritsHarness ? harness?.command : undefined) ??
+                    (provider === "codex" ? "codex" : "claude"),
+            };
+        });
+    });
+};
+const caseRubricCliIdentities = (execution, authored) => Promise.all(caseRubricCliReferences(execution, authored).map(async ({ command, ...reference }) => ({
+    ...reference,
+    cliIdentity: await cliIdentity(command, execution.config.rootDir),
+})));
+export const agentCorpusHash = async (execution, cases, splits) => {
+    const selected = [
+        ...new Map(splits
+            .flatMap((split) => selectCases(cases, split.selector))
+            .map((evalCase) => [evalCase.path, evalCase])).values(),
+    ];
+    const caseBehaviorHashes = new Map();
+    const configuredSourceHashes = new Map();
+    const definitions = await Promise.all(selected.map(async (evalCase) => {
+        const authored = await loadEvalCase(evalCase.path);
+        const bound = bindSubjectRoot(authored, execution.loaded.subjectRoot);
+        const profileCliIdentities = await Promise.all(execution.loaded.campaign.profiles.map(async (profileId) => {
+            const profile = profileOrThrow(execution.config, profileId);
+            const harness = profile.harness ?? execution.config.harness;
+            const driver = resolveDriverReference(bound, profile.driver ?? execution.config.defaultDriver);
+            const answerer = resolveAnswererReference(bound, profile.answerer ?? execution.config.defaultAnswerer, harness);
+            const driverCli = referencedCli(driver);
+            const answererCli = referencedAnswererCli(answerer);
+            const configuredRoot = driverCli?.name === "claude-code" ? claudeConfigRoot(driver) : null;
+            let driverClaudeBehaviorHash = null;
+            if (configuredRoot) {
+                let hash = caseBehaviorHashes.get(configuredRoot);
+                if (!hash) {
+                    hash = claudeUserBehaviorHash(configuredRoot);
+                    caseBehaviorHashes.set(configuredRoot, hash);
+                }
+                driverClaudeBehaviorHash = await hash;
+            }
+            return {
+                profileId,
+                driver,
+                driverCliIdentity: driverCli
+                    ? await cliIdentity(driverCli.command, execution.config.rootDir)
+                    : null,
+                driverClaudeBehaviorHash,
+                configuredDriverSources: await configuredClaudeSourceHashes(driver, execution.config.rootDir, configuredSourceHashes),
+                answerer,
+                answererCliIdentity: answererCli
+                    ? await cliIdentity(answererCli.command, execution.config.rootDir)
+                    : null,
+            };
+        }));
+        return {
+            path: evalCase.path,
+            sourceHash: contentHash(await readFile(evalCase.path)),
+            case: authored,
+            fixturesHash: await fixtureEvidenceHash(execution, evalCase.path, authored.fixtures),
+            profileCliIdentities,
+            caseRubricCliIdentities: await caseRubricCliIdentities(execution, bound),
+        };
+    }));
+    return valueHash(definitions);
+};
+export const graderCorpusHash = async (execution, splits) => {
+    const loaded = execution.loaded.graderAnnotations;
+    if (!loaded)
+        throw new Error("grader experiment is missing its annotation dataset");
+    const selected = [
+        ...new Map(splits
+            .flatMap((split) => selectGraderAnnotations(loaded.annotations, split.selector))
+            .map((annotation) => [annotation.id, annotation])).values(),
+    ];
+    const evidence = await Promise.all(selected.map(async (annotation) => {
+        const run = await loadFrozenRegradableRun(annotation.runDir);
+        try {
+            const rubric = await loadRubricDefinition(join(execution.loaded.subjectRoot, annotation.rubricPath));
+            if (rubric.id !== run.case.id) {
+                throw new Error(`rubric '${annotation.rubricPath}' id '${rubric.id}' does not match persisted case '${run.case.id}'`);
+            }
+            const artifacts = await Promise.all(run.artifacts.map(async (artifact) => {
+                const root = artifact.root ?? join(annotation.runDir, "artifacts");
+                try {
+                    return {
+                        ...artifact,
+                        root: undefined,
+                        contentHash: contentHash(await readFile(join(root, artifact.path))),
+                    };
+                }
+                catch (error) {
+                    if (error.code === "ENOENT") {
+                        return { ...artifact, root: undefined, contentHash: null };
+                    }
+                    throw error;
+                }
+            }));
+            return {
+                annotationId: annotation.id,
+                runDir: annotation.runDir,
+                evidenceHash: valueHash({
+                    caseId: run.case.id,
+                    prompt: run.case.prompt,
+                    events: run.events.flatMap((event) => {
+                        if (event.type === "lifecycle" || event.type === "ready")
+                            return [];
+                        const { ts: _timestamp, ...visible } = event;
+                        return [visible];
+                    }),
+                    artifacts,
+                }),
+                rubric,
+            };
+        }
+        finally {
+            await releaseFrozenRegradableRun(run);
+        }
+    }));
+    const splitByAnnotation = new Map(splits.flatMap((split) => selectGraderAnnotations(loaded.annotations, split.selector).map((annotation) => [annotation.id, split.name])));
+    const ownerByEvidence = new Map();
+    for (const item of evidence) {
+        const split = splitByAnnotation.get(item.annotationId);
+        if (!split)
+            continue;
+        const owner = ownerByEvidence.get(item.evidenceHash);
+        if (owner) {
+            const scope = owner.split === split ? `duplicate evidence within ${split}` : "split leakage";
+            throw new Error(`grader ${scope}: annotations '${owner.annotationId}' (${owner.split}) and '${item.annotationId}' (${split}) contain identical frozen evidence`);
+        }
+        ownerByEvidence.set(item.evidenceHash, { annotationId: item.annotationId, split });
+    }
+    return valueHash({ annotationsHash: loaded.hash, evidence });
+};
+export const prepareCorpus = async (execution) => {
+    const campaign = execution.loaded.campaign;
+    const splits = selectorsBySplit(campaign);
+    if (campaign.evaluation.kind === "grader") {
+        const loaded = execution.loaded.graderAnnotations;
+        if (!loaded)
+            throw new Error("grader experiment is missing its annotation dataset");
+        const selectedSplits = splits.map((split) => ({
+            name: split.name,
+            annotations: selectGraderAnnotations(loaded.annotations, split.selector),
+        }));
+        for (const split of selectedSplits) {
+            if (split.annotations.length === 0) {
+                throw new Error(`experiment '${campaign.id}' ${split.name} selector matched no grader annotations`);
+            }
+        }
+        assertDisjointAnnotationSplits(selectedSplits);
+        return { kind: "grader", hash: await graderCorpusHash(execution, splits) };
+    }
+    const discovery = await discoverCases(execution.config.casesRoot ?? join(execution.config.rootDir, "cases"));
+    if (discovery.problems.length > 0) {
+        throw new Error(`experiment case discovery failed: ${discovery.problems.map((problem) => `${problem.path}: ${problem.reason}`).join("; ")}`);
+    }
+    const owners = new Map();
+    const caseIds = new Map();
+    const evidenceOwners = new Map();
+    for (const split of splits) {
+        const selected = selectCases(discovery.cases, split.selector);
+        if (selected.length === 0) {
+            throw new Error(`experiment '${campaign.id}' ${split.name} selector matched no cases`);
+        }
+        for (const evalCase of selected) {
+            const existingPath = caseIds.get(evalCase.id);
+            if (existingPath && existingPath !== evalCase.path) {
+                throw new Error(`experiment case id '${evalCase.id}' is duplicated by ${existingPath} and ${evalCase.path}`);
+            }
+            caseIds.set(evalCase.id, evalCase.path);
+            const owner = owners.get(evalCase.path);
+            if (owner) {
+                throw new Error(`eval case '${evalCase.id}' appears in both ${owner} and ${split.name} splits`);
+            }
+            owners.set(evalCase.path, split.name);
+            const evidenceHash = await agentCaseEvidenceHash(execution, evalCase.path);
+            const evidenceOwner = evidenceOwners.get(evidenceHash);
+            if (evidenceOwner) {
+                const scope = evidenceOwner.split === split.name
+                    ? `duplicate evidence within ${split.name}`
+                    : "split leakage";
+                throw new Error(`agent ${scope}: cases '${evidenceOwner.caseId}' (${evidenceOwner.split}) and '${evalCase.id}' (${split.name}) contain identical behavioral evidence`);
+            }
+            evidenceOwners.set(evidenceHash, {
+                caseId: evalCase.id,
+                path: evalCase.path,
+                split: split.name,
+            });
+        }
+    }
+    return {
+        kind: "agent",
+        cases: discovery.cases,
+        hash: await agentCorpusHash(execution, discovery.cases, splits),
+    };
+};
+export const assertGraderSubject = async (execution, baseline) => {
+    if (execution.loaded.campaign.evaluation.kind !== "grader")
+        return;
+    const loadedAnnotations = execution.loaded.graderAnnotations?.annotations ?? [];
+    const annotations = [
+        ...new Map(selectorsBySplit(execution.loaded.campaign)
+            .flatMap((split) => selectGraderAnnotations(loadedAnnotations, split.selector))
+            .map((annotation) => [annotation.id, annotation])).values(),
+    ];
+    const rubricPaths = [...new Set(annotations.map((annotation) => annotation.rubricPath))].sort();
+    const componentPaths = Object.keys(baseline.components).sort();
+    if (rubricPaths.length !== componentPaths.length ||
+        rubricPaths.some((path, index) => path !== componentPaths[index])) {
+        throw new Error("grader experiment mutable components must exactly match the annotation rubric paths");
+    }
+    for (const rubricPath of rubricPaths) {
+        const rubric = await loadRubricDefinition(join(execution.loaded.subjectRoot, rubricPath));
+        const baselineIds = new Set(rubric.assertions.map((assertion) => assertion.id));
+        const relevant = annotations.filter((annotation) => annotation.rubricPath === rubricPath);
+        const allowedAdditions = new Set(execution.loaded.campaign.evaluation.allowedAssertionAdditions[rubricPath] ?? []);
+        const allLabeledIds = new Set(relevant.flatMap((annotation) => Object.keys(annotation.expected.assertions)));
+        const undeclaredAdditions = [...allLabeledIds].filter((id) => !baselineIds.has(id) && !allowedAdditions.has(id));
+        if (undeclaredAdditions.length > 0) {
+            throw new Error(`annotation assertion IDs absent from '${rubricPath}' must be declared in evaluation.allowedAssertionAdditions: ${undeclaredAdditions.join(", ")}`);
+        }
+        const incompleteAdditions = [...allowedAdditions].filter((id) => baselineIds.has(id) ||
+            !relevant.every((annotation) => Object.hasOwn(annotation.expected.assertions, id)));
+        if (incompleteAdditions.length > 0) {
+            throw new Error(`allowed assertion additions for '${rubricPath}' must be absent from baseline and labeled on every frozen run: ${incompleteAdditions.join(", ")}`);
+        }
+    }
+    const unknownRubricPaths = Object.keys(execution.loaded.campaign.evaluation.allowedAssertionAdditions).filter((path) => !rubricPaths.includes(path));
+    if (unknownRubricPaths.length > 0) {
+        throw new Error(`allowed assertion additions reference rubric paths absent from annotations: ${unknownRubricPaths.join(", ")}`);
+    }
+};
+//# sourceMappingURL=experiment-corpus.js.map

package/lib/orchestrator/experiment-corpus.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-corpus.js","sourceRoot":"","sources":["../../src/orchestrator/experiment-corpus.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,kBAAkB,CAAC;AAC5C,OAAO,EAAE,OAAO,EAAE,IAAI,EAAE,MAAM,WAAW,CAAC;AAC1C,OAAO,EAAE,kBAAkB,EAAE,MAAM,+BAA+B,CAAC;AACnE,OAAO,EACL,eAAe,EAEf,WAAW,EAGX,SAAS,GACV,MAAM,kBAAkB,CAAC;AAC1B,OAAO,EAAE,8BAA8B,EAAE,uBAAuB,EAAE,MAAM,wBAAwB,CAAC;AACjG,OAAO,EAAE,aAAa,EAAE,WAAW,EAAE,MAAM,eAAe,CAAC;AAE3D,OAAO,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AACzD,OAAO,EACL,gBAAgB,EAChB,sBAAsB,EACtB,WAAW,EACX,4BAA4B,EAC5B,qBAAqB,EACrB,aAAa,GACd,MAAM,kCAAkC,CAAC;AAC1C,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AACxD,OAAO,EAAE,YAAY,EAAE,MAAM,aAAa,CAAC;AAC3C,OAAO,EAAE,wBAAwB,EAAE,sBAAsB,EAAE,MAAM,mBAAmB,CAAC;AACrF,OAAO,EAAE,uBAAuB,EAAE,0BAA0B,EAAE,MAAM,cAAc,CAAC;AACnF,OAAO,EAAE,oBAAoB,EAAE,MAAM,oBAAoB,CAAC;AAI1D,MAAM,CAAC,MAAM,gBAAgB,GAAG,CAC9B,QAA4B,EACoC,EAAE,CAAC;IACnE,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,KAAK,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,OAAgB,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,KAAK,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;IAC/F,EAAE,IAAI,EAAE,YAAY,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,UAAU,EAAE;IAC5D,GAAG,CAAC,QAAQ,CAAC,MAAM,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC,EAAE,IAAI,EAAE,MAAe,EAAE,QAAQ,EAAE,QAAQ,CAAC,MAAM,CAAC,IAAI,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAAC;CAC7F,CAAC;AAEF,MAAM,mBAAmB,GAAG,KAAK,EAC/B,SAAmC,EACnC,QAAgB,EAChB,QAA8D,EACtC,EAAE;IAC1B,IAAI,CAAC,QAAQ;QAAE,OAAO,IAAI,CAAC;IAC3B,OAAO,CAAC,MAAM,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,YAAY,IAAI,OAAO,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;AACpG,CAAC,CAAC;AAEF,MAAM,qBAAqB,GAAG,KAAK,EACjC,SAAmC,EACnC,QAAgB,EACC,EAAE;IACnB,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,CAAC;IAC9C,MAAM,EAAE,EAAE,EAAE,OAAO,EAAE,IAAI,EAAE,KAAK,EAAE,QAAQ,EAAE,UAAU,GAAG,EAAE,EAAE,GAAG,QAAQ,EAAE,GAAG,QAAQ,CAAC;IACtF,MAAM,oBAAoB,GAAG,UAAU,CAAC,GAAG,CACzC,CAAC,EAAE,EAAE,EAAE,YAAY,EAAE,IAAI,EAAE,cAAc,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,SAAS,CACxE,CAAC;IACF,OAAO,SAAS,CAAC;QACf,IAAI,EAAE,EAAE,GAAG,QAAQ,EAAE,UAAU,EAAE,oBAAoB,EAAE;QACvD,QAAQ,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,EAAE,QAAQ,CAAC;KACnE,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,uBAAuB,GAAG,CACrC,SAAmC,EACnC,QAAkD,EAMjD,EAAE;IACH,MAAM,OAAO,GAAG,CAAC,QAAQ,CAAC,UAAU,IAAI,EAAE,CAAC,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,KAAK,EAAE,EAAE,CACvE,SAAS,CAAC,IAAI,KAAK,QAAQ,CAAC,CAAC,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,kBAAkB,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC,CAAC,EAAE,CAC5F,CAAC;IACF,OAAO,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC,SAAS,EAAE,EAAE;QAC9D,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;QAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;QAC5D,OAAO,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,KAAK,EAAE,MAAM,EAAE,EAAE,EAAE;YACvC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,IAAI,OAAO,EAAE,QAAQ,IAAI,aAAa,CAAC;YACvE,MAAM,eAAe,GACnB,MAAM,CAAC,QAAQ,KAAK,SAAS,IAAI,MAAM,CAAC,QAAQ,KAAK,OAAO,EAAE,QAAQ,CAAC;YACzE,OAAO;gBACL,WAAW,EAAE,MAAM,CAAC,EAAE,IAAI,aAAa,KAAK,EAAE;gBAC9C,SAAS;gBACT,QAAQ;gBACR,OAAO,EACL,MAAM,CAAC,OAAO;oBACd,CAAC,eAAe,CAAC,CAAC,CAAC,OAAO,EAAE,OAAO,CAAC,CAAC,CAAC,SAAS,CAAC;oBAChD,CAAC,QAAQ,KAAK,OAAO,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC,CAAC,QAAQ,CAAC;aAC9C,CAAC;QACJ,CAAC,CAAC,CAAC;IACL,CAAC,CAAC,CAAC;AACL,CAAC,CAAC;AAEF,MAAM,uBAAuB,GAAG,CAC9B,SAAmC,EACnC,QAAkD,EAQlD,EAAE,CACF,OAAO,CAAC,GAAG,CACT,uBAAuB,CAAC,SAAS,EAAE,QAAQ,CAAC,CAAC,GAAG,CAAC,KAAK,EAAE,EAAE,OAAO,EAAE,GAAG,SAAS,EAAE,EAAE,EAAE,CAAC,CAAC;IACrF,GAAG,SAAS;IACZ,WAAW,EAAE,MAAM,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;CAClE,CAAC,CAAC,CACJ,CAAC;AAEJ,MAAM,CAAC,MAAM,eAAe,GAAG,KAAK,EAClC,SAAmC,EACnC,KAAyD,EACzD,MAA2C,EAC1B,EAAE;IACnB,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,WAAW,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aACtD,GAAG,CAAC,CAAC,QAAQ,EAAE,EAAE,CAAC,CAAC,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,CAAC,CAChD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,kBAAkB,GAAG,IAAI,GAAG,EAAkC,CAAC;IACrE,MAAM,sBAAsB,GAAG,IAAI,GAAG,EAA2B,CAAC;IAClE,MAAM,WAAW,GAAG,MAAM,OAAO,CAAC,GAAG,CACnC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;QAC9B,MAAM,QAAQ,GAAG,MAAM,YAAY,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;QACnD,MAAM,KAAK,GAAG,eAAe,CAAC,QAAQ,EAAE,SAAS,CAAC,MAAM,CAAC,WAAW,CAAC,CAAC;QACtE,MAAM,oBAAoB,GAAG,MAAM,OAAO,CAAC,GAAG,CAC5C,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,SAAS,EAAE,EAAE;YACzD,MAAM,OAAO,GAAG,cAAc,CAAC,SAAS,CAAC,MAAM,EAAE,SAAS,CAAC,CAAC;YAC5D,MAAM,OAAO,GAAG,OAAO,CAAC,OAAO,IAAI,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;YAC5D,MAAM,MAAM,GAAG,sBAAsB,CACnC,KAAK,EACL,OAAO,CAAC,MAAM,IAAI,SAAS,CAAC,MAAM,CAAC,aAAa,CACjD,CAAC;YACF,MAAM,QAAQ,GAAG,wBAAwB,CACvC,KAAK,EACL,OAAO,CAAC,QAAQ,IAAI,SAAS,CAAC,MAAM,CAAC,eAAe,EACpD,OAAO,CACR,CAAC;YACF,MAAM,SAAS,GAAG,aAAa,CAAC,MAAM,CAAC,CAAC;YACxC,MAAM,WAAW,GAAG,qBAAqB,CAAC,QAAQ,CAAC,CAAC;YACpD,MAAM,cAAc,GAClB,SAAS,EAAE,IAAI,KAAK,aAAa,CAAC,CAAC,CAAC,gBAAgB,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,IAAI,CAAC;YACtE,IAAI,wBAAwB,GAAkB,IAAI,CAAC;YACnD,IAAI,cAAc,EAAE,CAAC;gBACnB,IAAI,IAAI,GAAG,kBAAkB,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;gBAClD,IAAI,CAAC,IAAI,EAAE,CAAC;oBACV,IAAI,GAAG,sBAAsB,CAAC,cAAc,CAAC,CAAC;oBAC9C,kBAAkB,CAAC,GAAG,CAAC,cAAc,EAAE,IAAI,CAAC,CAAC;gBAC/C,CAAC;gBACD,wBAAwB,GAAG,MAAM,IAAI,CAAC;YACxC,CAAC;YACD,OAAO;gBACL,SAAS;gBACT,MAAM;gBACN,iBAAiB,EAAE,SAAS;oBAC1B,CAAC,CAAC,MAAM,WAAW,CAAC,SAAS,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAChE,CAAC,CAAC,IAAI;gBACR,wBAAwB;gBACxB,uBAAuB,EAAE,MAAM,4BAA4B,CACzD,MAAM,EACN,SAAS,CAAC,MAAM,CAAC,OAAO,EACxB,sBAAsB,CACvB;gBACD,QAAQ;gBACR,mBAAmB,EAAE,WAAW;oBAC9B,CAAC,CAAC,MAAM,WAAW,CAAC,WAAW,CAAC,OAAO,EAAE,SAAS,CAAC,MAAM,CAAC,OAAO,CAAC;oBAClE,CAAC,CAAC,IAAI;aACT,CAAC;QACJ,CAAC,CAAC,CACH,CAAC;QACF,OAAO;YACL,IAAI,EAAE,QAAQ,CAAC,IAAI;YACnB,UAAU,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACtD,IAAI,EAAE,QAAQ;YACd,YAAY,EAAE,MAAM,mBAAmB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,EAAE,QAAQ,CAAC,QAAQ,CAAC;YACpF,oBAAoB;YACpB,uBAAuB,EAAE,MAAM,uBAAuB,CAAC,SAAS,EAAE,KAAK,CAAC;SACzE,CAAC;IACJ,CAAC,CAAC,CACH,CAAC;IACF,OAAO,SAAS,CAAC,WAAW,CAAC,CAAC;AAChC,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,gBAAgB,GAAG,KAAK,EACnC,SAAmC,EACnC,MAA2C,EAC1B,EAAE;IACnB,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;IAClD,IAAI,CAAC,MAAM;QAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;IACpF,MAAM,QAAQ,GAAG;QACf,GAAG,IAAI,GAAG,CACR,MAAM;aACH,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC/E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,QAAQ,GAAG,MAAM,OAAO,CAAC,GAAG,CAChC,QAAQ,CAAC,GAAG,CAAC,KAAK,EAAE,UAAU,EAAE,EAAE;QAChC,MAAM,GAAG,GAAG,MAAM,uBAAuB,CAAC,UAAU,CAAC,MAAM,CAAC,CAAC;QAC7D,IAAI,CAAC;YACH,MAAM,MAAM,GAAG,MAAM,oBAAoB,CACvC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,UAAU,CAAC,CAC1D,CAAC;YACF,IAAI,MAAM,CAAC,EAAE,KAAK,GAAG,CAAC,IAAI,CAAC,EAAE,EAAE,CAAC;gBAC9B,MAAM,IAAI,KAAK,CACb,WAAW,UAAU,CAAC,UAAU,SAAS,MAAM,CAAC,EAAE,oCAAoC,GAAG,CAAC,IAAI,CAAC,EAAE,GAAG,CACrG,CAAC;YACJ,CAAC;YACD,MAAM,SAAS,GAAG,MAAM,OAAO,CAAC,GAAG,CACjC,GAAG,CAAC,SAAS,CAAC,GAAG,CAAC,KAAK,EAAE,QAAQ,EAAE,EAAE;gBACnC,MAAM,IAAI,GAAG,QAAQ,CAAC,IAAI,IAAI,IAAI,CAAC,UAAU,CAAC,MAAM,EAAE,WAAW,CAAC,CAAC;gBACnE,IAAI,CAAC;oBACH,OAAO;wBACL,GAAG,QAAQ;wBACX,IAAI,EAAE,SAAS;wBACf,WAAW,EAAE,WAAW,CAAC,MAAM,QAAQ,CAAC,IAAI,CAAC,IAAI,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;qBACpE,CAAC;gBACJ,CAAC;gBAAC,OAAO,KAAK,EAAE,CAAC;oBACf,IAAK,KAA+B,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;wBACvD,OAAO,EAAE,GAAG,QAAQ,EAAE,IAAI,EAAE,SAAS,EAAE,WAAW,EAAE,IAAI,EAAE,CAAC;oBAC7D,CAAC;oBACD,MAAM,KAAK,CAAC;gBACd,CAAC;YACH,CAAC,CAAC,CACH,CAAC;YACF,OAAO;gBACL,YAAY,EAAE,UAAU,CAAC,EAAE;gBAC3B,MAAM,EAAE,UAAU,CAAC,MAAM;gBACzB,YAAY,EAAE,SAAS,CAAC;oBACtB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,EAAE;oBACnB,MAAM,EAAE,GAAG,CAAC,IAAI,CAAC,MAAM;oBACvB,MAAM,EAAE,GAAG,CAAC,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE;wBACnC,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,IAAI,KAAK,CAAC,IAAI,KAAK,OAAO;4BAAE,OAAO,EAAE,CAAC;wBACpE,MAAM,EAAE,EAAE,EAAE,UAAU,EAAE,GAAG,OAAO,EAAE,GAAG,KAAK,CAAC;wBAC7C,OAAO,CAAC,OAAO,CAAC,CAAC;oBACnB,CAAC,CAAC;oBACF,SAAS;iBACV,CAAC;gBACF,MAAM;aACP,CAAC;QACJ,CAAC;gBAAS,CAAC;YACT,MAAM,0BAA0B,CAAC,GAAG,CAAC,CAAC;QACxC,CAAC;IACH,CAAC,CAAC,CACH,CAAC;IACF,MAAM,iBAAiB,GAAG,IAAI,GAAG,CAC/B,MAAM,CAAC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CACvB,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAC7D,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,KAAK,CAAC,IAAI,CAAU,CACrD,CACF,CACF,CAAC;IACF,MAAM,eAAe,GAAG,IAAI,GAAG,EAA4D,CAAC;IAC5F,KAAK,MAAM,IAAI,IAAI,QAAQ,EAAE,CAAC;QAC5B,MAAM,KAAK,GAAG,iBAAiB,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACvD,IAAI,CAAC,KAAK;YAAE,SAAS;QACrB,MAAM,KAAK,GAAG,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,CAAC,CAAC;QACrD,IAAI,KAAK,EAAE,CAAC;YACV,MAAM,KAAK,GAAG,KAAK,CAAC,KAAK,KAAK,KAAK,CAAC,CAAC,CAAC,6BAA6B,KAAK,EAAE,CAAC,CAAC,CAAC,eAAe,CAAC;YAC7F,MAAM,IAAI,KAAK,CACb,UAAU,KAAK,kBAAkB,KAAK,CAAC,YAAY,MAAM,KAAK,CAAC,KAAK,UAAU,IAAI,CAAC,YAAY,MAAM,KAAK,qCAAqC,CAChJ,CAAC;QACJ,CAAC;QACD,eAAe,CAAC,GAAG,CAAC,IAAI,CAAC,YAAY,EAAE,EAAE,YAAY,EAAE,IAAI,CAAC,YAAY,EAAE,KAAK,EAAE,CAAC,CAAC;IACrF,CAAC;IACD,OAAO,SAAS,CAAC,EAAE,eAAe,EAAE,MAAM,CAAC,IAAI,EAAE,QAAQ,EAAE,CAAC,CAAC;AAC/D,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,aAAa,GAAG,KAAK,EAChC,SAAmC,EACR,EAAE;IAC7B,MAAM,QAAQ,GAAG,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;IAC3C,MAAM,MAAM,GAAG,gBAAgB,CAAC,QAAQ,CAAC,CAAC;IAC1C,IAAI,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ,EAAE,CAAC;QAC1C,MAAM,MAAM,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,CAAC;QAClD,IAAI,CAAC,MAAM;YAAE,MAAM,IAAI,KAAK,CAAC,qDAAqD,CAAC,CAAC;QACpF,MAAM,cAAc,GAAG,MAAM,CAAC,GAAG,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,CAAC;YAC5C,IAAI,EAAE,KAAK,CAAC,IAAI;YAChB,WAAW,EAAE,uBAAuB,CAAC,MAAM,CAAC,WAAW,EAAE,KAAK,CAAC,QAAQ,CAAC;SACzE,CAAC,CAAC,CAAC;QACJ,KAAK,MAAM,KAAK,IAAI,cAAc,EAAE,CAAC;YACnC,IAAI,KAAK,CAAC,WAAW,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;gBACnC,MAAM,IAAI,KAAK,CACb,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,yCAAyC,CACnF,CAAC;YACJ,CAAC;QACH,CAAC;QACD,8BAA8B,CAAC,cAAc,CAAC,CAAC;QAC/C,OAAO,EAAE,IAAI,EAAE,QAAQ,EAAE,IAAI,EAAE,MAAM,gBAAgB,CAAC,SAAS,EAAE,MAAM,CAAC,EAAE,CAAC;IAC7E,CAAC;IACD,MAAM,SAAS,GAAG,MAAM,aAAa,CACnC,SAAS,CAAC,MAAM,CAAC,SAAS,IAAI,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,OAAO,EAAE,OAAO,CAAC,CACtE,CAAC;IACF,IAAI,SAAS,CAAC,QAAQ,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,qCAAqC,SAAS,CAAC,QAAQ,CAAC,GAAG,CAAC,CAAC,OAAO,EAAE,EAAE,CAAC,GAAG,OAAO,CAAC,IAAI,KAAK,OAAO,CAAC,MAAM,EAAE,CAAC,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC5H,CAAC;IACJ,CAAC;IACD,MAAM,MAAM,GAAG,IAAI,GAAG,EAAkB,CAAC;IACzC,MAAM,OAAO,GAAG,IAAI,GAAG,EAAkB,CAAC;IAC1C,MAAM,cAAc,GAAG,IAAI,GAAG,EAG3B,CAAC;IACJ,KAAK,MAAM,KAAK,IAAI,MAAM,EAAE,CAAC;QAC3B,MAAM,QAAQ,GAAG,WAAW,CAAC,SAAS,CAAC,KAAK,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;QAC9D,IAAI,QAAQ,CAAC,MAAM,KAAK,CAAC,EAAE,CAAC;YAC1B,MAAM,IAAI,KAAK,CAAC,eAAe,QAAQ,CAAC,EAAE,KAAK,KAAK,CAAC,IAAI,4BAA4B,CAAC,CAAC;QACzF,CAAC;QACD,KAAK,MAAM,QAAQ,IAAI,QAAQ,EAAE,CAAC;YAChC,MAAM,YAAY,GAAG,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,CAAC,CAAC;YAC9C,IAAI,YAAY,IAAI,YAAY,KAAK,QAAQ,CAAC,IAAI,EAAE,CAAC;gBACnD,MAAM,IAAI,KAAK,CACb,uBAAuB,QAAQ,CAAC,EAAE,sBAAsB,YAAY,QAAQ,QAAQ,CAAC,IAAI,EAAE,CAC5F,CAAC;YACJ,CAAC;YACD,OAAO,CAAC,GAAG,CAAC,QAAQ,CAAC,EAAE,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,MAAM,KAAK,GAAG,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC;YACxC,IAAI,KAAK,EAAE,CAAC;gBACV,MAAM,IAAI,KAAK,CACb,cAAc,QAAQ,CAAC,EAAE,qBAAqB,KAAK,QAAQ,KAAK,CAAC,IAAI,SAAS,CAC/E,CAAC;YACJ,CAAC;YACD,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC,IAAI,EAAE,KAAK,CAAC,IAAI,CAAC,CAAC;YACtC,MAAM,YAAY,GAAG,MAAM,qBAAqB,CAAC,SAAS,EAAE,QAAQ,CAAC,IAAI,CAAC,CAAC;YAC3E,MAAM,aAAa,GAAG,cAAc,CAAC,GAAG,CAAC,YAAY,CAAC,CAAC;YACvD,IAAI,aAAa,EAAE,CAAC;gBAClB,MAAM,KAAK,GACT,aAAa,CAAC,KAAK,KAAK,KAAK,CAAC,IAAI;oBAChC,CAAC,CAAC,6BAA6B,KAAK,CAAC,IAAI,EAAE;oBAC3C,CAAC,CAAC,eAAe,CAAC;gBACtB,MAAM,IAAI,KAAK,CACb,SAAS,KAAK,YAAY,aAAa,CAAC,MAAM,MAAM,aAAa,CAAC,KAAK,UAAU,QAAQ,CAAC,EAAE,MAAM,KAAK,CAAC,IAAI,yCAAyC,CACtJ,CAAC;YACJ,CAAC;YACD,cAAc,CAAC,GAAG,CAAC,YAAY,EAAE;gBAC/B,MAAM,EAAE,QAAQ,CAAC,EAAE;gBACnB,IAAI,EAAE,QAAQ,CAAC,IAAI;gBACnB,KAAK,EAAE,KAAK,CAAC,IAAI;aAClB,CAAC,CAAC;QACL,CAAC;IACH,CAAC;IACD,OAAO;QACL,IAAI,EAAE,OAAO;QACb,KAAK,EAAE,SAAS,CAAC,KAAK;QACtB,IAAI,EAAE,MAAM,eAAe,CAAC,SAAS,EAAE,SAAS,CAAC,KAAK,EAAE,MAAM,CAAC;KAChE,CAAC;AACJ,CAAC,CAAC;AAEF,MAAM,CAAC,MAAM,mBAAmB,GAAG,KAAK,EACtC,SAAmC,EACnC,QAAmB,EACJ,EAAE;IACjB,IAAI,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,IAAI,KAAK,QAAQ;QAAE,OAAO;IACnE,MAAM,iBAAiB,GAAG,SAAS,CAAC,MAAM,CAAC,iBAAiB,EAAE,WAAW,IAAI,EAAE,CAAC;IAChF,MAAM,WAAW,GAAG;QAClB,GAAG,IAAI,GAAG,CACR,gBAAgB,CAAC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC;aACxC,OAAO,CAAC,CAAC,KAAK,EAAE,EAAE,CAAC,uBAAuB,CAAC,iBAAiB,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAC;aAC9E,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,UAAU,CAAC,EAAE,EAAE,UAAU,CAAC,CAAC,CACpD,CAAC,MAAM,EAAE;KACX,CAAC;IACF,MAAM,WAAW,GAAG,CAAC,GAAG,IAAI,GAAG,CAAC,WAAW,CAAC,GAAG,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,CAAC,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC;IAChG,MAAM,cAAc,GAAG,MAAM,CAAC,IAAI,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,IAAI,EAAE,CAAC;IAC/D,IACE,WAAW,CAAC,MAAM,KAAK,cAAc,CAAC,MAAM;QAC5C,WAAW,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,KAAK,EAAE,EAAE,CAAC,IAAI,KAAK,cAAc,CAAC,KAAK,CAAC,CAAC,EACjE,CAAC;QACD,MAAM,IAAI,KAAK,CACb,qFAAqF,CACtF,CAAC;IACJ,CAAC;IACD,KAAK,MAAM,UAAU,IAAI,WAAW,EAAE,CAAC;QACrC,MAAM,MAAM,GAAG,MAAM,oBAAoB,CAAC,IAAI,CAAC,SAAS,CAAC,MAAM,CAAC,WAAW,EAAE,UAAU,CAAC,CAAC,CAAC;QAC1F,MAAM,WAAW,GAAG,IAAI,GAAG,CAAC,MAAM,CAAC,UAAU,CAAC,GAAG,CAAC,CAAC,SAAS,EAAE,EAAE,CAAC,SAAS,CAAC,EAAE,CAAC,CAAC,CAAC;QAChF,MAAM,QAAQ,GAAG,WAAW,CAAC,MAAM,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,UAAU,CAAC,UAAU,KAAK,UAAU,CAAC,CAAC;QAC1F,MAAM,gBAAgB,GAAG,IAAI,GAAG,CAC9B,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAAC,UAAU,CAAC,IAAI,EAAE,CACjF,CAAC;QACF,MAAM,aAAa,GAAG,IAAI,GAAG,CAC3B,QAAQ,CAAC,OAAO,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,IAAI,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,CAAC,CAAC,CAC9E,CAAC;QACF,MAAM,mBAAmB,GAAG,CAAC,GAAG,aAAa,CAAC,CAAC,MAAM,CACnD,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC,IAAI,CAAC,gBAAgB,CAAC,GAAG,CAAC,EAAE,CAAC,CAC1D,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,yCAAyC,UAAU,+DAA+D,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CACnJ,CAAC;QACJ,CAAC;QACD,MAAM,mBAAmB,GAAG,CAAC,GAAG,gBAAgB,CAAC,CAAC,MAAM,CACtD,CAAC,EAAE,EAAE,EAAE,CACL,WAAW,CAAC,GAAG,CAAC,EAAE,CAAC;YACnB,CAAC,QAAQ,CAAC,KAAK,CAAC,CAAC,UAAU,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,CAAC,UAAU,CAAC,QAAQ,CAAC,UAAU,EAAE,EAAE,CAAC,CAAC,CACrF,CAAC;QACF,IAAI,mBAAmB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;YACnC,MAAM,IAAI,KAAK,CACb,oCAAoC,UAAU,mEAAmE,mBAAmB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAClJ,CAAC;QACJ,CAAC;IACH,CAAC;IACD,MAAM,kBAAkB,GAAG,MAAM,CAAC,IAAI,CACpC,SAAS,CAAC,MAAM,CAAC,QAAQ,CAAC,UAAU,CAAC,yBAAyB,CAC/D,CAAC,MAAM,CAAC,CAAC,IAAI,EAAE,EAAE,CAAC,CAAC,WAAW,CAAC,QAAQ,CAAC,IAAI,CAAC,CAAC,CAAC;IAChD,IAAI,kBAAkB,CAAC,MAAM,GAAG,CAAC,EAAE,CAAC;QAClC,MAAM,IAAI,KAAK,CACb,+EAA+E,kBAAkB,CAAC,IAAI,CAAC,IAAI,CAAC,EAAE,CAC/G,CAAC;IACJ,CAAC;AACH,CAAC,CAAC"}

package/lib/orchestrator/experiment-evaluation.d.ts

@@ -0,0 +1,44 @@
+import { z } from "zod";
+import { type Candidate, type ExperimentSelector } from "../core/index.js";
+import { type discoverCases } from "./discover.js";
+import type { CampaignExecutionOptions } from "./experiment-context.js";
+import type { ExperimentEvaluation, OpenedExperimentRun } from "./experiment-runs.js";
+type ExperimentSplit = "train" | "validation" | "test";
+declare const EvaluationAccountingSchema: z.ZodObject<{
+    metricCalls: z.ZodOptional<z.ZodNumber>;
+    judgeCalls: z.ZodOptional<z.ZodNumber>;
+    unpricedModelCalls: z.ZodOptional<z.ZodNumber>;
+    usage: z.ZodOptional<z.ZodObject<{
+        input: z.ZodNumber;
+        output: z.ZodNumber;
+        cacheRead: z.ZodNumber;
+        cacheCreation: z.ZodNumber;
+    }, z.core.$strip>>;
+    costUsd: z.ZodOptional<z.ZodNumber>;
+}, z.core.$strict>;
+export type EvaluationAccounting = z.infer<typeof EvaluationAccountingSchema>;
+export declare const invokeWithVerifiedExecutionIdentity: <T>(verifyExecutionIdentity: () => Promise<void>, invocation: () => Promise<T>) => Promise<T>;
+export declare const evaluateAgentCandidate: (options: {
+    execution: CampaignExecutionOptions;
+    experiment: OpenedExperimentRun;
+    candidate: Candidate;
+    allCases: Awaited<ReturnType<typeof discoverCases>>["cases"];
+    profileId: string;
+    split: ExperimentSplit;
+    selector: ExperimentSelector;
+    repetitions: number;
+    verifyExecutionIdentity: () => Promise<void>;
+}) => Promise<ExperimentEvaluation>;
+export declare const evaluateOfflineGraderCandidate: (options: {
+    execution: CampaignExecutionOptions;
+    experiment: OpenedExperimentRun;
+    candidate: Candidate;
+    materializedRoot: string;
+    profileId: string;
+    split: ExperimentSplit;
+    selector: ExperimentSelector;
+    repetitions: number;
+    verifyExecutionIdentity: () => Promise<void>;
+}) => Promise<ExperimentEvaluation>;
+export {};
+//# sourceMappingURL=experiment-evaluation.d.ts.map

package/lib/orchestrator/experiment-evaluation.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiment-evaluation.d.ts","sourceRoot":"","sources":["../../src/orchestrator/experiment-evaluation.ts"],"names":[],"mappings":"AAGA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,OAAO,EAEL,KAAK,SAAS,EAGd,KAAK,kBAAkB,EAIxB,MAAM,kBAAkB,CAAC;AAG1B,OAAO,EAAE,KAAK,aAAa,EAAe,MAAM,eAAe,CAAC;AAEhE,OAAO,KAAK,EAAE,wBAAwB,EAAE,MAAM,yBAAyB,CAAC;AAExE,OAAO,KAAK,EAAE,oBAAoB,EAAE,mBAAmB,EAAE,MAAM,sBAAsB,CAAC;AAOtF,KAAK,eAAe,GAAG,OAAO,GAAG,YAAY,GAAG,MAAM,CAAC;AAuFvD,QAAA,MAAM,0BAA0B;;;;;;;;;;;kBAQrB,CAAC;AACZ,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAkQ9E,eAAO,MAAM,mCAAmC,GAAU,CAAC,2BAChC,MAAM,OAAO,CAAC,IAAI,CAAC,cAChC,MAAM,OAAO,CAAC,CAAC,CAAC,KAC3B,OAAO,CAAC,CAAC,CAOX,CAAC;AAYF,eAAO,MAAM,sBAAsB,YAAmB;IACpD,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,QAAQ,EAAE,OAAO,CAAC,UAAU,CAAC,OAAO,aAAa,CAAC,CAAC,CAAC,OAAO,CAAC,CAAC;IAC7D,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CAgP/B,CAAC;AAEF,eAAO,MAAM,8BAA8B,YAAmB;IAC5D,SAAS,EAAE,wBAAwB,CAAC;IACpC,UAAU,EAAE,mBAAmB,CAAC;IAChC,SAAS,EAAE,SAAS,CAAC;IACrB,gBAAgB,EAAE,MAAM,CAAC;IACzB,SAAS,EAAE,MAAM,CAAC;IAClB,KAAK,EAAE,eAAe,CAAC;IACvB,QAAQ,EAAE,kBAAkB,CAAC;IAC7B,WAAW,EAAE,MAAM,CAAC;IACpB,uBAAuB,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,CAAC;CAC9C,KAAG,OAAO,CAAC,oBAAoB,CA+E/B,CAAC"}