@openpond/evals 0.11.10 → 0.11.11

package/dist/experiments.js

@@ -0,0 +1,169 @@
+import { z } from "zod";
+import { ImmutableArtifactRefSchema, ImmutableReleaseRefSchema, ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema, assertContentHash, contentHash, } from "@openpond/harness";
+import { ProfileEvaluationRunSourceSchema } from "./profile-evaluations.js";
+import { FeedbackKeySchema } from "./rewards.js";
+const MoneySchema = z.number().finite().nonnegative();
+export const ExperimentTargetSchema = z.discriminatedUnion("kind", [
+    z.object({
+        kind: z.literal("model"),
+        modelId: ReleaseIdSchema,
+        configurationHash: ReleaseHashSchema,
+    }).strict(),
+    z.object({
+        kind: z.literal("harness"),
+        source: ProfileEvaluationRunSourceSchema,
+    }).strict(),
+]);
+export const ExperimentEvaluatorSchema = z.object({
+    release: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
+    feedbackKey: FeedbackKeySchema,
+    output: z.enum(["boolean", "score", "category"]),
+    /** Category labels are meaningful only for category output. */
+    categories: z.array(z.string().trim().min(1).max(120)).max(50),
+}).strict().superRefine((value, context) => {
+    if ((value.output === "category") !== (value.categories.length > 0))
+        context.addIssue({ code: "custom", path: ["categories"], message: "Only category feedback declares categories." });
+    if (new Set(value.categories).size !== value.categories.length)
+        context.addIssue({ code: "custom", path: ["categories"], message: "Feedback categories must be unique." });
+});
+export const ExperimentCaseIdentitySchema = z.object({
+    caseId: ReleaseIdSchema,
+    seed: z.string().trim().min(1).max(500),
+    fixtureId: ReleaseIdSchema.nullable(),
+}).strict();
+export function experimentCaseKey(identity) {
+    const value = ExperimentCaseIdentitySchema.parse(identity);
+    return JSON.stringify([value.caseId, value.seed, value.fixtureId]);
+}
+export const ExperimentManifestContentSchema = z.object({
+    schemaVersion: z.literal("openpond.experimentManifest.v1"),
+    id: ReleaseIdSchema,
+    name: z.string().trim().min(1).max(500),
+    teamId: ReleaseIdSchema,
+    operationId: ReleaseIdSchema,
+    maximumCostUsd: MoneySchema,
+    dataset: ImmutableReleaseRefSchema.extend({ revision: z.number().int().positive() }).strict(),
+    target: ExperimentTargetSchema,
+    evaluators: z.array(ExperimentEvaluatorSchema).max(100),
+    population: z.array(ExperimentCaseIdentitySchema).min(1).max(10_000),
+    createdAt: ReleaseTimestampSchema,
+}).strict().superRefine((value, context) => {
+    if (new Set(value.population.map(experimentCaseKey)).size !== value.population.length)
+        context.addIssue({ code: "custom", path: ["population"], message: "Experiment cases must be unique." });
+    if (new Set(value.evaluators.map((item) => item.feedbackKey)).size !== value.evaluators.length)
+        context.addIssue({ code: "custom", path: ["evaluators"], message: "Experiment feedback keys must be unique." });
+});
+export const ExperimentManifestSchema = ExperimentManifestContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export const ExperimentFeedbackSchema = z.object({
+    feedbackKey: FeedbackKeySchema,
+    evaluator: ExperimentEvaluatorSchema.shape.release,
+    status: z.enum(["scored", "unavailable", "failed", "pending"]),
+    value: z.union([z.boolean(), z.number().finite(), z.string().max(120)]).nullable(),
+    reasoning: z.string().max(20_000).nullable(),
+    evidenceRefs: z.array(ImmutableArtifactRefSchema).max(1_000),
+}).strict();
+export const ExperimentUsageSchema = z.object({
+    inputTokens: z.number().int().nonnegative().nullable(),
+    outputTokens: z.number().int().nonnegative().nullable(),
+    totalTokens: z.number().int().nonnegative().nullable(),
+    costUsd: MoneySchema.nullable(),
+    latencyMs: z.number().finite().nonnegative().nullable(),
+}).strict().superRefine((usage, context) => {
+    if (usage.inputTokens !== null && usage.outputTokens !== null && usage.totalTokens !== null
+        && usage.totalTokens < usage.inputTokens + usage.outputTokens)
+        context.addIssue({ code: "custom", path: ["totalTokens"], message: "Total tokens cannot be less than input plus output." });
+});
+export const ExperimentCaseResultSchema = z.object({
+    identity: ExperimentCaseIdentitySchema,
+    status: z.enum(["completed", "failed", "cancelled", "unavailable"]),
+    output: z.unknown().nullable(),
+    error: z.object({ code: ReleaseIdSchema, message: z.string().max(2_000) }).strict().nullable(),
+    feedback: z.array(ExperimentFeedbackSchema).max(100),
+    usage: ExperimentUsageSchema,
+    traceRef: ImmutableArtifactRefSchema.nullable(),
+    startedAt: ReleaseTimestampSchema.nullable(),
+    completedAt: ReleaseTimestampSchema.nullable(),
+}).strict();
+export const ExperimentResultContentSchema = z.object({
+    schemaVersion: z.literal("openpond.experimentResult.v1"),
+    manifest: ImmutableReleaseRefSchema,
+    status: z.enum(["completed", "failed", "cancelled"]),
+    cases: z.array(ExperimentCaseResultSchema).min(1).max(10_000),
+    completedAt: ReleaseTimestampSchema,
+}).strict();
+export const ExperimentResultSchema = ExperimentResultContentSchema.extend({ contentHash: ReleaseHashSchema }).strict();
+export function createExperimentManifest(input) {
+    const value = ExperimentManifestContentSchema.parse(input);
+    return ExperimentManifestSchema.parse({ ...value, contentHash: contentHash(value) });
+}
+export function createExperimentResult(input, manifest) {
+    ExperimentManifestSchema.parse(manifest);
+    assertContentHash(manifest, "Experiment manifest");
+    const value = ExperimentResultContentSchema.parse(input);
+    if (value.manifest.id !== manifest.id || value.manifest.contentHash !== manifest.contentHash)
+        throw new Error("Experiment result differs from its manifest.");
+    const expected = new Set(manifest.population.map(experimentCaseKey));
+    const seen = new Set();
+    const evaluators = new Map(manifest.evaluators.map((item) => [item.feedbackKey, item]));
+    for (const result of value.cases) {
+        const key = experimentCaseKey(result.identity);
+        if (!expected.has(key) || seen.has(key))
+            throw new Error("Experiment result has an unexpected or repeated case.");
+        seen.add(key);
+        const feedbackKeys = new Set();
+        for (const feedback of result.feedback) {
+            const evaluator = evaluators.get(feedback.feedbackKey);
+            if (!evaluator || feedbackKeys.has(feedback.feedbackKey)
+                || feedback.evaluator.id !== evaluator.release.id
+                || feedback.evaluator.contentHash !== evaluator.release.contentHash
+                || feedback.evaluator.revision !== evaluator.release.revision)
+                throw new Error("Experiment feedback differs from its pinned evaluator.");
+            feedbackKeys.add(feedback.feedbackKey);
+            if (feedback.status === "scored") {
+                if (feedback.value === null || typeof feedback.value !== { boolean: "boolean", score: "number", category: "string" }[evaluator.output]
+                    || (evaluator.output === "category" && !evaluator.categories.includes(feedback.value))
+                    || (evaluator.output === "score" && (feedback.value < 0 || feedback.value > 1)))
+                    throw new Error("Experiment feedback value violates its output contract.");
+            }
+            else if (feedback.value !== null)
+                throw new Error("Unscored experiment feedback cannot claim a value.");
+        }
+    }
+    if (seen.size !== expected.size)
+        throw new Error("Experiment result omits admitted cases.");
+    return ExperimentResultSchema.parse({ ...value, contentHash: contentHash(value) });
+}
+/** A comparison reads retained results; it never executes or regrades a case. */
+export function compareExperiments(baseline, candidate) {
+    for (const pair of [baseline, candidate]) {
+        const { contentHash: hash, ...value } = ExperimentResultSchema.parse(pair.result);
+        if (contentHash(value) !== hash)
+            throw new Error("Experiment result integrity failed.");
+        createExperimentResult(value, pair.manifest);
+    }
+    const reasons = [];
+    if (baseline.manifest.teamId !== candidate.manifest.teamId)
+        reasons.push("different_workspace");
+    if (baseline.manifest.dataset.id !== candidate.manifest.dataset.id
+        || baseline.manifest.dataset.contentHash !== candidate.manifest.dataset.contentHash
+        || baseline.manifest.dataset.revision !== candidate.manifest.dataset.revision)
+        reasons.push("different_dataset_version");
+    if (baseline.manifest.target.kind !== candidate.manifest.target.kind)
+        reasons.push("different_target_kind");
+    if (contentHash(baseline.manifest.evaluators) !== contentHash(candidate.manifest.evaluators))
+        reasons.push("different_evaluators");
+    const left = new Map(baseline.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
+    const right = new Map(candidate.result.cases.map((item) => [experimentCaseKey(item.identity), item]));
+    const keys = [...new Set([...left.keys(), ...right.keys()])].sort();
+    if (left.size !== right.size || keys.some((key) => !left.has(key) || !right.has(key)))
+        reasons.push("different_population");
+    return {
+        comparable: reasons.length === 0,
+        reasons,
+        cases: keys.map((key) => ({
+            identity: (left.get(key) ?? right.get(key)).identity,
+            baseline: left.get(key) ?? null,
+            candidate: right.get(key) ?? null,
+        })),
+    };
+}

package/dist/index.js

@@ -4,6 +4,7 @@ export * from "./artifact-verification.js";
 export * from "./evidence/index.js";
 export * from "./execution-contracts.js";
 export * from "./execution-receipts.js";
+export * from "./experiments.js";
 export * from "./graders.js";
 export * from "./harness.js";
 export * from "./learned-preference.js";

package/dist/learning/authoring.js

@@ -1,6 +1,6 @@
 import { z } from "zod";
 import { ReleaseHashSchema, ReleaseIdSchema, ReleaseTimestampSchema } from "@openpond/harness";
-import { RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
+import { FeedbackKeySchema, RewardBindingSourceSchema, RewardReleaseRefSchema } from "../rewards.js";
 import { LearningRevisionRefSchema } from "./contracts.js";
 /** Strings intentionally preserve incomplete JSON/code until explicit publication. */
 export const RewardFixtureAuthoringFieldsSchema = z.object({
@@ -14,6 +14,7 @@ export const RewardFixtureAuthoringFieldsSchema = z.object({
 }).strict();
 export const RewardAuthoringFieldsSchema = z.object({
     name: z.string().max(500), description: z.string().max(10_000),
+    feedbackKey: FeedbackKeySchema.optional(),
     kind: z.enum(["custom_verifier", "state", "content", "schema", "artifact", "runtime_event", "model_judge", "learned_model", "human"]),
     fields: z.string(), outputField: z.string(), expectedField: z.string(), expectedValue: z.string(),
     schema: z.string(), reference: z.string(), events: z.string(), code: z.string(), exportName: z.string(), timeout: z.string(),

package/dist/learning/reward-authoring.js

@@ -1,4 +1,4 @@
-import { RewardReleaseSchema, createRewardRelease } from "../rewards.js";
+import { RewardReleaseSchema, createRewardRelease, feedbackKeyForReward } from "../rewards.js";
 import { assertBoundedTaskJson } from "../task-schema.js";
 import { RewardAuthoringFieldsSchema, RewardFixtureAuthoringFieldsSchema } from "./authoring.js";
 import { createLearningTextAsset, verifyLearningTextAsset } from "./assets.js";
@@ -64,7 +64,7 @@ export function compileRewardAuthoring(input) {
     const assets = [...(asset ? [asset] : []), ...(fixtureAsset ? [fixtureAsset] : [])];
     const reward = createRewardRelease({
         schemaVersion: "openpond.rewardRelease.v1", id: input.id, revision: (base?.revision ?? 0) + 1,
-        name: fields.name, description: fields.description, implementation,
+        name: fields.name, description: fields.description, feedbackKey: fields.feedbackKey ?? feedbackKeyForReward({ id: input.id }), implementation,
         rawScore: fields.kind === "learned_model" ? { minimum: authoringNumber(fields.minimum, "Minimum score"), maximum: authoringNumber(fields.maximum, "Maximum score") } : { minimum: 0, maximum: 1 },
         assets: assets.map(value => value.asset), ...(fixtureAsset ? { fixtureSetRef: fixtureAsset.asset } : {}),
         ...(calibrationCheckRef ? { calibrationCheckRef } : {}),
@@ -84,7 +84,7 @@ export function rewardAuthoringFields(reward, sourceAsset, fixtureAsset = null)
     const fixtures = reward?.fixtureSetRef && fixtureAsset ? RewardFixtureAuthoringFieldsSchema.array().max(50).parse(JSON.parse(verifyLearningTextAsset(fixtureAsset, reward.fixtureSetRef))) : [];
     const config = implementation && "config" in implementation ? implementation.config : {};
     return {
-        name: reward?.name ?? "", description: reward?.description ?? "", kind: implementation?.kind ?? "state",
+        name: reward?.name ?? "", description: reward?.description ?? "", ...(reward?.feedbackKey ? { feedbackKey: reward.feedbackKey } : {}), kind: implementation?.kind ?? "state",
         fields: strings(config.fields).join(", ") || "answer", outputField: text(config.outputField) || "text", expectedField: text(config.expectedField) || "text", expectedValue: text(config.expectedValue),
         schema: JSON.stringify(config.jsonSchema ?? { type: "object", properties: { answer: { type: "string" } }, required: ["answer"] }, null, 2),
         reference: text(config.refIncludes), events: strings(config.requiredEvents).join(", "),

package/dist/rewards.js

@@ -5,6 +5,10 @@ import { gradeEvidence, GraderEvidenceSchema } from "./graders.js";
 import { CustomVerifierGraderSpecSchema, DeterministicGraderSpecSchema, GraderSpecSchema, HumanGraderSpecSchema, ModelJudgeGraderSpecSchema } from "./tasksets.js";
 import { assertBoundedTaskJson, validateTaskSchema } from "./task-schema.js";
 const bindingFields = { id: true, version: true, weight: true, hardGate: true, rewardEligible: true, privileged: true };
+export const FeedbackKeySchema = z.string().regex(/^[a-z][a-z0-9_]{0,79}$/);
+export function feedbackKeyForReward(reward) {
+    return reward.feedbackKey ?? `reward_${contentHash(reward.id).slice(0, 24)}`;
+}
 export const RewardImplementationSchema = z.union([
     DeterministicGraderSpecSchema.omit(bindingFields),
     ModelJudgeGraderSpecSchema.omit(bindingFields),
@@ -18,6 +22,7 @@ export const RewardReleaseContentSchema = z.object({
     revision: z.number().int().positive(),
     name: z.string().trim().min(1).max(500),
     description: z.string().max(10_000),
+    feedbackKey: FeedbackKeySchema.optional(),
     implementation: RewardImplementationSchema,
     rawScore: z.object({ minimum: z.number().finite(), maximum: z.number().finite() }).strict(),
     assets: z.array(ImmutableAssetRefSchema).max(1_000),

package/dist/types/experiments.d.ts

@@ -0,0 +1,426 @@
+import { z } from "zod";
+export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObject<{
+    kind: z.ZodLiteral<"model">;
+    modelId: z.ZodString;
+    configurationHash: z.ZodString;
+}, z.core.$strict>, z.ZodObject<{
+    kind: z.ZodLiteral<"harness">;
+    source: z.ZodObject<{
+        profileId: z.ZodString;
+        sourceRevision: z.ZodString;
+        harnessRelease: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        catalogHash: z.ZodString;
+        definitionId: z.ZodString;
+        definitionHash: z.ZodString;
+        target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+            kind: z.ZodLiteral<"profile">;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"workflow">;
+            workflowId: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"skill">;
+            skillPath: z.ZodString;
+        }, z.core.$strict>, z.ZodObject<{
+            kind: z.ZodLiteral<"agent_action">;
+            actionId: z.ZodString;
+        }, z.core.$strict>], "kind">;
+        environmentHash: z.ZodString;
+    }, z.core.$strict>;
+}, z.core.$strict>], "kind">;
+export declare const ExperimentEvaluatorSchema: z.ZodObject<{
+    release: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    feedbackKey: z.ZodString;
+    output: z.ZodEnum<{
+        boolean: "boolean";
+        score: "score";
+        category: "category";
+    }>;
+    categories: z.ZodArray<z.ZodString>;
+}, z.core.$strict>;
+export declare const ExperimentCaseIdentitySchema: z.ZodObject<{
+    caseId: z.ZodString;
+    seed: z.ZodString;
+    fixtureId: z.ZodNullable<z.ZodString>;
+}, z.core.$strict>;
+export type ExperimentCaseIdentity = z.infer<typeof ExperimentCaseIdentitySchema>;
+export declare function experimentCaseKey(identity: ExperimentCaseIdentity): string;
+export declare const ExperimentManifestContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.experimentManifest.v1">;
+    id: z.ZodString;
+    name: z.ZodString;
+    teamId: z.ZodString;
+    operationId: z.ZodString;
+    maximumCostUsd: z.ZodNumber;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        modelId: z.ZodString;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"harness">;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>], "kind">;
+    evaluators: z.ZodArray<z.ZodObject<{
+        release: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        feedbackKey: z.ZodString;
+        output: z.ZodEnum<{
+            boolean: "boolean";
+            score: "score";
+            category: "category";
+        }>;
+        categories: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    population: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+}, z.core.$strict>;
+export declare const ExperimentManifestSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.experimentManifest.v1">;
+    id: z.ZodString;
+    name: z.ZodString;
+    teamId: z.ZodString;
+    operationId: z.ZodString;
+    maximumCostUsd: z.ZodNumber;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        modelId: z.ZodString;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"harness">;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>], "kind">;
+    evaluators: z.ZodArray<z.ZodObject<{
+        release: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        feedbackKey: z.ZodString;
+        output: z.ZodEnum<{
+            boolean: "boolean";
+            score: "score";
+            category: "category";
+        }>;
+        categories: z.ZodArray<z.ZodString>;
+    }, z.core.$strict>>;
+    population: z.ZodArray<z.ZodObject<{
+        caseId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    createdAt: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ExperimentManifest = z.infer<typeof ExperimentManifestSchema>;
+export declare const ExperimentFeedbackSchema: z.ZodObject<{
+    feedbackKey: z.ZodString;
+    evaluator: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        revision: z.ZodNumber;
+    }, z.core.$strict>;
+    status: z.ZodEnum<{
+        pending: "pending";
+        failed: "failed";
+        scored: "scored";
+        unavailable: "unavailable";
+    }>;
+    value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
+    reasoning: z.ZodNullable<z.ZodString>;
+    evidenceRefs: z.ZodArray<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+}, z.core.$strict>;
+export declare const ExperimentUsageSchema: z.ZodObject<{
+    inputTokens: z.ZodNullable<z.ZodNumber>;
+    outputTokens: z.ZodNullable<z.ZodNumber>;
+    totalTokens: z.ZodNullable<z.ZodNumber>;
+    costUsd: z.ZodNullable<z.ZodNumber>;
+    latencyMs: z.ZodNullable<z.ZodNumber>;
+}, z.core.$strict>;
+export declare const ExperimentCaseResultSchema: z.ZodObject<{
+    identity: z.ZodObject<{
+        caseId: z.ZodString;
+        seed: z.ZodString;
+        fixtureId: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>;
+    status: z.ZodEnum<{
+        failed: "failed";
+        completed: "completed";
+        cancelled: "cancelled";
+        unavailable: "unavailable";
+    }>;
+    output: z.ZodNullable<z.ZodUnknown>;
+    error: z.ZodNullable<z.ZodObject<{
+        code: z.ZodString;
+        message: z.ZodString;
+    }, z.core.$strict>>;
+    feedback: z.ZodArray<z.ZodObject<{
+        feedbackKey: z.ZodString;
+        evaluator: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            revision: z.ZodNumber;
+        }, z.core.$strict>;
+        status: z.ZodEnum<{
+            pending: "pending";
+            failed: "failed";
+            scored: "scored";
+            unavailable: "unavailable";
+        }>;
+        value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
+        reasoning: z.ZodNullable<z.ZodString>;
+        evidenceRefs: z.ZodArray<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+        }, z.core.$strict>>;
+    }, z.core.$strict>>;
+    usage: z.ZodObject<{
+        inputTokens: z.ZodNullable<z.ZodNumber>;
+        outputTokens: z.ZodNullable<z.ZodNumber>;
+        totalTokens: z.ZodNullable<z.ZodNumber>;
+        costUsd: z.ZodNullable<z.ZodNumber>;
+        latencyMs: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>;
+    traceRef: z.ZodNullable<z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+        mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+        sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+    }, z.core.$strict>>;
+    startedAt: z.ZodNullable<z.ZodString>;
+    completedAt: z.ZodNullable<z.ZodString>;
+}, z.core.$strict>;
+export declare const ExperimentResultContentSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.experimentResult.v1">;
+    manifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    status: z.ZodEnum<{
+        failed: "failed";
+        completed: "completed";
+        cancelled: "cancelled";
+    }>;
+    cases: z.ZodArray<z.ZodObject<{
+        identity: z.ZodObject<{
+            caseId: z.ZodString;
+            seed: z.ZodString;
+            fixtureId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        status: z.ZodEnum<{
+            failed: "failed";
+            completed: "completed";
+            cancelled: "cancelled";
+            unavailable: "unavailable";
+        }>;
+        output: z.ZodNullable<z.ZodUnknown>;
+        error: z.ZodNullable<z.ZodObject<{
+            code: z.ZodString;
+            message: z.ZodString;
+        }, z.core.$strict>>;
+        feedback: z.ZodArray<z.ZodObject<{
+            feedbackKey: z.ZodString;
+            evaluator: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                revision: z.ZodNumber;
+            }, z.core.$strict>;
+            status: z.ZodEnum<{
+                pending: "pending";
+                failed: "failed";
+                scored: "scored";
+                unavailable: "unavailable";
+            }>;
+            value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
+            reasoning: z.ZodNullable<z.ZodString>;
+            evidenceRefs: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+            }, z.core.$strict>>;
+        }, z.core.$strict>>;
+        usage: z.ZodObject<{
+            inputTokens: z.ZodNullable<z.ZodNumber>;
+            outputTokens: z.ZodNullable<z.ZodNumber>;
+            totalTokens: z.ZodNullable<z.ZodNumber>;
+            costUsd: z.ZodNullable<z.ZodNumber>;
+            latencyMs: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>;
+        traceRef: z.ZodNullable<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+        }, z.core.$strict>>;
+        startedAt: z.ZodNullable<z.ZodString>;
+        completedAt: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    completedAt: z.ZodString;
+}, z.core.$strict>;
+export declare const ExperimentResultSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.experimentResult.v1">;
+    manifest: z.ZodObject<{
+        id: z.ZodString;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    status: z.ZodEnum<{
+        failed: "failed";
+        completed: "completed";
+        cancelled: "cancelled";
+    }>;
+    cases: z.ZodArray<z.ZodObject<{
+        identity: z.ZodObject<{
+            caseId: z.ZodString;
+            seed: z.ZodString;
+            fixtureId: z.ZodNullable<z.ZodString>;
+        }, z.core.$strict>;
+        status: z.ZodEnum<{
+            failed: "failed";
+            completed: "completed";
+            cancelled: "cancelled";
+            unavailable: "unavailable";
+        }>;
+        output: z.ZodNullable<z.ZodUnknown>;
+        error: z.ZodNullable<z.ZodObject<{
+            code: z.ZodString;
+            message: z.ZodString;
+        }, z.core.$strict>>;
+        feedback: z.ZodArray<z.ZodObject<{
+            feedbackKey: z.ZodString;
+            evaluator: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                revision: z.ZodNumber;
+            }, z.core.$strict>;
+            status: z.ZodEnum<{
+                pending: "pending";
+                failed: "failed";
+                scored: "scored";
+                unavailable: "unavailable";
+            }>;
+            value: z.ZodNullable<z.ZodUnion<readonly [z.ZodBoolean, z.ZodNumber, z.ZodString]>>;
+            reasoning: z.ZodNullable<z.ZodString>;
+            evidenceRefs: z.ZodArray<z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+                sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+            }, z.core.$strict>>;
+        }, z.core.$strict>>;
+        usage: z.ZodObject<{
+            inputTokens: z.ZodNullable<z.ZodNumber>;
+            outputTokens: z.ZodNullable<z.ZodNumber>;
+            totalTokens: z.ZodNullable<z.ZodNumber>;
+            costUsd: z.ZodNullable<z.ZodNumber>;
+            latencyMs: z.ZodNullable<z.ZodNumber>;
+        }, z.core.$strict>;
+        traceRef: z.ZodNullable<z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+            mediaType: z.ZodDefault<z.ZodNullable<z.ZodString>>;
+            sizeBytes: z.ZodDefault<z.ZodNullable<z.ZodNumber>>;
+        }, z.core.$strict>>;
+        startedAt: z.ZodNullable<z.ZodString>;
+        completedAt: z.ZodNullable<z.ZodString>;
+    }, z.core.$strict>>;
+    completedAt: z.ZodString;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type ExperimentResult = z.infer<typeof ExperimentResultSchema>;
+export declare function createExperimentManifest(input: z.input<typeof ExperimentManifestContentSchema>): ExperimentManifest;
+export declare function createExperimentResult(input: z.input<typeof ExperimentResultContentSchema>, manifest: ExperimentManifest): ExperimentResult;
+export type ExperimentComparison = {
+    comparable: boolean;
+    reasons: string[];
+    cases: Array<{
+        identity: ExperimentCaseIdentity;
+        baseline: ExperimentCaseResult | null;
+        candidate: ExperimentCaseResult | null;
+    }>;
+};
+export type ExperimentCaseResult = z.infer<typeof ExperimentCaseResultSchema>;
+/** A comparison reads retained results; it never executes or regrades a case. */
+export declare function compareExperiments(baseline: {
+    manifest: ExperimentManifest;
+    result: ExperimentResult;
+}, candidate: {
+    manifest: ExperimentManifest;
+    result: ExperimentResult;
+}): ExperimentComparison;
+//# sourceMappingURL=experiments.d.ts.map

package/dist/types/experiments.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAexB,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAUjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;kBAWpC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAiB1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;kBAO1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAA;KAAE,CAAC,CAAC;CACnI,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAE9E,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA0BtB"}