@openpond/evals 0.12.0 → 0.12.1

package/dist/types/experiments.d.ts

@@ -82,10 +82,10 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
                 }, z.core.$strict>;
             }, z.core.$strict>>;
             split: z.ZodEnum<{
+                frozen_eval: "frozen_eval";
+                test: "test";
                 train: "train";
                 validation: "validation";
-                test: "test";
-                frozen_eval: "frozen_eval";
             }>;
             population: z.ZodArray<z.ZodObject<{
                 taskId: z.ZodString;
@@ -102,8 +102,8 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
                 configurationHash: z.ZodString;
                 output: z.ZodEnum<{
                     boolean: "boolean";
-                    score: "score";
                     category: "category";
+                    score: "score";
                 }>;
                 categories: z.ZodArray<z.ZodString>;
             }, z.core.$strict>>;
@@ -112,17 +112,17 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
                 graderId: z.ZodString;
                 fields: z.ZodArray<z.ZodObject<{
                     destination: z.ZodEnum<{
-                        output: "output";
-                        input: "input";
-                        expectedOutput: "expectedOutput";
                         evaluatorContext: "evaluatorContext";
+                        expectedOutput: "expectedOutput";
+                        input: "input";
+                        output: "output";
                     }>;
                     path: z.ZodArray<z.ZodString>;
                     source: z.ZodEnum<{
-                        output: "output";
-                        input: "input";
-                        expectedOutput: "expectedOutput";
                         evaluatorContext: "evaluatorContext";
+                        expectedOutput: "expectedOutput";
+                        input: "input";
+                        output: "output";
                     }>;
                     sourcePath: z.ZodArray<z.ZodString>;
                 }, z.core.$strict>>;
@@ -181,8 +181,8 @@ export declare const ExperimentEvaluatorSchema: z.ZodObject<{
     configurationHash: z.ZodOptional<z.ZodString>;
     output: z.ZodEnum<{
         boolean: "boolean";
-        score: "score";
         category: "category";
+        score: "score";
     }>;
     categories: z.ZodArray<z.ZodString>;
 }, z.core.$strict>;
@@ -290,10 +290,10 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
                     }, z.core.$strict>;
                 }, z.core.$strict>>;
                 split: z.ZodEnum<{
+                    frozen_eval: "frozen_eval";
+                    test: "test";
                     train: "train";
                     validation: "validation";
-                    test: "test";
-                    frozen_eval: "frozen_eval";
                 }>;
                 population: z.ZodArray<z.ZodObject<{
                     taskId: z.ZodString;
@@ -310,8 +310,8 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
                     configurationHash: z.ZodString;
                     output: z.ZodEnum<{
                         boolean: "boolean";
-                        score: "score";
                         category: "category";
+                        score: "score";
                     }>;
                     categories: z.ZodArray<z.ZodString>;
                 }, z.core.$strict>>;
@@ -320,17 +320,17 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
                     graderId: z.ZodString;
                     fields: z.ZodArray<z.ZodObject<{
                         destination: z.ZodEnum<{
-                            output: "output";
-                            input: "input";
-                            expectedOutput: "expectedOutput";
                             evaluatorContext: "evaluatorContext";
+                            expectedOutput: "expectedOutput";
+                            input: "input";
+                            output: "output";
                         }>;
                         path: z.ZodArray<z.ZodString>;
                         source: z.ZodEnum<{
-                            output: "output";
-                            input: "input";
-                            expectedOutput: "expectedOutput";
                             evaluatorContext: "evaluatorContext";
+                            expectedOutput: "expectedOutput";
+                            input: "input";
+                            output: "output";
                         }>;
                         sourcePath: z.ZodArray<z.ZodString>;
                     }, z.core.$strict>>;
@@ -387,9 +387,9 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
             protocol: z.ZodLiteral<"openpond.evaluation-execution.v1">;
             targetKind: z.ZodEnum<{
                 agent: "agent";
-                model: "model";
                 fixture: "fixture";
                 harness: "harness";
+                model: "model";
             }>;
         }, z.core.$strict>>;
     }, z.core.$strict>>;
@@ -423,8 +423,8 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
         configurationHash: z.ZodOptional<z.ZodString>;
         output: z.ZodEnum<{
             boolean: "boolean";
-            score: "score";
             category: "category";
+            score: "score";
         }>;
         categories: z.ZodArray<z.ZodString>;
     }, z.core.$strict>>;
@@ -528,10 +528,10 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
                     }, z.core.$strict>;
                 }, z.core.$strict>>;
                 split: z.ZodEnum<{
+                    frozen_eval: "frozen_eval";
+                    test: "test";
                     train: "train";
                     validation: "validation";
-                    test: "test";
-                    frozen_eval: "frozen_eval";
                 }>;
                 population: z.ZodArray<z.ZodObject<{
                     taskId: z.ZodString;
@@ -548,8 +548,8 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
                     configurationHash: z.ZodString;
                     output: z.ZodEnum<{
                         boolean: "boolean";
-                        score: "score";
                         category: "category";
+                        score: "score";
                     }>;
                     categories: z.ZodArray<z.ZodString>;
                 }, z.core.$strict>>;
@@ -558,17 +558,17 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
                     graderId: z.ZodString;
                     fields: z.ZodArray<z.ZodObject<{
                         destination: z.ZodEnum<{
-                            output: "output";
-                            input: "input";
-                            expectedOutput: "expectedOutput";
                             evaluatorContext: "evaluatorContext";
+                            expectedOutput: "expectedOutput";
+                            input: "input";
+                            output: "output";
                         }>;
                         path: z.ZodArray<z.ZodString>;
                         source: z.ZodEnum<{
-                            output: "output";
-                            input: "input";
-                            expectedOutput: "expectedOutput";
                             evaluatorContext: "evaluatorContext";
+                            expectedOutput: "expectedOutput";
+                            input: "input";
+                            output: "output";
                         }>;
                         sourcePath: z.ZodArray<z.ZodString>;
                     }, z.core.$strict>>;
@@ -625,9 +625,9 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
             protocol: z.ZodLiteral<"openpond.evaluation-execution.v1">;
             targetKind: z.ZodEnum<{
                 agent: "agent";
-                model: "model";
                 fixture: "fixture";
                 harness: "harness";
+                model: "model";
             }>;
         }, z.core.$strict>>;
     }, z.core.$strict>>;
@@ -661,8 +661,8 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
         configurationHash: z.ZodOptional<z.ZodString>;
         output: z.ZodEnum<{
             boolean: "boolean";
-            score: "score";
             category: "category";
+            score: "score";
         }>;
         categories: z.ZodArray<z.ZodString>;
     }, z.core.$strict>>;
@@ -683,8 +683,8 @@ export declare const ExperimentFeedbackSchema: z.ZodObject<{
         revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
     }, z.core.$strict>;
     status: z.ZodEnum<{
-        pending: "pending";
         failed: "failed";
+        pending: "pending";
         scored: "scored";
         unavailable: "unavailable";
     }>;
@@ -713,8 +713,8 @@ export declare const ExperimentCaseResultSchema: z.ZodObject<{
     }, z.core.$strict>;
     status: z.ZodEnum<{
         cancelled: "cancelled";
-        failed: "failed";
         completed: "completed";
+        failed: "failed";
         unavailable: "unavailable";
     }>;
     output: z.ZodNullable<z.ZodUnknown>;
@@ -730,8 +730,8 @@ export declare const ExperimentCaseResultSchema: z.ZodObject<{
             revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
         }, z.core.$strict>;
         status: z.ZodEnum<{
-            pending: "pending";
             failed: "failed";
+            pending: "pending";
             scored: "scored";
             unavailable: "unavailable";
         }>;
@@ -769,8 +769,8 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
     }, z.core.$strict>;
     status: z.ZodEnum<{
         cancelled: "cancelled";
-        failed: "failed";
         completed: "completed";
+        failed: "failed";
     }>;
     cases: z.ZodArray<z.ZodObject<{
         identity: z.ZodObject<{
@@ -780,8 +780,8 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
         }, z.core.$strict>;
         status: z.ZodEnum<{
             cancelled: "cancelled";
-            failed: "failed";
             completed: "completed";
+            failed: "failed";
             unavailable: "unavailable";
         }>;
         output: z.ZodNullable<z.ZodUnknown>;
@@ -797,8 +797,8 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
                 revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
             }, z.core.$strict>;
             status: z.ZodEnum<{
-                pending: "pending";
                 failed: "failed";
+                pending: "pending";
                 scored: "scored";
                 unavailable: "unavailable";
             }>;
@@ -838,8 +838,8 @@ export declare const ExperimentResultSchema: z.ZodObject<{
     }, z.core.$strict>;
     status: z.ZodEnum<{
         cancelled: "cancelled";
-        failed: "failed";
         completed: "completed";
+        failed: "failed";
     }>;
     cases: z.ZodArray<z.ZodObject<{
         identity: z.ZodObject<{
@@ -849,8 +849,8 @@ export declare const ExperimentResultSchema: z.ZodObject<{
         }, z.core.$strict>;
         status: z.ZodEnum<{
             cancelled: "cancelled";
-            failed: "failed";
             completed: "completed";
+            failed: "failed";
             unavailable: "unavailable";
         }>;
         output: z.ZodNullable<z.ZodUnknown>;
@@ -866,8 +866,8 @@ export declare const ExperimentResultSchema: z.ZodObject<{
                 revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
             }, z.core.$strict>;
             status: z.ZodEnum<{
-                pending: "pending";
                 failed: "failed";
+                pending: "pending";
                 scored: "scored";
                 unavailable: "unavailable";
             }>;
@@ -1004,7 +1004,35 @@ export declare function verifyExperimentEvidence(input: {
                     };
                     declaredProfileCatalogHash: string | null;
                     protectedProfileClosureHash: string;
-                    split: "train" | "validation" | "test" | "frozen_eval";
+                    recordedOrigin?: {
+                        execution: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        originalEvidence: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        members: {
+                            caseId: string;
+                            seed: string;
+                            fixtureId: string;
+                            sourceId: string;
+                            snapshotHash: string;
+                            boundaryRevisionHash: string;
+                        }[];
+                    } | undefined;
+                    gradingSource?: {
+                        pass: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        evidence: {
+                            id: string;
+                            contentHash: string;
+                        };
+                    } | undefined;
+                    split: "frozen_eval" | "test" | "train" | "validation";
                     population: {
                         taskId: string;
                         seed: string;
@@ -1018,16 +1046,16 @@ export declare function verifyExperimentEvidence(input: {
                         };
                         feedbackKey: string;
                         configurationHash: string;
-                        output: "boolean" | "score" | "category";
+                        output: "boolean" | "category" | "score";
                         categories: string[];
                     }[];
                     fieldMappingsHash: string;
                     fieldMappings: {
                         graderId: string;
                         fields: {
-                            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+                            destination: "evaluatorContext" | "expectedOutput" | "input" | "output";
                             path: string[];
-                            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+                            source: "evaluatorContext" | "expectedOutput" | "input" | "output";
                             sourcePath: string[];
                         }[];
                     }[];
@@ -1038,34 +1066,6 @@ export declare function verifyExperimentEvidence(input: {
                         requireComplete: boolean;
                     };
                     contentHash: string;
-                    recordedOrigin?: {
-                        execution: {
-                            id: string;
-                            contentHash: string;
-                        };
-                        originalEvidence: {
-                            id: string;
-                            contentHash: string;
-                        };
-                        members: {
-                            caseId: string;
-                            seed: string;
-                            fixtureId: string;
-                            sourceId: string;
-                            snapshotHash: string;
-                            boundaryRevisionHash: string;
-                        }[];
-                    } | undefined;
-                    gradingSource?: {
-                        pass: {
-                            id: string;
-                            contentHash: string;
-                        };
-                        evidence: {
-                            id: string;
-                            contentHash: string;
-                        };
-                    } | undefined;
                 } | undefined;
                 sourceCandidate?: {
                     candidateId: string;
@@ -1103,31 +1103,13 @@ export declare function verifyExperimentEvidence(input: {
             kind: "fixture";
             configurationHash: string;
         };
-        evaluators: {
-            release: {
-                id: string;
-                contentHash: string;
-                revision: string | number;
-            };
-            feedbackKey: string;
-            output: "boolean" | "score" | "category";
-            categories: string[];
-            configurationHash?: string | undefined;
-        }[];
-        population: {
-            caseId: string;
-            seed: string;
-            fixtureId: string | null;
-        }[];
-        createdAt: string;
-        contentHash: string;
         execution?: {
             packageHash: string;
             runtimeTargetHash: string;
             metricPolicyHash: string;
             compatibility?: {
                 protocol: "openpond.evaluation-execution.v1";
-                targetKind: "agent" | "model" | "fixture" | "harness";
+                targetKind: "agent" | "fixture" | "harness" | "model";
             } | undefined;
         } | undefined;
         lineage?: {
@@ -1150,6 +1132,24 @@ export declare function verifyExperimentEvidence(input: {
                 rulesHash: string;
             } | undefined;
         } | undefined;
+        evaluators: {
+            release: {
+                id: string;
+                contentHash: string;
+                revision: string | number;
+            };
+            feedbackKey: string;
+            configurationHash?: string | undefined;
+            output: "boolean" | "category" | "score";
+            categories: string[];
+        }[];
+        population: {
+            caseId: string;
+            seed: string;
+            fixtureId: string | null;
+        }[];
+        createdAt: string;
+        contentHash: string;
     };
     result: {
         schemaVersion: "openpond.experimentResult.v1";
@@ -1157,14 +1157,14 @@ export declare function verifyExperimentEvidence(input: {
             id: string;
             contentHash: string;
         };
-        status: "cancelled" | "failed" | "completed";
+        status: "cancelled" | "completed" | "failed";
         cases: {
             identity: {
                 caseId: string;
                 seed: string;
                 fixtureId: string | null;
             };
-            status: "cancelled" | "failed" | "completed" | "unavailable";
+            status: "cancelled" | "completed" | "failed" | "unavailable";
             output: unknown;
             error: {
                 code: string;
@@ -1177,8 +1177,9 @@ export declare function verifyExperimentEvidence(input: {
                     contentHash: string;
                     revision: string | number;
                 };
-                status: "pending" | "failed" | "scored" | "unavailable";
+                status: "failed" | "pending" | "scored" | "unavailable";
                 value: string | number | boolean | null;
+                passed?: boolean | null | undefined;
                 reasoning: string | null;
                 evidenceRefs: {
                     id: string;
@@ -1186,7 +1187,6 @@ export declare function verifyExperimentEvidence(input: {
                     mediaType: string | null;
                     sizeBytes: number | null;
                 }[];
-                passed?: boolean | null | undefined;
             }[];
             usage: {
                 inputTokens: number | null;

package/dist/types/experiments.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,wBAAgB,qBAAqB,CAAC,gBAAgB,EAAE;IAAC,EAAE,EAAC,MAAM,CAAC;IAAA,WAAW,EAAC,MAAM,CAAA;CAAC,EAAE,aAAa,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,GAAG,MAAM,CAEtI;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA8C1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA+EtB"}
+{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,wBAAgB,qBAAqB,CAAC,gBAAgB,EAAE;IAAC,EAAE,EAAC,MAAM,CAAC;IAAA,WAAW,EAAC,MAAM,CAAA;CAAC,EAAE,aAAa,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,GAAG,MAAM,CAEtI;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA8C1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA+EtB"}

package/dist/types/graders.d.ts

@@ -5,12 +5,12 @@ export declare const ModelJudgeTransportPolicySchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.modelJudgeTransportPolicy.v1">;
     responseFormat: z.ZodLiteral<"judgment_json_schema">;
     reasoningEffort: z.ZodEnum<{
-        off: "off";
+        high: "high";
         low: "low";
+        max: "max";
         medium: "medium";
-        high: "high";
+        off: "off";
         xhigh: "xhigh";
-        max: "max";
     }>;
     maximumOutputTokens: z.ZodNumber;
 }, z.core.$strict>;
@@ -30,12 +30,12 @@ export declare const ModelJudgeReceiptSchema: z.ZodObject<{
         schemaVersion: z.ZodLiteral<"openpond.modelJudgeTransportPolicy.v1">;
         responseFormat: z.ZodLiteral<"judgment_json_schema">;
         reasoningEffort: z.ZodEnum<{
-            off: "off";
+            high: "high";
             low: "low";
+            max: "max";
             medium: "medium";
-            high: "high";
+            off: "off";
             xhigh: "xhigh";
-            max: "max";
         }>;
         maximumOutputTokens: z.ZodNumber;
     }, z.core.$strict>>;
@@ -48,8 +48,8 @@ export declare const GraderEvidenceContentSchema: z.ZodObject<{
     graderId: z.ZodString;
     graderVersion: z.ZodString;
     status: z.ZodOptional<z.ZodEnum<{
-        pending: "pending";
         failed: "failed";
+        pending: "pending";
         scored: "scored";
         unavailable: "unavailable";
     }>>;
@@ -57,12 +57,12 @@ export declare const GraderEvidenceContentSchema: z.ZodObject<{
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
     failureClass: z.ZodNullable<z.ZodEnum<{
-        policy_failure: "policy_failure";
-        grader_failure: "grader_failure";
+        cancelled: "cancelled";
         environment_failure: "environment_failure";
+        grader_failure: "grader_failure";
         infrastructure_failure: "infrastructure_failure";
+        policy_failure: "policy_failure";
         timeout: "timeout";
-        cancelled: "cancelled";
     }>>;
     feedback: z.ZodArray<z.ZodString>;
     visibleEvidenceRefs: z.ZodArray<z.ZodString>;
@@ -82,12 +82,12 @@ export declare const GraderEvidenceContentSchema: z.ZodObject<{
             schemaVersion: z.ZodLiteral<"openpond.modelJudgeTransportPolicy.v1">;
             responseFormat: z.ZodLiteral<"judgment_json_schema">;
             reasoningEffort: z.ZodEnum<{
-                off: "off";
+                high: "high";
                 low: "low";
+                max: "max";
                 medium: "medium";
-                high: "high";
+                off: "off";
                 xhigh: "xhigh";
-                max: "max";
             }>;
             maximumOutputTokens: z.ZodNumber;
         }, z.core.$strict>>;
@@ -101,8 +101,8 @@ export declare const GraderEvidenceSchema: z.ZodObject<{
     graderId: z.ZodString;
     graderVersion: z.ZodString;
     status: z.ZodOptional<z.ZodEnum<{
-        pending: "pending";
         failed: "failed";
+        pending: "pending";
         scored: "scored";
         unavailable: "unavailable";
     }>>;
@@ -110,12 +110,12 @@ export declare const GraderEvidenceSchema: z.ZodObject<{
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
     failureClass: z.ZodNullable<z.ZodEnum<{
-        policy_failure: "policy_failure";
-        grader_failure: "grader_failure";
+        cancelled: "cancelled";
         environment_failure: "environment_failure";
+        grader_failure: "grader_failure";
         infrastructure_failure: "infrastructure_failure";
+        policy_failure: "policy_failure";
         timeout: "timeout";
-        cancelled: "cancelled";
     }>>;
     feedback: z.ZodArray<z.ZodString>;
     visibleEvidenceRefs: z.ZodArray<z.ZodString>;
@@ -135,12 +135,12 @@ export declare const GraderEvidenceSchema: z.ZodObject<{
             schemaVersion: z.ZodLiteral<"openpond.modelJudgeTransportPolicy.v1">;
             responseFormat: z.ZodLiteral<"judgment_json_schema">;
             reasoningEffort: z.ZodEnum<{
-                off: "off";
+                high: "high";
                 low: "low";
+                max: "max";
                 medium: "medium";
-                high: "high";
+                off: "off";
                 xhigh: "xhigh";
-                max: "max";
             }>;
             maximumOutputTokens: z.ZodNumber;
         }, z.core.$strict>>;
@@ -192,18 +192,18 @@ declare const GradeSummarySchema: z.ZodObject<{
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
     failureClass: z.ZodNullable<z.ZodEnum<{
-        policy_failure: "policy_failure";
-        grader_failure: "grader_failure";
+        cancelled: "cancelled";
         environment_failure: "environment_failure";
+        grader_failure: "grader_failure";
         infrastructure_failure: "infrastructure_failure";
+        policy_failure: "policy_failure";
         timeout: "timeout";
-        cancelled: "cancelled";
     }>>;
     gradingStatus: z.ZodEnum<{
+        not_configured: "not_configured";
         pending: "pending";
         scored: "scored";
         unscorable: "unscorable";
-        not_configured: "not_configured";
     }>;
 }, z.core.$strict>;
 /** Shared ordinary grading policy. Results must match the declared population;
@@ -218,18 +218,18 @@ export declare const TaskGradeSchema: z.ZodObject<{
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
     failureClass: z.ZodNullable<z.ZodEnum<{
-        policy_failure: "policy_failure";
-        grader_failure: "grader_failure";
+        cancelled: "cancelled";
         environment_failure: "environment_failure";
+        grader_failure: "grader_failure";
         infrastructure_failure: "infrastructure_failure";
+        policy_failure: "policy_failure";
         timeout: "timeout";
-        cancelled: "cancelled";
     }>>;
     gradingStatus: z.ZodEnum<{
+        not_configured: "not_configured";
         pending: "pending";
         scored: "scored";
         unscorable: "unscorable";
-        not_configured: "not_configured";
     }>;
     schemaVersion: z.ZodLiteral<"openpond.taskGrade.v1">;
     taskHash: z.ZodString;
@@ -240,8 +240,8 @@ export declare const TaskGradeSchema: z.ZodObject<{
         graderId: z.ZodString;
         graderVersion: z.ZodString;
         status: z.ZodOptional<z.ZodEnum<{
-            pending: "pending";
             failed: "failed";
+            pending: "pending";
             scored: "scored";
             unavailable: "unavailable";
         }>>;
@@ -249,12 +249,12 @@ export declare const TaskGradeSchema: z.ZodObject<{
         passed: z.ZodBoolean;
         rewardEligible: z.ZodBoolean;
         failureClass: z.ZodNullable<z.ZodEnum<{
-            policy_failure: "policy_failure";
-            grader_failure: "grader_failure";
+            cancelled: "cancelled";
             environment_failure: "environment_failure";
+            grader_failure: "grader_failure";
             infrastructure_failure: "infrastructure_failure";
+            policy_failure: "policy_failure";
             timeout: "timeout";
-            cancelled: "cancelled";
         }>>;
         feedback: z.ZodArray<z.ZodString>;
         visibleEvidenceRefs: z.ZodArray<z.ZodString>;
@@ -274,12 +274,12 @@ export declare const TaskGradeSchema: z.ZodObject<{
                 schemaVersion: z.ZodLiteral<"openpond.modelJudgeTransportPolicy.v1">;
                 responseFormat: z.ZodLiteral<"judgment_json_schema">;
                 reasoningEffort: z.ZodEnum<{
-                    off: "off";
+                    high: "high";
                     low: "low";
+                    max: "max";
                     medium: "medium";
-                    high: "high";
+                    off: "off";
                     xhigh: "xhigh";
-                    max: "max";
                 }>;
                 maximumOutputTokens: z.ZodNumber;
             }, z.core.$strict>>;