@openpond/evals 0.11.16 → 0.11.18

package/dist/types/experiments.d.ts

@@ -30,6 +30,121 @@ export declare const ExperimentTargetSchema: z.ZodDiscriminatedUnion<[z.ZodObjec
             actionId: z.ZodString;
         }, z.core.$strict>], "kind">;
         environmentHash: z.ZodString;
+        externalDatasetBinding: z.ZodOptional<z.ZodObject<{
+            schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+            dataset: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+                revision: z.ZodNumber;
+            }, z.core.$strict>;
+            packageHash: z.ZodString;
+            profileId: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+            protectedProfileClosureHash: z.ZodString;
+            recordedOrigin: z.ZodOptional<z.ZodObject<{
+                execution: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                originalEvidence: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                members: z.ZodArray<z.ZodObject<{
+                    caseId: z.ZodString;
+                    seed: z.ZodString;
+                    fixtureId: z.ZodString;
+                    sourceId: z.ZodString;
+                    snapshotHash: z.ZodString;
+                    boundaryRevisionHash: z.ZodString;
+                }, z.core.$strict>>;
+            }, z.core.$strict>>;
+            gradingSource: z.ZodOptional<z.ZodObject<{
+                pass: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                evidence: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+            }, z.core.$strict>>;
+            split: z.ZodEnum<{
+                train: "train";
+                validation: "validation";
+                test: "test";
+                frozen_eval: "frozen_eval";
+            }>;
+            population: z.ZodArray<z.ZodObject<{
+                taskId: z.ZodString;
+                seed: z.ZodString;
+                fixtureId: z.ZodNullable<z.ZodString>;
+            }, z.core.$strict>>;
+            evaluators: z.ZodArray<z.ZodObject<{
+                release: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                    revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+                }, z.core.$strict>;
+                feedbackKey: z.ZodString;
+                configurationHash: z.ZodString;
+                output: z.ZodEnum<{
+                    boolean: "boolean";
+                    score: "score";
+                    category: "category";
+                }>;
+                categories: z.ZodArray<z.ZodString>;
+            }, z.core.$strict>>;
+            fieldMappingsHash: z.ZodString;
+            fieldMappings: z.ZodArray<z.ZodObject<{
+                graderId: z.ZodString;
+                fields: z.ZodArray<z.ZodObject<{
+                    destination: z.ZodEnum<{
+                        output: "output";
+                        input: "input";
+                        expectedOutput: "expectedOutput";
+                        evaluatorContext: "evaluatorContext";
+                    }>;
+                    path: z.ZodArray<z.ZodString>;
+                    source: z.ZodEnum<{
+                        output: "output";
+                        input: "input";
+                        expectedOutput: "expectedOutput";
+                        evaluatorContext: "evaluatorContext";
+                    }>;
+                    sourcePath: z.ZodArray<z.ZodString>;
+                }, z.core.$strict>>;
+            }, z.core.$strict>>;
+            environmentHash: z.ZodString;
+            privateDatasetClosureHash: z.ZodString;
+            criterion: z.ZodObject<{
+                minimumPassRate: z.ZodNumber;
+                requireComplete: z.ZodBoolean;
+            }, z.core.$strict>;
+            contentHash: z.ZodString;
+        }, z.core.$strict>>;
+        sourceCandidate: z.ZodOptional<z.ZodObject<{
+            candidateId: z.ZodString;
+            frozenRevision: z.ZodNumber;
+            frozenHash: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            protectedClosureHash: z.ZodString;
+        }, z.core.$strict>>;
     }, z.core.$strict>;
     model: z.ZodObject<{
         modelId: z.ZodString;
@@ -78,6 +193,10 @@ export declare const ExperimentCaseIdentitySchema: z.ZodObject<{
 }, z.core.$strict>;
 export type ExperimentCaseIdentity = z.infer<typeof ExperimentCaseIdentitySchema>;
 export declare function experimentCaseKey(identity: ExperimentCaseIdentity): string;
+export declare function humanProjectionViewId(originalManifest: {
+    id: string;
+    contentHash: string;
+}, selectionHash: string, rulesHash: string): string;
 export declare const ExperimentManifestContentSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.experimentManifest.v1">;
     id: z.ZodString;
@@ -119,6 +238,121 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
                 actionId: z.ZodString;
             }, z.core.$strict>], "kind">;
             environmentHash: z.ZodString;
+            externalDatasetBinding: z.ZodOptional<z.ZodObject<{
+                schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+                dataset: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                    revision: z.ZodNumber;
+                }, z.core.$strict>;
+                packageHash: z.ZodString;
+                profileId: z.ZodString;
+                target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"profile">;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"workflow">;
+                    workflowId: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"skill">;
+                    skillPath: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"agent_action">;
+                    actionId: z.ZodString;
+                }, z.core.$strict>], "kind">;
+                declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+                protectedProfileClosureHash: z.ZodString;
+                recordedOrigin: z.ZodOptional<z.ZodObject<{
+                    execution: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    originalEvidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    members: z.ZodArray<z.ZodObject<{
+                        caseId: z.ZodString;
+                        seed: z.ZodString;
+                        fixtureId: z.ZodString;
+                        sourceId: z.ZodString;
+                        snapshotHash: z.ZodString;
+                        boundaryRevisionHash: z.ZodString;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                gradingSource: z.ZodOptional<z.ZodObject<{
+                    pass: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    evidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                }, z.core.$strict>>;
+                split: z.ZodEnum<{
+                    train: "train";
+                    validation: "validation";
+                    test: "test";
+                    frozen_eval: "frozen_eval";
+                }>;
+                population: z.ZodArray<z.ZodObject<{
+                    taskId: z.ZodString;
+                    seed: z.ZodString;
+                    fixtureId: z.ZodNullable<z.ZodString>;
+                }, z.core.$strict>>;
+                evaluators: z.ZodArray<z.ZodObject<{
+                    release: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                        revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+                    }, z.core.$strict>;
+                    feedbackKey: z.ZodString;
+                    configurationHash: z.ZodString;
+                    output: z.ZodEnum<{
+                        boolean: "boolean";
+                        score: "score";
+                        category: "category";
+                    }>;
+                    categories: z.ZodArray<z.ZodString>;
+                }, z.core.$strict>>;
+                fieldMappingsHash: z.ZodString;
+                fieldMappings: z.ZodArray<z.ZodObject<{
+                    graderId: z.ZodString;
+                    fields: z.ZodArray<z.ZodObject<{
+                        destination: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        path: z.ZodArray<z.ZodString>;
+                        source: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        sourcePath: z.ZodArray<z.ZodString>;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                environmentHash: z.ZodString;
+                privateDatasetClosureHash: z.ZodString;
+                criterion: z.ZodObject<{
+                    minimumPassRate: z.ZodNumber;
+                    requireComplete: z.ZodBoolean;
+                }, z.core.$strict>;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+            sourceCandidate: z.ZodOptional<z.ZodObject<{
+                candidateId: z.ZodString;
+                frozenRevision: z.ZodNumber;
+                frozenHash: z.ZodString;
+                harnessRelease: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                protectedClosureHash: z.ZodString;
+            }, z.core.$strict>>;
         }, z.core.$strict>;
         model: z.ZodObject<{
             modelId: z.ZodString;
@@ -170,6 +404,14 @@ export declare const ExperimentManifestContentSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
         scoringPassId: z.ZodNullable<z.ZodString>;
+        humanProjection: z.ZodOptional<z.ZodObject<{
+            originalManifest: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            selectionHash: z.ZodString;
+            rulesHash: z.ZodString;
+        }, z.core.$strict>>;
     }, z.core.$strict>>;
     evaluators: z.ZodArray<z.ZodObject<{
         release: z.ZodObject<{
@@ -234,6 +476,121 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
                 actionId: z.ZodString;
             }, z.core.$strict>], "kind">;
             environmentHash: z.ZodString;
+            externalDatasetBinding: z.ZodOptional<z.ZodObject<{
+                schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+                dataset: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                    revision: z.ZodNumber;
+                }, z.core.$strict>;
+                packageHash: z.ZodString;
+                profileId: z.ZodString;
+                target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"profile">;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"workflow">;
+                    workflowId: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"skill">;
+                    skillPath: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"agent_action">;
+                    actionId: z.ZodString;
+                }, z.core.$strict>], "kind">;
+                declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+                protectedProfileClosureHash: z.ZodString;
+                recordedOrigin: z.ZodOptional<z.ZodObject<{
+                    execution: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    originalEvidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    members: z.ZodArray<z.ZodObject<{
+                        caseId: z.ZodString;
+                        seed: z.ZodString;
+                        fixtureId: z.ZodString;
+                        sourceId: z.ZodString;
+                        snapshotHash: z.ZodString;
+                        boundaryRevisionHash: z.ZodString;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                gradingSource: z.ZodOptional<z.ZodObject<{
+                    pass: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    evidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                }, z.core.$strict>>;
+                split: z.ZodEnum<{
+                    train: "train";
+                    validation: "validation";
+                    test: "test";
+                    frozen_eval: "frozen_eval";
+                }>;
+                population: z.ZodArray<z.ZodObject<{
+                    taskId: z.ZodString;
+                    seed: z.ZodString;
+                    fixtureId: z.ZodNullable<z.ZodString>;
+                }, z.core.$strict>>;
+                evaluators: z.ZodArray<z.ZodObject<{
+                    release: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                        revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+                    }, z.core.$strict>;
+                    feedbackKey: z.ZodString;
+                    configurationHash: z.ZodString;
+                    output: z.ZodEnum<{
+                        boolean: "boolean";
+                        score: "score";
+                        category: "category";
+                    }>;
+                    categories: z.ZodArray<z.ZodString>;
+                }, z.core.$strict>>;
+                fieldMappingsHash: z.ZodString;
+                fieldMappings: z.ZodArray<z.ZodObject<{
+                    graderId: z.ZodString;
+                    fields: z.ZodArray<z.ZodObject<{
+                        destination: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        path: z.ZodArray<z.ZodString>;
+                        source: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        sourcePath: z.ZodArray<z.ZodString>;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                environmentHash: z.ZodString;
+                privateDatasetClosureHash: z.ZodString;
+                criterion: z.ZodObject<{
+                    minimumPassRate: z.ZodNumber;
+                    requireComplete: z.ZodBoolean;
+                }, z.core.$strict>;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+            sourceCandidate: z.ZodOptional<z.ZodObject<{
+                candidateId: z.ZodString;
+                frozenRevision: z.ZodNumber;
+                frozenHash: z.ZodString;
+                harnessRelease: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                protectedClosureHash: z.ZodString;
+            }, z.core.$strict>>;
         }, z.core.$strict>;
         model: z.ZodObject<{
             modelId: z.ZodString;
@@ -285,6 +642,14 @@ export declare const ExperimentManifestSchema: z.ZodObject<{
             contentHash: z.ZodString;
         }, z.core.$strict>;
         scoringPassId: z.ZodNullable<z.ZodString>;
+        humanProjection: z.ZodOptional<z.ZodObject<{
+            originalManifest: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            selectionHash: z.ZodString;
+            rulesHash: z.ZodString;
+        }, z.core.$strict>>;
     }, z.core.$strict>>;
     evaluators: z.ZodArray<z.ZodObject<{
         release: z.ZodObject<{
@@ -347,9 +712,9 @@ export declare const ExperimentCaseResultSchema: z.ZodObject<{
         fixtureId: z.ZodNullable<z.ZodString>;
     }, z.core.$strict>;
     status: z.ZodEnum<{
+        cancelled: "cancelled";
         failed: "failed";
         completed: "completed";
-        cancelled: "cancelled";
         unavailable: "unavailable";
     }>;
     output: z.ZodNullable<z.ZodUnknown>;
@@ -403,9 +768,9 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     status: z.ZodEnum<{
+        cancelled: "cancelled";
         failed: "failed";
         completed: "completed";
-        cancelled: "cancelled";
     }>;
     cases: z.ZodArray<z.ZodObject<{
         identity: z.ZodObject<{
@@ -414,9 +779,9 @@ export declare const ExperimentResultContentSchema: z.ZodObject<{
             fixtureId: z.ZodNullable<z.ZodString>;
         }, z.core.$strict>;
         status: z.ZodEnum<{
+            cancelled: "cancelled";
             failed: "failed";
             completed: "completed";
-            cancelled: "cancelled";
             unavailable: "unavailable";
         }>;
         output: z.ZodNullable<z.ZodUnknown>;
@@ -472,9 +837,9 @@ export declare const ExperimentResultSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>;
     status: z.ZodEnum<{
+        cancelled: "cancelled";
         failed: "failed";
         completed: "completed";
-        cancelled: "cancelled";
     }>;
     cases: z.ZodArray<z.ZodObject<{
         identity: z.ZodObject<{
@@ -483,9 +848,9 @@ export declare const ExperimentResultSchema: z.ZodObject<{
             fixtureId: z.ZodNullable<z.ZodString>;
         }, z.core.$strict>;
         status: z.ZodEnum<{
+            cancelled: "cancelled";
             failed: "failed";
             completed: "completed";
-            cancelled: "cancelled";
             unavailable: "unavailable";
         }>;
         output: z.ZodNullable<z.ZodUnknown>;
@@ -616,6 +981,102 @@ export declare function verifyExperimentEvidence(input: {
                     actionId: string;
                 };
                 environmentHash: string;
+                externalDatasetBinding?: {
+                    schemaVersion: "openpond.profileExternalDatasetBinding.v1";
+                    dataset: {
+                        id: string;
+                        contentHash: string;
+                        revision: number;
+                    };
+                    packageHash: string;
+                    profileId: string;
+                    target: {
+                        kind: "profile";
+                    } | {
+                        kind: "workflow";
+                        workflowId: string;
+                    } | {
+                        kind: "skill";
+                        skillPath: string;
+                    } | {
+                        kind: "agent_action";
+                        actionId: string;
+                    };
+                    declaredProfileCatalogHash: string | null;
+                    protectedProfileClosureHash: string;
+                    split: "train" | "validation" | "test" | "frozen_eval";
+                    population: {
+                        taskId: string;
+                        seed: string;
+                        fixtureId: string | null;
+                    }[];
+                    evaluators: {
+                        release: {
+                            id: string;
+                            contentHash: string;
+                            revision: string | number;
+                        };
+                        feedbackKey: string;
+                        configurationHash: string;
+                        output: "boolean" | "score" | "category";
+                        categories: string[];
+                    }[];
+                    fieldMappingsHash: string;
+                    fieldMappings: {
+                        graderId: string;
+                        fields: {
+                            destination: "output" | "input" | "expectedOutput" | "evaluatorContext";
+                            path: string[];
+                            source: "output" | "input" | "expectedOutput" | "evaluatorContext";
+                            sourcePath: string[];
+                        }[];
+                    }[];
+                    environmentHash: string;
+                    privateDatasetClosureHash: string;
+                    criterion: {
+                        minimumPassRate: number;
+                        requireComplete: boolean;
+                    };
+                    contentHash: string;
+                    recordedOrigin?: {
+                        execution: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        originalEvidence: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        members: {
+                            caseId: string;
+                            seed: string;
+                            fixtureId: string;
+                            sourceId: string;
+                            snapshotHash: string;
+                            boundaryRevisionHash: string;
+                        }[];
+                    } | undefined;
+                    gradingSource?: {
+                        pass: {
+                            id: string;
+                            contentHash: string;
+                        };
+                        evidence: {
+                            id: string;
+                            contentHash: string;
+                        };
+                    } | undefined;
+                } | undefined;
+                sourceCandidate?: {
+                    candidateId: string;
+                    frozenRevision: number;
+                    frozenHash: string;
+                    harnessRelease: {
+                        id: string;
+                        contentHash: string;
+                    };
+                    protectedClosureHash: string;
+                } | undefined;
             };
             model: {
                 modelId: string;
@@ -680,6 +1141,14 @@ export declare function verifyExperimentEvidence(input: {
                 contentHash: string;
             };
             scoringPassId: string | null;
+            humanProjection?: {
+                originalManifest: {
+                    id: string;
+                    contentHash: string;
+                };
+                selectionHash: string;
+                rulesHash: string;
+            } | undefined;
         } | undefined;
     };
     result: {
@@ -688,14 +1157,14 @@ export declare function verifyExperimentEvidence(input: {
             id: string;
             contentHash: string;
         };
-        status: "failed" | "completed" | "cancelled";
+        status: "cancelled" | "failed" | "completed";
         cases: {
             identity: {
                 caseId: string;
                 seed: string;
                 fixtureId: string | null;
             };
-            status: "failed" | "completed" | "cancelled" | "unavailable";
+            status: "cancelled" | "failed" | "completed" | "unavailable";
             output: unknown;
             error: {
                 code: string;

package/dist/types/experiments.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAsC1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA+EtB"}
+{"version":3,"file":"experiments.d.ts","sourceRoot":"","sources":["../../../../src/experiments.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAaxB,OAAO,EACL,uCAAuC,EACvC,sCAAsC,EACtC,gCAAgC,EAChC,gCAAgC,EAChC,sCAAsC,EACtC,qCAAqC,EACrC,KAAK,iCAAiC,GACvC,MAAM,gCAAgC,CAAC;AACxC,OAAO,EACL,qCAAqC,EACrC,2CAA2C,EAC3C,oCAAoC,EACpC,oCAAoC,EACpC,oCAAoC,EACpC,KAAK,+BAA+B,EACpC,KAAK,8BAA8B,GACpC,MAAM,wCAAwC,CAAC;AAIhD,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;4BAiBjC,CAAC;AAEH,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;kBAepC,CAAC;AAEH,eAAO,MAAM,4BAA4B;;;;kBAI9B,CAAC;AACZ,MAAM,MAAM,sBAAsB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,4BAA4B,CAAC,CAAC;AAClF,wBAAgB,iBAAiB,CAAC,QAAQ,EAAE,sBAAsB,GAAG,MAAM,CAG1E;AAED,wBAAgB,qBAAqB,CAAC,gBAAgB,EAAE;IAAC,EAAE,EAAC,MAAM,CAAC;IAAA,WAAW,EAAC,MAAM,CAAA;CAAC,EAAE,aAAa,EAAC,MAAM,EAAE,SAAS,EAAC,MAAM,GAAG,MAAM,CAEtI;AAED,eAAO,MAAM,+BAA+B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA8C1C,CAAC;AACH,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAsF,CAAC;AAC5H,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAE1E,eAAO,MAAM,wBAAwB;;;;;;;;;;;;;;;;;;;;;;kBAQ1B,CAAC;AAEZ,eAAO,MAAM,qBAAqB;;;;;;kBAUhC,CAAC;AAEH,eAAO,MAAM,0BAA0B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAU5B,CAAC;AAEZ,eAAO,MAAM,6BAA6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAM/B,CAAC;AACZ,eAAO,MAAM,sBAAsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAAoF,CAAC;AACxH,MAAM,MAAM,gBAAgB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,sBAAsB,CAAC,CAAC;AAEtE,wBAAgB,wBAAwB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,+BAA+B,CAAC,GAAG,kBAAkB,CAGnH;AAED,wBAAgB,sBAAsB,CAAC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,6BAA6B,CAAC,EAAE,QAAQ,EAAE,kBAAkB,GAAG,gBAAgB,CAgC3I;AAED,MAAM,MAAM,oBAAoB,GAAG;IACjC,UAAU,EAAE,OAAO,CAAC;IACpB,OAAO,EAAE,MAAM,EAAE,CAAC;IAClB,KAAK,EAAE,KAAK,CAAC;QAAE,QAAQ,EAAE,sBAAsB,CAAC;QAAC,QAAQ,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAAC,SAAS,EAAE,oBAAoB,GAAG,IAAI,CAAC;QAC7H,QAAQ,EAAE,4BAA4B,EAAE,CAAA;KAAE,CAAC,CAAC;IAC9C,OAAO,EAAE,0BAA0B,EAAE,CAAC;CACvC,CAAC;AACF,MAAM,MAAM,oBAAoB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,0BAA0B,CAAC,CAAC;AAC9E,MAAM,MAAM,4BAA4B,GAAG;IACzC,WAAW,EAAE,MAAM,CAAC;IACpB,QAAQ,EAAE,OAAO,CAAC;IAClB,MAAM,EAAE,0BAA0B,GAAG,cAAc,GAAG,oBAAoB,GAAG,sBAAsB,GAAG,sBAAsB,GAAG,IAAI,CAAC;IACpI,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,cAAc,EAAE,OAAO,GAAG,IAAI,CAAC;IAC/B,eAAe,EAAE,OAAO,GAAG,IAAI,CAAC;CACjC,CAAC;AACF,MAAM,MAAM,0BAA0B,GAAG;IACvC,WAAW,EAAE,MAAM,CAAC;IACpB,aAAa,EAAE,MAAM,CAAC;IACtB,aAAa,EAAE,MAAM,CAAC;IACtB,QAAQ,EAAE,MAAM,GAAG,IAAI,CAAC;IACxB,SAAS,EAAE,MAAM,GAAG,IAAI,CAAC;IACzB,KAAK,EAAE,MAAM,GAAG,IAAI,CAAC;IACrB,gBAAgB,EAAE,MAAM,GAAG,IAAI,CAAC;IAChC,iBAAiB,EAAE,MAAM,GAAG,IAAI,CAAC;IACjC,aAAa,EAAE,MAAM,GAAG,IAAI,CAAC;CAC9B,CAAC;AAEF,wBAAgB,wBAAwB,CAAC,KAAK,EAAE;IAAE,QAAQ,EAAE,OAAO,CAAC;IAAC,MAAM,EAAE,OAAO,CAAA;CAAE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EAOrF;AAED,iFAAiF;AACjF,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,EACpE,SAAS,EAAE;IAAE,QAAQ,EAAE,kBAAkB,CAAC;IAAC,MAAM,EAAE,gBAAgB,CAAA;CAAE,GACpE,oBAAoB,CA+EtB"}

package/dist/types/graders.d.ts

@@ -17,6 +17,12 @@ export declare const GraderEvidenceContentSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
     graderId: z.ZodString;
     graderVersion: z.ZodString;
+    status: z.ZodOptional<z.ZodEnum<{
+        pending: "pending";
+        failed: "failed";
+        scored: "scored";
+        unavailable: "unavailable";
+    }>>;
     score: z.ZodNullable<z.ZodNumber>;
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
@@ -48,6 +54,12 @@ export declare const GraderEvidenceSchema: z.ZodObject<{
     schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
     graderId: z.ZodString;
     graderVersion: z.ZodString;
+    status: z.ZodOptional<z.ZodEnum<{
+        pending: "pending";
+        failed: "failed";
+        scored: "scored";
+        unavailable: "unavailable";
+    }>>;
     score: z.ZodNullable<z.ZodNumber>;
     passed: z.ZodBoolean;
     rewardEligible: z.ZodBoolean;
@@ -120,6 +132,7 @@ declare const GradeSummarySchema: z.ZodObject<{
         cancelled: "cancelled";
     }>>;
     gradingStatus: z.ZodEnum<{
+        pending: "pending";
         scored: "scored";
         unscorable: "unscorable";
         not_configured: "not_configured";
@@ -129,7 +142,7 @@ declare const GradeSummarySchema: z.ZodObject<{
  * missing checks cannot disappear from the denominator or the hard gates. */
 export declare function aggregateGraderScores(input: {
     graders: ReadonlyArray<Pick<GraderSpec, "id" | "version" | "weight" | "hardGate" | "rewardEligible">>;
-    components: ReadonlyArray<Pick<GraderEvidence, "graderId" | "graderVersion" | "score" | "passed" | "rewardEligible" | "failureClass">>;
+    components: ReadonlyArray<Pick<GraderEvidence, "graderId" | "graderVersion" | "score" | "passed" | "rewardEligible" | "failureClass" | "status">>;
     infrastructureError?: string | null;
 }): z.infer<typeof GradeSummarySchema>;
 export declare const TaskGradeSchema: z.ZodObject<{
@@ -145,6 +158,7 @@ export declare const TaskGradeSchema: z.ZodObject<{
         cancelled: "cancelled";
     }>>;
     gradingStatus: z.ZodEnum<{
+        pending: "pending";
         scored: "scored";
         unscorable: "unscorable";
         not_configured: "not_configured";
@@ -157,6 +171,12 @@ export declare const TaskGradeSchema: z.ZodObject<{
         schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
         graderId: z.ZodString;
         graderVersion: z.ZodString;
+        status: z.ZodOptional<z.ZodEnum<{
+            pending: "pending";
+            failed: "failed";
+            scored: "scored";
+            unavailable: "unavailable";
+        }>>;
         score: z.ZodNullable<z.ZodNumber>;
         passed: z.ZodBoolean;
         rewardEligible: z.ZodBoolean;