@openpond/evals 0.11.16 → 0.11.18

package/dist/types/learning/reward-calibration.d.ts

@@ -56,6 +56,80 @@ export declare function qualifyRewardCheck(draft: AuthoringDraftFor<"reward">, b
                 visibility: "policy" | "verifier" | "host_private";
             };
             reviewerRole: string;
+            form?: {
+                schemaVersion: "openpond.humanForm.v1";
+                mode: "individual" | "pairwise";
+                instructions: string;
+                criteria: ({
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "score";
+                    minimum: number;
+                    maximum: number;
+                    step: number;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "category";
+                    options: string[];
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "boolean";
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "text";
+                    maximumLength: number;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                    minimumLength?: number | undefined;
+                } | {
+                    id: string;
+                    label: string;
+                    instructions: string;
+                    required: boolean;
+                    allowAbstain: boolean;
+                    kind: "preference";
+                    allowTie: boolean;
+                    requireRationale?: boolean | undefined;
+                    anchors?: {
+                        value: string | number | boolean;
+                        description: string;
+                    }[] | undefined;
+                    requireStrength?: boolean | undefined;
+                })[];
+            } | undefined;
         } | {
             kind: "learned_model";
             modelVersion: {

package/dist/types/learning/reward-calibration.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reward-calibration.d.ts","sourceRoot":"","sources":["../../../../../src/learning/reward-calibration.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EAAmD,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAA0C,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEjG;4EAC4E;AAC5E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,aAAa,GAAG,IAAI,EAAE,KAAK,EAAE,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4BvH;AAED;yEACyE;AACzE,wBAAsB,uBAAuB,CAAC,EAAE,EAAE,mBAAmB,EAAE,MAAM,EAAE,aAAa,iBAe3F"}
+{"version":3,"file":"reward-calibration.d.ts","sourceRoot":"","sources":["../../../../../src/learning/reward-calibration.ts"],"names":[],"mappings":"AACA,OAAO,EAAuB,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AACxE,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,gBAAgB,CAAC;AAGxD,OAAO,EAAmD,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAC5G,OAAO,EAA0C,KAAK,cAAc,EAAE,MAAM,oBAAoB,CAAC;AAEjG;4EAC4E;AAC5E,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,aAAa,GAAG,IAAI,EAAE,KAAK,EAAE,cAAc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;EA4BvH;AAED;yEACyE;AACzE,wBAAsB,uBAAuB,CAAC,EAAE,EAAE,mBAAmB,EAAE,MAAM,EAAE,aAAa,iBAe3F"}

package/dist/types/learning/reward-checks.d.ts

@@ -62,6 +62,12 @@ export declare const RewardFixtureCheckResultSchema: z.ZodObject<{
             schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
             graderId: z.ZodString;
             graderVersion: z.ZodString;
+            status: z.ZodOptional<z.ZodEnum<{
+                pending: "pending";
+                failed: "failed";
+                scored: "scored";
+                unavailable: "unavailable";
+            }>>;
             score: z.ZodNullable<z.ZodNumber>;
             passed: z.ZodBoolean;
             rewardEligible: z.ZodBoolean;
@@ -115,12 +121,12 @@ export declare const RewardCheckRunSchema: z.ZodObject<{
         contentHash: z.ZodString;
     }, z.core.$strict>>;
     status: z.ZodEnum<{
+        queued: "queued";
+        cancelled: "cancelled";
         failed: "failed";
         completed: "completed";
-        cancelled: "cancelled";
         running: "running";
         cancelling: "cancelling";
-        queued: "queued";
     }>;
     runtime: z.ZodNullable<z.ZodObject<{
         id: z.ZodString;
@@ -157,6 +163,12 @@ export declare const RewardCheckRunSchema: z.ZodObject<{
                 schemaVersion: z.ZodLiteral<"openpond.graderEvidence.v1">;
                 graderId: z.ZodString;
                 graderVersion: z.ZodString;
+                status: z.ZodOptional<z.ZodEnum<{
+                    pending: "pending";
+                    failed: "failed";
+                    scored: "scored";
+                    unavailable: "unavailable";
+                }>>;
                 score: z.ZodNullable<z.ZodNumber>;
                 passed: z.ZodBoolean;
                 rewardEligible: z.ZodBoolean;

package/dist/types/learning/reward-checks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"reward-checks.d.ts","sourceRoot":"","sources":["../../../../../src/learning/reward-checks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAsE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAC/I,OAAO,KAAK,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAI5E,OAAO,KAAK,EAAE,iBAAiB,EAAE,4BAA4B,EAAE,MAAM,gBAAgB,CAAC;AAItF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;kBAY9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,eAAO,MAAM,wBAAwB;;;;kBAA6H,CAAC;AACnK,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGhC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYtB,CAAC;AACZ,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,4BAA4B,EAAE,GAAG,aAAa,EAAE,CAW7F;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,aAAa,GAAG,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKhG;AAED,sFAAsF;AACtF,wBAAsB,oBAAoB,CAAC,KAAK,EAAE;IAChD,MAAM,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IACpE,cAAc,CAAC,EAAE,oBAAoB,CAAC;IAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACtE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgB7B;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,iBAAiB,GAAG,wBAAwB,CAK9G"}
+{"version":3,"file":"reward-checks.d.ts","sourceRoot":"","sources":["../../../../../src/learning/reward-checks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAExB,OAAO,EAAsE,KAAK,iBAAiB,EAAE,KAAK,aAAa,EAAE,MAAM,eAAe,CAAC;AAC/I,OAAO,KAAK,EAAE,oBAAoB,EAAE,gBAAgB,EAAE,MAAM,eAAe,CAAC;AAI5E,OAAO,KAAK,EAAE,iBAAiB,EAAE,4BAA4B,EAAE,MAAM,gBAAgB,CAAC;AAItF,eAAO,MAAM,mBAAmB;;;;;;;;;;;;;;;;;;;;;;kBAY9B,CAAC;AACH,MAAM,MAAM,aAAa,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,mBAAmB,CAAC,CAAC;AAChE,eAAO,MAAM,wBAAwB;;;;kBAA6H,CAAC;AACnK,MAAM,MAAM,kBAAkB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,wBAAwB,CAAC,CAAC;AAC1E,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAGhC,CAAC;AACZ,MAAM,MAAM,wBAAwB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,8BAA8B,CAAC,CAAC;AACtF,eAAO,MAAM,oBAAoB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAYtB,CAAC;AACZ,MAAM,MAAM,cAAc,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,oBAAoB,CAAC,CAAC;AAElE,wBAAgB,qBAAqB,CAAC,MAAM,EAAE,4BAA4B,EAAE,GAAG,aAAa,EAAE,CAW7F;AAED,wBAAgB,kBAAkB,CAAC,KAAK,EAAE,iBAAiB,CAAC,QAAQ,CAAC,EAAE,IAAI,EAAE,aAAa,GAAG,IAAI;;;;;;;;;;;;;;;;;;;;;;;;;;;EAKhG;AAED,sFAAsF;AACtF,wBAAsB,oBAAoB,CAAC,KAAK,EAAE;IAChD,MAAM,EAAE,aAAa,CAAC;IAAC,OAAO,EAAE,aAAa,CAAC;IAAC,MAAM,CAAC,EAAE,WAAW,CAAC;IACpE,cAAc,CAAC,EAAE,oBAAoB,CAAC;IAAC,UAAU,CAAC,EAAE,gBAAgB,CAAC;CACtE,GAAG,OAAO,CAAC,iBAAiB,CAAC,CAgB7B;AAED,wBAAgB,kBAAkB,CAAC,OAAO,EAAE,aAAa,EAAE,MAAM,EAAE,iBAAiB,GAAG,wBAAwB,CAK9G"}

package/dist/types/learning/training-handoff.d.ts

@@ -0,0 +1,201 @@
+import { z } from "zod";
+export declare const TrainingHandoffOriginSchema: z.ZodObject<{
+    schemaVersion: z.ZodLiteral<"openpond.trainingHandoffOrigin.v1">;
+    teamId: z.ZodString;
+    projectId: z.ZodNullable<z.ZodString>;
+    configurationId: z.ZodString;
+    source: z.ZodObject<{
+        executionId: z.ZodString;
+        passId: z.ZodNullable<z.ZodString>;
+        manifestHash: z.ZodString;
+        resultHash: z.ZodString;
+    }, z.core.$strict>;
+    dataset: z.ZodObject<{
+        id: z.ZodString;
+        revision: z.ZodNumber;
+        contentHash: z.ZodString;
+    }, z.core.$strict>;
+    target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+        kind: z.ZodLiteral<"model">;
+        modelId: z.ZodString;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"harness">;
+        source: z.ZodObject<{
+            profileId: z.ZodString;
+            sourceRevision: z.ZodString;
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            catalogHash: z.ZodString;
+            definitionId: z.ZodString;
+            definitionHash: z.ZodString;
+            target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                kind: z.ZodLiteral<"profile">;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"workflow">;
+                workflowId: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"skill">;
+                skillPath: z.ZodString;
+            }, z.core.$strict>, z.ZodObject<{
+                kind: z.ZodLiteral<"agent_action">;
+                actionId: z.ZodString;
+            }, z.core.$strict>], "kind">;
+            environmentHash: z.ZodString;
+            externalDatasetBinding: z.ZodOptional<z.ZodObject<{
+                schemaVersion: z.ZodLiteral<"openpond.profileExternalDatasetBinding.v1">;
+                dataset: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                    revision: z.ZodNumber;
+                }, z.core.$strict>;
+                packageHash: z.ZodString;
+                profileId: z.ZodString;
+                target: z.ZodDiscriminatedUnion<[z.ZodObject<{
+                    kind: z.ZodLiteral<"profile">;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"workflow">;
+                    workflowId: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"skill">;
+                    skillPath: z.ZodString;
+                }, z.core.$strict>, z.ZodObject<{
+                    kind: z.ZodLiteral<"agent_action">;
+                    actionId: z.ZodString;
+                }, z.core.$strict>], "kind">;
+                declaredProfileCatalogHash: z.ZodNullable<z.ZodString>;
+                protectedProfileClosureHash: z.ZodString;
+                recordedOrigin: z.ZodOptional<z.ZodObject<{
+                    execution: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    originalEvidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    members: z.ZodArray<z.ZodObject<{
+                        caseId: z.ZodString;
+                        seed: z.ZodString;
+                        fixtureId: z.ZodString;
+                        sourceId: z.ZodString;
+                        snapshotHash: z.ZodString;
+                        boundaryRevisionHash: z.ZodString;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                gradingSource: z.ZodOptional<z.ZodObject<{
+                    pass: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                    evidence: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                    }, z.core.$strict>;
+                }, z.core.$strict>>;
+                split: z.ZodEnum<{
+                    train: "train";
+                    validation: "validation";
+                    test: "test";
+                    frozen_eval: "frozen_eval";
+                }>;
+                population: z.ZodArray<z.ZodObject<{
+                    taskId: z.ZodString;
+                    seed: z.ZodString;
+                    fixtureId: z.ZodNullable<z.ZodString>;
+                }, z.core.$strict>>;
+                evaluators: z.ZodArray<z.ZodObject<{
+                    release: z.ZodObject<{
+                        id: z.ZodString;
+                        contentHash: z.ZodString;
+                        revision: z.ZodUnion<readonly [z.ZodNumber, z.ZodString]>;
+                    }, z.core.$strict>;
+                    feedbackKey: z.ZodString;
+                    configurationHash: z.ZodString;
+                    output: z.ZodEnum<{
+                        boolean: "boolean";
+                        score: "score";
+                        category: "category";
+                    }>;
+                    categories: z.ZodArray<z.ZodString>;
+                }, z.core.$strict>>;
+                fieldMappingsHash: z.ZodString;
+                fieldMappings: z.ZodArray<z.ZodObject<{
+                    graderId: z.ZodString;
+                    fields: z.ZodArray<z.ZodObject<{
+                        destination: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        path: z.ZodArray<z.ZodString>;
+                        source: z.ZodEnum<{
+                            output: "output";
+                            input: "input";
+                            expectedOutput: "expectedOutput";
+                            evaluatorContext: "evaluatorContext";
+                        }>;
+                        sourcePath: z.ZodArray<z.ZodString>;
+                    }, z.core.$strict>>;
+                }, z.core.$strict>>;
+                environmentHash: z.ZodString;
+                privateDatasetClosureHash: z.ZodString;
+                criterion: z.ZodObject<{
+                    minimumPassRate: z.ZodNumber;
+                    requireComplete: z.ZodBoolean;
+                }, z.core.$strict>;
+                contentHash: z.ZodString;
+            }, z.core.$strict>>;
+            sourceCandidate: z.ZodOptional<z.ZodObject<{
+                candidateId: z.ZodString;
+                frozenRevision: z.ZodNumber;
+                frozenHash: z.ZodString;
+                harnessRelease: z.ZodObject<{
+                    id: z.ZodString;
+                    contentHash: z.ZodString;
+                }, z.core.$strict>;
+                protectedClosureHash: z.ZodString;
+            }, z.core.$strict>>;
+        }, z.core.$strict>;
+        model: z.ZodObject<{
+            modelId: z.ZodString;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"agent">;
+        source: z.ZodObject<{
+            harnessRelease: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            agentSnapshot: z.ZodObject<{
+                id: z.ZodString;
+                contentHash: z.ZodString;
+            }, z.core.$strict>;
+            sourcePackageHash: z.ZodString;
+        }, z.core.$strict>;
+        model: z.ZodObject<{
+            modelId: z.ZodString;
+            configurationHash: z.ZodString;
+        }, z.core.$strict>;
+    }, z.core.$strict>, z.ZodObject<{
+        kind: z.ZodLiteral<"fixture">;
+        configurationHash: z.ZodString;
+    }, z.core.$strict>], "kind">;
+    graders: z.ZodArray<z.ZodObject<{
+        feedbackKey: z.ZodString;
+        evaluator: z.ZodObject<{
+            id: z.ZodString;
+            contentHash: z.ZodString;
+        }, z.core.$strict>;
+        expectedCount: z.ZodNumber;
+        scoredCount: z.ZodNumber;
+        score: z.ZodNullable<z.ZodNumber>;
+    }, z.core.$strict>>;
+    contentHash: z.ZodString;
+}, z.core.$strict>;
+export type TrainingHandoffOrigin = z.infer<typeof TrainingHandoffOriginSchema>;
+//# sourceMappingURL=training-handoff.d.ts.map

package/dist/types/learning/training-handoff.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"training-handoff.d.ts","sourceRoot":"","sources":["../../../../../src/learning/training-handoff.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAKxB,eAAO,MAAM,2BAA2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBA4B7B,CAAC;AACZ,MAAM,MAAM,qBAAqB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,2BAA2B,CAAC,CAAC"}