@openpond/evals 0.11.21 → 0.12.1

package/dist/types/learning/batch-service.d.ts

@@ -22,7 +22,7 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             contentHash: string;
             revision: number;
         };
-        purpose: "evaluation" | "supervised_training" | "reward_training";
+        purpose: "evaluation" | "reward_training" | "supervised_training";
         examples: {
             evidence: {
                 id: string;
@@ -36,7 +36,7 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             };
             familyKey: string;
             inputHash: string;
-            split: "train" | "validation" | "test" | "frozen_eval";
+            split: "frozen_eval" | "test" | "train" | "validation";
         }[];
         sealedAt: string;
         sealedBy: string;
@@ -54,14 +54,18 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
         };
         environment: {
             protocolVersion: "openpond.environment.v1";
-            kind: "text" | "agent" | "work" | "custom_program";
+            kind: "agent" | "custom_program" | "text" | "work";
             entrypoint: string;
             stateful: boolean;
             deterministicSeeds: boolean;
-            lifecycle: ("step" | "create" | "reset" | "collect" | "destroy")[];
-            networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+            lifecycle: ("collect" | "create" | "destroy" | "reset" | "step")[];
+            networkPolicy: "declared_read_only" | "declared_scoped" | "none";
             defaultTimeoutMs: number;
         };
+        environmentRelease?: {
+            id: string;
+            contentHash: string;
+        } | undefined;
         tools: {
             name: string;
             description: string;
@@ -74,12 +78,12 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             id: string;
             required: boolean;
             scopes: string[];
-            portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
+            portability: "host_adapter" | "hosted_only" | "local_only" | "portable";
         }[];
         tasks: {
             id: string;
             clusterKey: string;
-            split: "train" | "validation" | "test" | "frozen_eval";
+            split: "frozen_eval" | "test" | "train" | "validation";
             input: Record<string, unknown>;
             expectedOutput: Record<string, unknown> | null;
             policyVisibleContext: Record<string, unknown>;
@@ -90,9 +94,8 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             }[];
-            tags: string[];
             requiredOutputs?: {
                 path: string;
                 mediaType: string;
@@ -102,23 +105,27 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                     contentHash: string;
                     sizeBytes: number;
                     mediaType: string;
-                    visibility: "policy" | "verifier" | "host_private";
+                    visibility: "host_private" | "policy" | "verifier";
                 } | null;
+                submissionMode?: "structured_json" | undefined;
                 maxBytes: number | null;
                 metadata: Record<string, unknown>;
             }[] | undefined;
+            tags: string[];
         }[];
         graders: ({
             id: string;
+            taskIds?: string[] | undefined;
             version: string;
             weight: number;
             hardGate: boolean;
             rewardEligible: boolean;
             privileged: boolean;
-            kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
+            kind: "artifact" | "content" | "runtime_event" | "schema" | "state";
             config: Record<string, unknown>;
         } | {
             id: string;
+            taskIds?: string[] | undefined;
             version: string;
             weight: number;
             hardGate: boolean;
@@ -131,9 +138,9 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             };
-            calibrationStatus: "pending" | "passed" | "failed";
+            calibrationStatus: "failed" | "passed" | "pending";
             model?: {
                 providerId: string;
                 modelId: string;
@@ -142,26 +149,28 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             temperature?: number | undefined;
         } | {
             id: string;
+            taskIds?: string[] | undefined;
             version: string;
             weight: number;
             hardGate: boolean;
             rewardEligible: boolean;
             privileged: boolean;
             kind: "custom_verifier";
+            runtime?: "isolated_javascript" | "sandbox_process" | undefined;
             verifierRef: {
                 id: string;
                 path: string;
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             };
+            exportName?: string | undefined;
             timeoutMs: number;
             networkPolicy: "none";
-            runtime?: "isolated_javascript" | "sandbox_process" | undefined;
-            exportName?: string | undefined;
         } | {
             id: string;
+            taskIds?: string[] | undefined;
             version: string;
             weight: number;
             hardGate: boolean;
@@ -174,7 +183,7 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             };
             reviewerRole: string;
             form?: {
@@ -187,77 +196,71 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                     instructions: string;
                     required: boolean;
                     allowAbstain: boolean;
-                    kind: "score";
-                    minimum: number;
-                    maximum: number;
-                    step: number;
                     requireRationale?: boolean | undefined;
                     anchors?: {
                         value: string | number | boolean;
                         description: string;
                     }[] | undefined;
+                    kind: "score";
+                    minimum: number;
+                    maximum: number;
+                    step: number;
                 } | {
                     id: string;
                     label: string;
                     instructions: string;
                     required: boolean;
                     allowAbstain: boolean;
-                    kind: "category";
-                    options: string[];
                     requireRationale?: boolean | undefined;
                     anchors?: {
                         value: string | number | boolean;
                         description: string;
                     }[] | undefined;
+                    kind: "category";
+                    options: string[];
                 } | {
                     id: string;
                     label: string;
                     instructions: string;
                     required: boolean;
                     allowAbstain: boolean;
-                    kind: "boolean";
                     requireRationale?: boolean | undefined;
                     anchors?: {
                         value: string | number | boolean;
                         description: string;
                     }[] | undefined;
+                    kind: "boolean";
                 } | {
                     id: string;
                     label: string;
                     instructions: string;
                     required: boolean;
                     allowAbstain: boolean;
-                    kind: "text";
-                    maximumLength: number;
                     requireRationale?: boolean | undefined;
                     anchors?: {
                         value: string | number | boolean;
                         description: string;
                     }[] | undefined;
+                    kind: "text";
                     minimumLength?: number | undefined;
+                    maximumLength: number;
                 } | {
                     id: string;
                     label: string;
                     instructions: string;
                     required: boolean;
                     allowAbstain: boolean;
-                    kind: "preference";
-                    allowTie: boolean;
                     requireRationale?: boolean | undefined;
                     anchors?: {
                         value: string | number | boolean;
                         description: string;
                     }[] | undefined;
+                    kind: "preference";
                     requireStrength?: boolean | undefined;
+                    allowTie: boolean;
                 })[];
             } | undefined;
         })[];
-        metadata: Record<string, unknown>;
-        contentHash: string;
-        environmentRelease?: {
-            id: string;
-            contentHash: string;
-        } | undefined;
         verifierSetRelease?: {
             id: string;
             contentHash: string;
@@ -265,8 +268,9 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
         metrics?: {
             schemaVersion: "openpond.tasksetMetricPolicy.v1";
             primaryMetric: string;
-            aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
-            missingReward: "zero" | "exclude";
+            aggregation: "custom" | "mean_score" | "pass_rate" | "weighted_mean";
+            missingReward: "exclude" | "zero";
+            taskWeights?: Record<string, number> | undefined;
             customAggregator: {
                 module: string;
                 exportName: string;
@@ -274,8 +278,9 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 timeoutMs: number;
                 networkPolicy: "none";
             } | null;
-            taskWeights?: Record<string, number> | undefined;
         } | undefined;
+        metadata: Record<string, unknown>;
+        contentHash: string;
     };
     evidence: {
         schemaVersion: "openpond.taskEvidence.v1";
@@ -299,7 +304,7 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             attemptId: string;
             occurredAt: string;
             familyKey: string | null;
-            split: "train" | "validation" | "test" | "frozen_eval";
+            split: "frozen_eval" | "test" | "train" | "validation";
             input: Record<string, import("zod").JSONType>;
             observedOutput: Record<string, import("zod").JSONType> | null;
             expected: Record<string, import("zod").JSONType> | null;
@@ -310,12 +315,8 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             }[];
-            provenance: {
-                sourceRecordRef: string | null;
-                mappingHash: string | null;
-            };
             requiredOutputs?: {
                 path: string;
                 mediaType: string;
@@ -325,11 +326,16 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                     contentHash: string;
                     sizeBytes: number;
                     mediaType: string;
-                    visibility: "policy" | "verifier" | "host_private";
+                    visibility: "host_private" | "policy" | "verifier";
                 } | null;
+                submissionMode?: "structured_json" | undefined;
                 maxBytes: number | null;
                 metadata: Record<string, unknown>;
             }[] | undefined;
+            provenance: {
+                sourceRecordRef: string | null;
+                mappingHash: string | null;
+            };
         };
         supersedes: {
             id: string;
@@ -347,10 +353,26 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
         name: string;
         description: string;
         instructions: string;
-        category: "custom" | "structured" | "question_answering" | "coding" | "tool_workflow";
+        category: "coding" | "custom" | "question_answering" | "structured" | "tool_workflow";
         familyNamespace: string;
         inputSchema: Record<string, unknown>;
         outputSchema: Record<string, unknown>;
+        requiredOutputs?: {
+            path: string;
+            mediaType: string;
+            schemaRef: {
+                id: string;
+                path: string;
+                contentHash: string;
+                sizeBytes: number;
+                mediaType: string;
+                visibility: "host_private" | "policy" | "verifier";
+            } | null;
+            submissionMode?: "structured_json" | undefined;
+            maxBytes: number | null;
+            metadata: Record<string, unknown>;
+        }[] | undefined;
+        contextRequirements?: string[] | undefined;
         rewardBinding: {
             id: string;
             contentHash: string;
@@ -369,14 +391,18 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
             };
             environment: {
                 protocolVersion: "openpond.environment.v1";
-                kind: "text" | "agent" | "work" | "custom_program";
+                kind: "agent" | "custom_program" | "text" | "work";
                 entrypoint: string;
                 stateful: boolean;
                 deterministicSeeds: boolean;
-                lifecycle: ("step" | "create" | "reset" | "collect" | "destroy")[];
-                networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+                lifecycle: ("collect" | "create" | "destroy" | "reset" | "step")[];
+                networkPolicy: "declared_read_only" | "declared_scoped" | "none";
                 defaultTimeoutMs: number;
             };
+            environmentRelease?: {
+                id: string;
+                contentHash: string;
+            } | undefined;
             tools: {
                 name: string;
                 description: string;
@@ -389,33 +415,14 @@ export declare function prepareLearningBatch(transaction: LearningTransaction, i
                 id: string;
                 required: boolean;
                 scopes: string[];
-                portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
+                portability: "host_adapter" | "hosted_only" | "local_only" | "portable";
             }[];
-            environmentRelease?: {
-                id: string;
-                contentHash: string;
-            } | undefined;
             verifierSetRelease?: {
                 id: string;
                 contentHash: string;
             } | undefined;
         };
         contentHash: string;
-        requiredOutputs?: {
-            path: string;
-            mediaType: string;
-            schemaRef: {
-                id: string;
-                path: string;
-                contentHash: string;
-                sizeBytes: number;
-                mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
-            } | null;
-            maxBytes: number | null;
-            metadata: Record<string, unknown>;
-        }[] | undefined;
-        contextRequirements?: string[] | undefined;
     };
 }>;
 export declare function requireCurrentLearningEvidence(transaction: LearningTransaction, ref: LearningRevisionRef): Promise<TaskEvidence>;

package/dist/types/learning/batch-service.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"batch-service.d.ts","sourceRoot":"","sources":["../../../../../src/learning/batch-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgC,KAAK,mBAAmB,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE3G,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAA8E,KAAK,uBAAuB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAErK,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAQ3M;AAED,kGAAkG;AAClG,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsBzK;AAED,wBAAsB,8BAA8B,CAAC,WAAW,EAAE,mBAAmB,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAKtI"}
+{"version":3,"file":"batch-service.d.ts","sourceRoot":"","sources":["../../../src/learning/batch-service.ts"],"names":[],"mappings":"AAEA,OAAO,EAAgC,KAAK,mBAAmB,EAAE,KAAK,YAAY,EAAE,MAAM,gBAAgB,CAAC;AAE3G,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,iBAAiB,CAAC;AACvD,OAAO,EAA8E,KAAK,uBAAuB,EAAE,KAAK,mBAAmB,EAAE,MAAM,iBAAiB,CAAC;AAErK,wBAAsB,iBAAiB,CAAC,WAAW,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,EAAE,CAAC,CAQ3M;AAED,kGAAkG;AAClG,wBAAsB,oBAAoB,CAAC,WAAW,EAAE,mBAAmB,EAAE,KAAK,EAAE,OAAO,CAAC,eAAe,EAAE;IAAE,MAAM,EAAE,YAAY,CAAA;CAAE,CAAC,EAAE,OAAO,EAAE,MAAM,EAAE,SAAS,EAAE,MAAM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAsBzK;AAED,wBAAsB,8BAA8B,CAAC,WAAW,EAAE,mBAAmB,EAAE,GAAG,EAAE,mBAAmB,GAAG,OAAO,CAAC,YAAY,CAAC,CAKtI"}

package/dist/types/learning/budgeted-reward-fixtures.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"budgeted-reward-fixtures.d.ts","sourceRoot":"","sources":["../../../../../src/learning/budgeted-reward-fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAG3E,OAAO,EAAE,2BAA2B,EAAE,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGhH,OAAO,EAA2B,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEtE,MAAM,WAAW,kBAAkB;IACjC;mFAC+E;IAC/E,OAAO,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,aAAa,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;KAAE,CAAC,CAAC;IACtN,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,YAAY,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpG;AAED,wBAAgB,mCAAmC,CAAC,OAAO,EAAE;IAC3D,OAAO,EAAE,kBAAkB,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAC;IAC1F,iBAAiB,EAAE,OAAO,yBAAyB,CAAC;IACpD,eAAe,CAAC,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;CACxF,GAAG,qBAAqB,CAyBxB"}
+{"version":3,"file":"budgeted-reward-fixtures.d.ts","sourceRoot":"","sources":["../../../src/learning/budgeted-reward-fixtures.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,2BAA2B,CAAC;AAG3E,OAAO,EAAE,2BAA2B,EAAE,KAAK,iBAAiB,EAAE,KAAK,kBAAkB,EAAE,MAAM,kBAAkB,CAAC;AAGhH,OAAO,EAA2B,KAAK,kBAAkB,EAAE,MAAM,iBAAiB,CAAC;AACnF,OAAO,KAAK,EAAE,cAAc,EAAE,kBAAkB,EAAE,MAAM,oBAAoB,CAAC;AAC7E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,0BAA0B,CAAC;AAEtE,MAAM,WAAW,kBAAkB;IACjC;mFAC+E;IAC/E,OAAO,CAAC,OAAO,EAAE,iBAAiB,EAAE,OAAO,CAAC,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,aAAa,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC;QAAE,gBAAgB,EAAE,MAAM,CAAC;QAAC,QAAQ,CAAC,MAAM,CAAC,EAAE,WAAW,GAAG,OAAO,CAAC,kBAAkB,CAAC,CAAA;KAAE,CAAC,CAAC;IACtN,MAAM,CAAC,KAAK,EAAE;QAAE,KAAK,EAAE,MAAM,CAAC;QAAC,GAAG,EAAE,IAAI,CAAC,cAAc,EAAE,IAAI,GAAG,YAAY,CAAC,CAAA;KAAE,GAAG,OAAO,CAAC,OAAO,CAAC,CAAC;CACpG;AAED,wBAAgB,mCAAmC,CAAC,OAAO,EAAE;IAC3D,OAAO,EAAE,kBAAkB,CAAC;IAAC,UAAU,EAAE,kBAAkB,CAAC;IAAC,QAAQ,EAAE,kBAAkB,CAAC;IAC1F,iBAAiB,EAAE,OAAO,yBAAyB,CAAC;IACpD,eAAe,CAAC,EAAE,UAAU,CAAC,OAAO,2BAA2B,CAAC,CAAC,CAAC,CAAC,CAAC,iBAAiB,CAAC,CAAC;CACxF,GAAG,qBAAqB,CAyBxB"}