@openpond/evals 0.12.0 → 0.12.2

package/dist/types/benchmarks.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}
+{"version":3,"file":"benchmarks.d.ts","sourceRoot":"","sources":["../../src/benchmarks.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,CAAC,EAAE,MAAM,KAAK,CAAC;AAYxB,OAAO,EAAE,KAAK,cAAc,EAAE,KAAK,gBAAgB,EAAE,MAAM,WAAW,CAAC;AAGvE,OAAO,EACL,6BAA6B,EAC7B,8BAA8B,GAC/B,MAAM,yCAAyC,CAAC;AAEjD,eAAO,MAAM,qBAAqB;;;;;EAKhC,CAAC;AACH,eAAO,MAAM,uBAAuB;;;EAAoC,CAAC;AAEzE,eAAO,MAAM,uBAAuB;;;;;;;;;;;;;;kBASzB,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAe3B,CAAC;AAEZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;kBAS3B,CAAC;AAQZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAkBlC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,eAAO,MAAM,gCAAgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAelC,CAAC;AACZ,eAAO,MAAM,yBAAyB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;kBAE3B,CAAC;AAEZ,wBAAgB,yBAAyB,CACvC,KAAK,EAAE,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,GAC/C,mBAAmB,CAErB;AAED,wBAAgB,yBAAyB,CAAC,KAAK,EAAE;IAC/C,EAAE,EAAE,MAAM,CAAC;IACX,KAAK,EAAE,iBAAiB,CAAC;IACzB,UAAU,EAAE,gBAAgB,CAAC;IAC7B,QAAQ,EAAE,cAAc,EAAE,CAAC;IAC3B,eAAe,EAAE,MAAM,GAAG,IAAI,CAAC;IAC/B,QAAQ,EAAE,iBAAiB,CAAC;IAC5B,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuCtB;AAED,wBAAgB,oBAAoB,CAAC,KAAK,EAAE;IAC1C,EAAE,EAAE,MAAM,CAAC;IACX,QAAQ,EAAE,mBAAmB,CAAC;IAC9B,SAAS,EAAE,mBAAmB,CAAC;IAC/B,aAAa,EAAE,eAAe,CAAC;IAC/B,WAAW,EAAE,MAAM,GAAG,gBAAgB,GAAG,UAAU,CAAC;IACpD,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,mBAAmB,CAuDtB;AAkDD,MAAM,MAAM,eAAe,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,qBAAqB,CAAC,CAAC;AACpE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,iBAAiB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,uBAAuB,CAAC,CAAC;AACxE,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC;AAC5E,MAAM,MAAM,mBAAmB,GAAG,CAAC,CAAC,KAAK,CAAC,OAAO,yBAAyB,CAAC,CAAC"}

package/dist/types/builtin-benchmarks/harness-refiner.d.ts

@@ -10,14 +10,18 @@ export declare const harnessRefinerBenchmarkRelease: {
     };
     environment: {
         protocolVersion: "openpond.environment.v1";
-        kind: "text" | "agent" | "work" | "custom_program";
+        kind: "agent" | "custom_program" | "text" | "work";
         entrypoint: string;
         stateful: boolean;
         deterministicSeeds: boolean;
-        lifecycle: ("step" | "create" | "reset" | "collect" | "destroy")[];
-        networkPolicy: "none" | "declared_read_only" | "declared_scoped";
+        lifecycle: ("collect" | "create" | "destroy" | "reset" | "step")[];
+        networkPolicy: "declared_read_only" | "declared_scoped" | "none";
         defaultTimeoutMs: number;
     };
+    environmentRelease?: {
+        id: string;
+        contentHash: string;
+    } | undefined;
     tools: {
         name: string;
         description: string;
@@ -30,12 +34,12 @@ export declare const harnessRefinerBenchmarkRelease: {
         id: string;
         required: boolean;
         scopes: string[];
-        portability: "portable" | "host_adapter" | "local_only" | "hosted_only";
+        portability: "host_adapter" | "hosted_only" | "local_only" | "portable";
     }[];
     tasks: {
         id: string;
         clusterKey: string;
-        split: "train" | "validation" | "test" | "frozen_eval";
+        split: "frozen_eval" | "test" | "train" | "validation";
         input: Record<string, unknown>;
         expectedOutput: Record<string, unknown> | null;
         policyVisibleContext: Record<string, unknown>;
@@ -46,9 +50,8 @@ export declare const harnessRefinerBenchmarkRelease: {
             contentHash: string;
             sizeBytes: number;
             mediaType: string;
-            visibility: "policy" | "verifier" | "host_private";
+            visibility: "host_private" | "policy" | "verifier";
         }[];
-        tags: string[];
         requiredOutputs?: {
             path: string;
             mediaType: string;
@@ -58,24 +61,27 @@ export declare const harnessRefinerBenchmarkRelease: {
                 contentHash: string;
                 sizeBytes: number;
                 mediaType: string;
-                visibility: "policy" | "verifier" | "host_private";
+                visibility: "host_private" | "policy" | "verifier";
             } | null;
+            submissionMode?: "structured_json" | undefined;
             maxBytes: number | null;
             metadata: Record<string, unknown>;
         }[] | undefined;
+        tags: string[];
     }[];
     graders: ({
         id: string;
+        taskIds?: string[] | undefined;
         version: string;
         weight: number;
         hardGate: boolean;
         rewardEligible: boolean;
         privileged: boolean;
-        kind: "content" | "schema" | "artifact" | "runtime_event" | "state";
+        kind: "artifact" | "content" | "runtime_event" | "schema" | "state";
         config: Record<string, unknown>;
-        taskIds?: string[] | undefined;
     } | {
         id: string;
+        taskIds?: string[] | undefined;
         version: string;
         weight: number;
         hardGate: boolean;
@@ -88,10 +94,9 @@ export declare const harnessRefinerBenchmarkRelease: {
             contentHash: string;
             sizeBytes: number;
             mediaType: string;
-            visibility: "policy" | "verifier" | "host_private";
+            visibility: "host_private" | "policy" | "verifier";
         };
-        calibrationStatus: "pending" | "passed" | "failed";
-        taskIds?: string[] | undefined;
+        calibrationStatus: "failed" | "passed" | "pending";
         model?: {
             providerId: string;
             modelId: string;
@@ -100,27 +105,28 @@ export declare const harnessRefinerBenchmarkRelease: {
         temperature?: number | undefined;
     } | {
         id: string;
+        taskIds?: string[] | undefined;
         version: string;
         weight: number;
         hardGate: boolean;
         rewardEligible: boolean;
         privileged: boolean;
         kind: "custom_verifier";
+        runtime?: "isolated_javascript" | "sandbox_process" | undefined;
         verifierRef: {
             id: string;
             path: string;
             contentHash: string;
             sizeBytes: number;
             mediaType: string;
-            visibility: "policy" | "verifier" | "host_private";
+            visibility: "host_private" | "policy" | "verifier";
         };
+        exportName?: string | undefined;
         timeoutMs: number;
         networkPolicy: "none";
-        taskIds?: string[] | undefined;
-        runtime?: "isolated_javascript" | "sandbox_process" | undefined;
-        exportName?: string | undefined;
     } | {
         id: string;
+        taskIds?: string[] | undefined;
         version: string;
         weight: number;
         hardGate: boolean;
@@ -133,10 +139,9 @@ export declare const harnessRefinerBenchmarkRelease: {
             contentHash: string;
             sizeBytes: number;
             mediaType: string;
-            visibility: "policy" | "verifier" | "host_private";
+            visibility: "host_private" | "policy" | "verifier";
         };
         reviewerRole: string;
-        taskIds?: string[] | undefined;
         form?: {
             schemaVersion: "openpond.humanForm.v1";
             mode: "individual" | "pairwise";
@@ -147,77 +152,71 @@ export declare const harnessRefinerBenchmarkRelease: {
                 instructions: string;
                 required: boolean;
                 allowAbstain: boolean;
-                kind: "score";
-                minimum: number;
-                maximum: number;
-                step: number;
                 requireRationale?: boolean | undefined;
                 anchors?: {
                     value: string | number | boolean;
                     description: string;
                 }[] | undefined;
+                kind: "score";
+                minimum: number;
+                maximum: number;
+                step: number;
             } | {
                 id: string;
                 label: string;
                 instructions: string;
                 required: boolean;
                 allowAbstain: boolean;
-                kind: "category";
-                options: string[];
                 requireRationale?: boolean | undefined;
                 anchors?: {
                     value: string | number | boolean;
                     description: string;
                 }[] | undefined;
+                kind: "category";
+                options: string[];
             } | {
                 id: string;
                 label: string;
                 instructions: string;
                 required: boolean;
                 allowAbstain: boolean;
-                kind: "boolean";
                 requireRationale?: boolean | undefined;
                 anchors?: {
                     value: string | number | boolean;
                     description: string;
                 }[] | undefined;
+                kind: "boolean";
             } | {
                 id: string;
                 label: string;
                 instructions: string;
                 required: boolean;
                 allowAbstain: boolean;
-                kind: "text";
-                maximumLength: number;
                 requireRationale?: boolean | undefined;
                 anchors?: {
                     value: string | number | boolean;
                     description: string;
                 }[] | undefined;
+                kind: "text";
                 minimumLength?: number | undefined;
+                maximumLength: number;
             } | {
                 id: string;
                 label: string;
                 instructions: string;
                 required: boolean;
                 allowAbstain: boolean;
-                kind: "preference";
-                allowTie: boolean;
                 requireRationale?: boolean | undefined;
                 anchors?: {
                     value: string | number | boolean;
                     description: string;
                 }[] | undefined;
+                kind: "preference";
                 requireStrength?: boolean | undefined;
+                allowTie: boolean;
             })[];
         } | undefined;
     })[];
-    metadata: Record<string, unknown>;
-    contentHash: string;
-    environmentRelease?: {
-        id: string;
-        contentHash: string;
-    } | undefined;
     verifierSetRelease?: {
         id: string;
         contentHash: string;
@@ -225,8 +224,9 @@ export declare const harnessRefinerBenchmarkRelease: {
     metrics?: {
         schemaVersion: "openpond.tasksetMetricPolicy.v1";
         primaryMetric: string;
-        aggregation: "mean_score" | "pass_rate" | "weighted_mean" | "custom";
-        missingReward: "zero" | "exclude";
+        aggregation: "custom" | "mean_score" | "pass_rate" | "weighted_mean";
+        missingReward: "exclude" | "zero";
+        taskWeights?: Record<string, number> | undefined;
         customAggregator: {
             module: string;
             exportName: string;
@@ -234,8 +234,9 @@ export declare const harnessRefinerBenchmarkRelease: {
             timeoutMs: number;
             networkPolicy: "none";
         } | null;
-        taskWeights?: Record<string, number> | undefined;
     } | undefined;
+    metadata: Record<string, unknown>;
+    contentHash: string;
 };
 export declare const harnessRefinerBenchmarkAssets: Readonly<Record<string, string>>;
 //# sourceMappingURL=harness-refiner.d.ts.map

package/dist/types/builtin-benchmarks/harness-refiner.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}
+{"version":3,"file":"harness-refiner.d.ts","sourceRoot":"","sources":["../../../src/builtin-benchmarks/harness-refiner.ts"],"names":[],"mappings":"AAGA,eAAO,MAAM,8BAA8B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;CAqqD1C,CAAC;AAEF,eAAO,MAAM,6BAA6B,EAAE,QAAQ,CAAC,MAAM,CAAC,MAAM,EAAE,MAAM,CAAC,CASzE,CAAC"}

package/dist/types/compatibility.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,yCAAyC,CAAC,KAAK,EAAE;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,cAAc,CAAC;IACnC,uBAAuB,EAAE,cAAc,CAAC;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,2BAA2B,CA4C9B"}
+{"version":3,"file":"compatibility.d.ts","sourceRoot":"","sources":["../../src/compatibility.ts"],"names":[],"mappings":"AACA,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,mBAAmB,CAAC;AAC3B,OAAO,EAEL,KAAK,2BAA2B,EACjC,MAAM,WAAW,CAAC;AACnB,OAAO,EAEL,KAAK,cAAc,EACpB,MAAM,eAAe,CAAC;AAEvB,wBAAgB,yCAAyC,CAAC,KAAK,EAAE;IAC/D,EAAE,EAAE,MAAM,CAAC;IACX,kBAAkB,EAAE,cAAc,CAAC;IACnC,uBAAuB,EAAE,cAAc,CAAC;IACxC,cAAc,EAAE,cAAc,CAAC;IAC/B,QAAQ,CAAC,EAAE,MAAM,CAAC,MAAM,EAAE,OAAO,CAAC,CAAC;CACpC,GAAG,2BAA2B,CA4C9B"}