@openpond/evals 0.11.20 → 0.12.0

package/schemas/learning/v1/package.schema.json

@@ -158,7 +158,6 @@
       }
     },
     "graders": {
-      "minItems": 1,
       "maxItems": 1000,
       "type": "array",
       "items": {
@@ -185,22 +184,22 @@
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema45"
+          "$ref": "#/$defs/__schema46"
         },
         "primaryMetric": {
-          "$ref": "#/$defs/__schema46"
+          "$ref": "#/$defs/__schema47"
         },
         "aggregation": {
-          "$ref": "#/$defs/__schema47"
+          "$ref": "#/$defs/__schema48"
         },
         "missingReward": {
-          "$ref": "#/$defs/__schema48"
+          "$ref": "#/$defs/__schema49"
         },
         "taskWeights": {
-          "$ref": "#/$defs/__schema49"
+          "$ref": "#/$defs/__schema50"
         },
         "customAggregator": {
-          "$ref": "#/$defs/__schema50"
+          "$ref": "#/$defs/__schema51"
         }
       },
       "required": [
@@ -543,21 +542,24 @@
             "id": {
               "$ref": "#/$defs/__schema0"
             },
-            "version": {
+            "taskIds": {
               "$ref": "#/$defs/__schema11"
             },
-            "weight": {
+            "version": {
               "$ref": "#/$defs/__schema12"
             },
-            "hardGate": {
+            "weight": {
               "$ref": "#/$defs/__schema13"
             },
-            "rewardEligible": {
+            "hardGate": {
               "$ref": "#/$defs/__schema14"
             },
-            "privileged": {
+            "rewardEligible": {
               "$ref": "#/$defs/__schema15"
             },
+            "privileged": {
+              "$ref": "#/$defs/__schema16"
+            },
             "kind": {
               "type": "string",
               "enum": [
@@ -594,21 +596,24 @@
             "id": {
               "$ref": "#/$defs/__schema0"
             },
-            "version": {
+            "taskIds": {
               "$ref": "#/$defs/__schema11"
             },
-            "weight": {
+            "version": {
               "$ref": "#/$defs/__schema12"
             },
-            "hardGate": {
+            "weight": {
               "$ref": "#/$defs/__schema13"
             },
-            "rewardEligible": {
+            "hardGate": {
               "$ref": "#/$defs/__schema14"
             },
-            "privileged": {
+            "rewardEligible": {
               "$ref": "#/$defs/__schema15"
             },
+            "privileged": {
+              "$ref": "#/$defs/__schema16"
+            },
             "kind": {
               "type": "string",
               "const": "model_judge"
@@ -678,21 +683,24 @@
             "id": {
               "$ref": "#/$defs/__schema0"
             },
-            "version": {
+            "taskIds": {
               "$ref": "#/$defs/__schema11"
             },
-            "weight": {
+            "version": {
               "$ref": "#/$defs/__schema12"
             },
-            "hardGate": {
+            "weight": {
               "$ref": "#/$defs/__schema13"
             },
-            "rewardEligible": {
+            "hardGate": {
               "$ref": "#/$defs/__schema14"
             },
-            "privileged": {
+            "rewardEligible": {
               "$ref": "#/$defs/__schema15"
             },
+            "privileged": {
+              "$ref": "#/$defs/__schema16"
+            },
             "kind": {
               "type": "string",
               "const": "custom_verifier"
@@ -742,21 +750,24 @@
             "id": {
               "$ref": "#/$defs/__schema0"
             },
-            "version": {
+            "taskIds": {
               "$ref": "#/$defs/__schema11"
             },
-            "weight": {
+            "version": {
               "$ref": "#/$defs/__schema12"
             },
-            "hardGate": {
+            "weight": {
               "$ref": "#/$defs/__schema13"
             },
-            "rewardEligible": {
+            "hardGate": {
               "$ref": "#/$defs/__schema14"
             },
-            "privileged": {
+            "rewardEligible": {
               "$ref": "#/$defs/__schema15"
             },
+            "privileged": {
+              "$ref": "#/$defs/__schema16"
+            },
             "kind": {
               "type": "string",
               "const": "human"
@@ -773,16 +784,16 @@
               "type": "object",
               "properties": {
                 "schemaVersion": {
-                  "$ref": "#/$defs/__schema16"
+                  "$ref": "#/$defs/__schema17"
                 },
                 "mode": {
-                  "$ref": "#/$defs/__schema17"
+                  "$ref": "#/$defs/__schema18"
                 },
                 "instructions": {
-                  "$ref": "#/$defs/__schema18"
+                  "$ref": "#/$defs/__schema19"
                 },
                 "criteria": {
-                  "$ref": "#/$defs/__schema19"
+                  "$ref": "#/$defs/__schema20"
                 }
               },
               "required": [
@@ -810,20 +821,24 @@
       ]
     },
     "__schema11": {
+      "minItems": 1,
+      "maxItems": 10000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema0"
+      }
+    },
+    "__schema12": {
       "type": "string",
       "minLength": 1,
       "maxLength": 100
     },
-    "__schema12": {
+    "__schema13": {
       "default": 1,
       "type": "number",
       "minimum": 0,
       "maximum": 1000
     },
-    "__schema13": {
-      "default": false,
-      "type": "boolean"
-    },
     "__schema14": {
       "default": false,
       "type": "boolean"
@@ -833,29 +848,33 @@
       "type": "boolean"
     },
     "__schema16": {
+      "default": false,
+      "type": "boolean"
+    },
+    "__schema17": {
       "type": "string",
       "const": "openpond.humanForm.v1"
     },
-    "__schema17": {
+    "__schema18": {
       "type": "string",
       "enum": [
         "individual",
         "pairwise"
       ]
     },
-    "__schema18": {
+    "__schema19": {
       "type": "string",
       "maxLength": 20000
     },
-    "__schema19": {
+    "__schema20": {
       "minItems": 1,
       "maxItems": 100,
       "type": "array",
       "items": {
-        "$ref": "#/$defs/__schema20"
+        "$ref": "#/$defs/__schema21"
       }
     },
-    "__schema20": {
+    "__schema21": {
       "oneOf": [
         {
           "type": "object",
@@ -864,34 +883,34 @@
               "$ref": "#/$defs/__schema0"
             },
             "label": {
-              "$ref": "#/$defs/__schema21"
+              "$ref": "#/$defs/__schema22"
             },
             "instructions": {
-              "$ref": "#/$defs/__schema22"
+              "$ref": "#/$defs/__schema23"
             },
             "required": {
-              "$ref": "#/$defs/__schema23"
+              "$ref": "#/$defs/__schema24"
             },
             "allowAbstain": {
-              "$ref": "#/$defs/__schema24"
+              "$ref": "#/$defs/__schema25"
             },
             "requireRationale": {
-              "$ref": "#/$defs/__schema25"
+              "$ref": "#/$defs/__schema26"
             },
             "anchors": {
-              "$ref": "#/$defs/__schema26"
+              "$ref": "#/$defs/__schema27"
             },
             "kind": {
-              "$ref": "#/$defs/__schema28"
+              "$ref": "#/$defs/__schema29"
             },
             "minimum": {
-              "$ref": "#/$defs/__schema29"
+              "$ref": "#/$defs/__schema30"
             },
             "maximum": {
-              "$ref": "#/$defs/__schema30"
+              "$ref": "#/$defs/__schema31"
             },
             "step": {
-              "$ref": "#/$defs/__schema31"
+              "$ref": "#/$defs/__schema32"
             }
           },
           "required": [
@@ -914,28 +933,28 @@
               "$ref": "#/$defs/__schema0"
             },
             "label": {
-              "$ref": "#/$defs/__schema32"
+              "$ref": "#/$defs/__schema33"
             },
             "instructions": {
-              "$ref": "#/$defs/__schema33"
+              "$ref": "#/$defs/__schema34"
             },
             "required": {
-              "$ref": "#/$defs/__schema34"
+              "$ref": "#/$defs/__schema35"
             },
             "allowAbstain": {
-              "$ref": "#/$defs/__schema35"
+              "$ref": "#/$defs/__schema36"
             },
             "requireRationale": {
-              "$ref": "#/$defs/__schema36"
+              "$ref": "#/$defs/__schema37"
             },
             "anchors": {
-              "$ref": "#/$defs/__schema37"
+              "$ref": "#/$defs/__schema38"
             },
             "kind": {
-              "$ref": "#/$defs/__schema39"
+              "$ref": "#/$defs/__schema40"
             },
             "options": {
-              "$ref": "#/$defs/__schema40"
+              "$ref": "#/$defs/__schema41"
             }
           },
           "required": [
@@ -977,7 +996,7 @@
               "maxItems": 100,
               "type": "array",
               "items": {
-                "$ref": "#/$defs/__schema42"
+                "$ref": "#/$defs/__schema43"
               }
             },
             "kind": {
@@ -1023,7 +1042,7 @@
               "maxItems": 100,
               "type": "array",
               "items": {
-                "$ref": "#/$defs/__schema43"
+                "$ref": "#/$defs/__schema44"
               }
             },
             "kind": {
@@ -1080,7 +1099,7 @@
               "maxItems": 100,
               "type": "array",
               "items": {
-                "$ref": "#/$defs/__schema44"
+                "$ref": "#/$defs/__schema45"
               }
             },
             "kind": {
@@ -1107,18 +1126,15 @@
         }
       ]
     },
-    "__schema21": {
+    "__schema22": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema22": {
+    "__schema23": {
       "type": "string",
       "maxLength": 20000
     },
-    "__schema23": {
-      "type": "boolean"
-    },
     "__schema24": {
       "type": "boolean"
     },
@@ -1126,13 +1142,16 @@
       "type": "boolean"
     },
     "__schema26": {
+      "type": "boolean"
+    },
+    "__schema27": {
       "maxItems": 100,
       "type": "array",
       "items": {
-        "$ref": "#/$defs/__schema27"
+        "$ref": "#/$defs/__schema28"
       }
     },
-    "__schema27": {
+    "__schema28": {
       "type": "object",
       "properties": {
         "value": {
@@ -1160,32 +1179,29 @@
       ],
       "additionalProperties": false
     },
-    "__schema28": {
+    "__schema29": {
       "type": "string",
       "const": "score"
     },
-    "__schema29": {
-      "type": "number"
-    },
     "__schema30": {
       "type": "number"
     },
     "__schema31": {
+      "type": "number"
+    },
+    "__schema32": {
       "type": "number",
       "exclusiveMinimum": 0
     },
-    "__schema32": {
+    "__schema33": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema33": {
+    "__schema34": {
       "type": "string",
       "maxLength": 20000
     },
-    "__schema34": {
-      "type": "boolean"
-    },
     "__schema35": {
       "type": "boolean"
     },
@@ -1193,13 +1209,16 @@
       "type": "boolean"
     },
     "__schema37": {
+      "type": "boolean"
+    },
+    "__schema38": {
       "maxItems": 100,
       "type": "array",
       "items": {
-        "$ref": "#/$defs/__schema38"
+        "$ref": "#/$defs/__schema39"
       }
     },
-    "__schema38": {
+    "__schema39": {
       "type": "object",
       "properties": {
         "value": {
@@ -1227,24 +1246,24 @@
       ],
       "additionalProperties": false
     },
-    "__schema39": {
+    "__schema40": {
       "type": "string",
       "const": "category"
     },
-    "__schema40": {
+    "__schema41": {
       "minItems": 2,
       "maxItems": 100,
       "type": "array",
       "items": {
-        "$ref": "#/$defs/__schema41"
+        "$ref": "#/$defs/__schema42"
       }
     },
-    "__schema41": {
+    "__schema42": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema42": {
+    "__schema43": {
       "type": "object",
       "properties": {
         "value": {
@@ -1272,7 +1291,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema43": {
+    "__schema44": {
       "type": "object",
       "properties": {
         "value": {
@@ -1300,7 +1319,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema44": {
+    "__schema45": {
       "type": "object",
       "properties": {
         "value": {
@@ -1328,16 +1347,16 @@
       ],
       "additionalProperties": false
     },
-    "__schema45": {
+    "__schema46": {
       "type": "string",
       "const": "openpond.tasksetMetricPolicy.v1"
     },
-    "__schema46": {
+    "__schema47": {
       "type": "string",
       "minLength": 1,
       "maxLength": 240
     },
-    "__schema47": {
+    "__schema48": {
       "type": "string",
       "enum": [
         "mean_score",
@@ -1346,17 +1365,17 @@
         "custom"
       ]
     },
-    "__schema48": {
+    "__schema49": {
       "type": "string",
       "enum": [
         "zero",
         "exclude"
       ]
     },
-    "__schema49": {
+    "__schema50": {
       "type": "object",
       "propertyNames": {
-        "$ref": "#/$defs/__schema46"
+        "$ref": "#/$defs/__schema47"
       },
       "additionalProperties": {
         "type": "number",
@@ -1364,7 +1383,7 @@
         "maximum": 1000000
       }
     },
-    "__schema50": {
+    "__schema51": {
       "anyOf": [
         {
           "type": "object",

package/schemas/learning/v1/reward_check.schema.json

@@ -264,7 +264,8 @@
                 "scored",
                 "pending",
                 "unavailable",
-                "failed"
+                "failed",
+                "not_applicable"
               ]
             },
             "rawScore": {