@openpond/evals 0.10.5 → 0.10.7

package/schemas/learning/v1/command-request.schema.json

@@ -625,10 +625,10 @@
                   "$ref": "#/$defs/__schema13"
                 },
                 "kind": {
-                  "$ref": "#/$defs/__schema37"
+                  "$ref": "#/$defs/__schema39"
                 },
                 "content": {
-                  "$ref": "#/$defs/__schema38"
+                  "$ref": "#/$defs/__schema40"
                 }
               },
               "required": [
@@ -656,10 +656,10 @@
                   "$ref": "#/$defs/__schema13"
                 },
                 "kind": {
-                  "$ref": "#/$defs/__schema47"
+                  "$ref": "#/$defs/__schema49"
                 },
                 "content": {
-                  "$ref": "#/$defs/__schema48"
+                  "$ref": "#/$defs/__schema50"
                 }
               },
               "required": [
@@ -687,10 +687,10 @@
                   "$ref": "#/$defs/__schema13"
                 },
                 "kind": {
-                  "$ref": "#/$defs/__schema57"
+                  "$ref": "#/$defs/__schema59"
                 },
                 "content": {
-                  "$ref": "#/$defs/__schema58"
+                  "$ref": "#/$defs/__schema60"
                 }
               },
               "required": [
@@ -718,10 +718,10 @@
                   "$ref": "#/$defs/__schema13"
                 },
                 "kind": {
-                  "$ref": "#/$defs/__schema70"
+                  "$ref": "#/$defs/__schema72"
                 },
                 "content": {
-                  "$ref": "#/$defs/__schema71"
+                  "$ref": "#/$defs/__schema73"
                 }
               },
               "required": [
@@ -753,7 +753,7 @@
               "maxItems": 20,
               "type": "array",
               "items": {
-                "$ref": "#/$defs/__schema84"
+                "$ref": "#/$defs/__schema86"
               }
             }
           },
@@ -810,15 +810,15 @@
                   ]
                 },
                 "split": {
-                  "$ref": "#/$defs/__schema65"
+                  "$ref": "#/$defs/__schema67"
                 },
                 "input": {
-                  "$ref": "#/$defs/__schema85"
+                  "$ref": "#/$defs/__schema87"
                 },
                 "observedOutput": {
                   "anyOf": [
                     {
-                      "$ref": "#/$defs/__schema85"
+                      "$ref": "#/$defs/__schema87"
                     },
                     {
                       "type": "null"
@@ -828,7 +828,7 @@
                 "expected": {
                   "anyOf": [
                     {
-                      "$ref": "#/$defs/__schema85"
+                      "$ref": "#/$defs/__schema87"
                     },
                     {
                       "type": "null"
@@ -838,7 +838,7 @@
                 "evaluatorContext": {
                   "anyOf": [
                     {
-                      "$ref": "#/$defs/__schema85"
+                      "$ref": "#/$defs/__schema87"
                     },
                     {
                       "type": "null"
@@ -964,7 +964,7 @@
                   ]
                 },
                 "value": {
-                  "$ref": "#/$defs/__schema85"
+                  "$ref": "#/$defs/__schema87"
                 },
                 "note": {
                   "type": "string",
@@ -1093,7 +1093,7 @@
             "proposedTarget": {
               "anyOf": [
                 {
-                  "$ref": "#/$defs/__schema85"
+                  "$ref": "#/$defs/__schema87"
                 },
                 {
                   "type": "null"
@@ -1162,7 +1162,7 @@
             "approvedTarget": {
               "anyOf": [
                 {
-                  "$ref": "#/$defs/__schema85"
+                  "$ref": "#/$defs/__schema87"
                 },
                 {
                   "type": "null"
@@ -1709,14 +1709,17 @@
         "outputSchema": {
           "$ref": "#/$defs/__schema30"
         },
+        "requiredOutputs": {
+          "$ref": "#/$defs/__schema31"
+        },
         "rewardBinding": {
           "$ref": "#/$defs/__schema1"
         },
         "harness": {
-          "$ref": "#/$defs/__schema31"
+          "$ref": "#/$defs/__schema33"
         },
         "execution": {
-          "$ref": "#/$defs/__schema32"
+          "$ref": "#/$defs/__schema34"
         }
       },
       "required": [
@@ -1784,6 +1787,68 @@
       "additionalProperties": {}
     },
     "__schema31": {
+      "maxItems": 1000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema32"
+      }
+    },
+    "__schema32": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "mediaType": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 200
+        },
+        "schemaRef": {
+          "default": null,
+          "anyOf": [
+            {
+              "$ref": "#/$defs/__schema19"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "maxBytes": {
+          "default": null,
+          "anyOf": [
+            {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 250000000
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "metadata": {
+          "default": {},
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        }
+      },
+      "required": [
+        "path",
+        "mediaType",
+        "schemaRef",
+        "maxBytes",
+        "metadata"
+      ],
+      "additionalProperties": false
+    },
+    "__schema33": {
       "anyOf": [
         {
           "$ref": "#/$defs/__schema7"
@@ -1793,7 +1858,7 @@
         }
       ]
     },
-    "__schema32": {
+    "__schema34": {
       "type": "object",
       "properties": {
         "policy": {
@@ -1872,7 +1937,7 @@
               "maxItems": 5,
               "type": "array",
               "items": {
-                "$ref": "#/$defs/__schema33"
+                "$ref": "#/$defs/__schema35"
               }
             },
             "networkPolicy": {
@@ -1921,14 +1986,14 @@
           "maxItems": 200,
           "type": "array",
           "items": {
-            "$ref": "#/$defs/__schema34"
+            "$ref": "#/$defs/__schema36"
           }
         },
         "capabilities": {
           "maxItems": 200,
           "type": "array",
           "items": {
-            "$ref": "#/$defs/__schema35"
+            "$ref": "#/$defs/__schema37"
           }
         },
         "verifierSetRelease": {
@@ -1956,7 +2021,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema33": {
+    "__schema35": {
       "type": "string",
       "enum": [
         "create",
@@ -1966,7 +2031,7 @@
         "destroy"
       ]
     },
-    "__schema34": {
+    "__schema36": {
       "type": "object",
       "properties": {
         "name": {
@@ -2013,7 +2078,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema35": {
+    "__schema37": {
       "type": "object",
       "properties": {
         "id": {
@@ -2027,7 +2092,7 @@
           "maxItems": 100,
           "type": "array",
           "items": {
-            "$ref": "#/$defs/__schema36"
+            "$ref": "#/$defs/__schema38"
           }
         },
         "portability": {
@@ -2048,44 +2113,44 @@
       ],
       "additionalProperties": false
     },
-    "__schema36": {
+    "__schema38": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema37": {
+    "__schema39": {
       "type": "string",
       "const": "reward"
     },
-    "__schema38": {
+    "__schema40": {
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema39"
+          "$ref": "#/$defs/__schema41"
         },
         "id": {
           "$ref": "#/$defs/__schema0"
         },
         "revision": {
-          "$ref": "#/$defs/__schema40"
+          "$ref": "#/$defs/__schema42"
         },
         "name": {
-          "$ref": "#/$defs/__schema41"
+          "$ref": "#/$defs/__schema43"
         },
         "description": {
-          "$ref": "#/$defs/__schema42"
+          "$ref": "#/$defs/__schema44"
         },
         "implementation": {
-          "$ref": "#/$defs/__schema43"
+          "$ref": "#/$defs/__schema45"
         },
         "rawScore": {
-          "$ref": "#/$defs/__schema44"
+          "$ref": "#/$defs/__schema46"
         },
         "assets": {
-          "$ref": "#/$defs/__schema45"
+          "$ref": "#/$defs/__schema47"
         },
         "fixtureSetRef": {
-          "$ref": "#/$defs/__schema46"
+          "$ref": "#/$defs/__schema48"
         }
       },
       "required": [
@@ -2100,25 +2165,25 @@
       ],
       "additionalProperties": false
     },
-    "__schema39": {
+    "__schema41": {
       "type": "string",
       "const": "openpond.rewardRelease.v1"
     },
-    "__schema40": {
+    "__schema42": {
       "type": "integer",
       "exclusiveMinimum": 0,
       "maximum": 9007199254740991
     },
-    "__schema41": {
+    "__schema43": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema42": {
+    "__schema44": {
       "type": "string",
       "maxLength": 10000
     },
-    "__schema43": {
+    "__schema45": {
       "anyOf": [
         {
           "type": "object",
@@ -2286,7 +2351,7 @@
         }
       ]
     },
-    "__schema44": {
+    "__schema46": {
       "type": "object",
       "properties": {
         "minimum": {
@@ -2302,49 +2367,49 @@
       ],
       "additionalProperties": false
     },
-    "__schema45": {
+    "__schema47": {
       "maxItems": 1000,
       "type": "array",
       "items": {
         "$ref": "#/$defs/__schema19"
       }
     },
-    "__schema46": {
+    "__schema48": {
       "$ref": "#/$defs/__schema19"
     },
-    "__schema47": {
+    "__schema49": {
       "type": "string",
       "const": "binding"
     },
-    "__schema48": {
+    "__schema50": {
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema49"
+          "$ref": "#/$defs/__schema51"
         },
         "id": {
           "$ref": "#/$defs/__schema0"
         },
         "revision": {
-          "$ref": "#/$defs/__schema50"
+          "$ref": "#/$defs/__schema52"
         },
         "name": {
-          "$ref": "#/$defs/__schema51"
+          "$ref": "#/$defs/__schema53"
         },
         "description": {
-          "$ref": "#/$defs/__schema52"
+          "$ref": "#/$defs/__schema54"
         },
         "recipeRef": {
-          "$ref": "#/$defs/__schema53"
+          "$ref": "#/$defs/__schema55"
         },
         "sources": {
-          "$ref": "#/$defs/__schema54"
+          "$ref": "#/$defs/__schema56"
         },
         "aggregation": {
-          "$ref": "#/$defs/__schema55"
+          "$ref": "#/$defs/__schema57"
         },
         "unscorable": {
-          "$ref": "#/$defs/__schema56"
+          "$ref": "#/$defs/__schema58"
         }
       },
       "required": [
@@ -2357,28 +2422,28 @@
       ],
       "additionalProperties": false
     },
-    "__schema49": {
+    "__schema51": {
       "type": "string",
       "const": "openpond.rewardBinding.v1"
     },
-    "__schema50": {
+    "__schema52": {
       "type": "integer",
       "exclusiveMinimum": 0,
       "maximum": 9007199254740991
     },
-    "__schema51": {
+    "__schema53": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema52": {
+    "__schema54": {
       "type": "string",
       "maxLength": 10000
     },
-    "__schema53": {
+    "__schema55": {
       "$ref": "#/$defs/__schema1"
     },
-    "__schema54": {
+    "__schema56": {
       "minItems": 1,
       "maxItems": 100,
       "type": "array",
@@ -2386,53 +2451,53 @@
         "$ref": "#/$defs/__schema6"
       }
     },
-    "__schema55": {
+    "__schema57": {
       "type": "string",
       "const": "weighted_mean"
     },
-    "__schema56": {
+    "__schema58": {
       "type": "string",
       "const": "exclude_optional_require_all_required"
     },
-    "__schema57": {
+    "__schema59": {
       "type": "string",
       "const": "source"
     },
-    "__schema58": {
+    "__schema60": {
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema59"
+          "$ref": "#/$defs/__schema61"
         },
         "id": {
           "$ref": "#/$defs/__schema0"
         },
         "revision": {
-          "$ref": "#/$defs/__schema60"
+          "$ref": "#/$defs/__schema62"
         },
         "name": {
-          "$ref": "#/$defs/__schema61"
+          "$ref": "#/$defs/__schema63"
         },
         "kind": {
-          "$ref": "#/$defs/__schema62"
+          "$ref": "#/$defs/__schema64"
         },
         "taskDefinition": {
           "$ref": "#/$defs/__schema1"
         },
         "enabled": {
-          "$ref": "#/$defs/__schema63"
+          "$ref": "#/$defs/__schema65"
         },
         "allowedSplits": {
-          "$ref": "#/$defs/__schema64"
+          "$ref": "#/$defs/__schema66"
         },
         "mapping": {
-          "$ref": "#/$defs/__schema66"
+          "$ref": "#/$defs/__schema68"
         },
         "adapterVersion": {
-          "$ref": "#/$defs/__schema68"
+          "$ref": "#/$defs/__schema70"
         },
         "reviewOrigin": {
-          "$ref": "#/$defs/__schema69"
+          "$ref": "#/$defs/__schema71"
         }
       },
       "required": [
@@ -2449,21 +2514,21 @@
       ],
       "additionalProperties": false
     },
-    "__schema59": {
+    "__schema61": {
       "type": "string",
       "const": "openpond.learningSource.v1"
     },
-    "__schema60": {
+    "__schema62": {
       "type": "integer",
       "exclusiveMinimum": 0,
       "maximum": 9007199254740991
     },
-    "__schema61": {
+    "__schema63": {
       "type": "string",
       "minLength": 1,
       "maxLength": 500
     },
-    "__schema62": {
+    "__schema64": {
       "type": "string",
       "enum": [
         "direct",
@@ -2473,18 +2538,18 @@
         "provider"
       ]
     },
-    "__schema63": {
+    "__schema65": {
       "type": "boolean"
     },
-    "__schema64": {
+    "__schema66": {
       "minItems": 1,
       "maxItems": 4,
       "type": "array",
       "items": {
-        "$ref": "#/$defs/__schema65"
+        "$ref": "#/$defs/__schema67"
       }
     },
-    "__schema65": {
+    "__schema67": {
       "type": "string",
       "enum": [
         "train",
@@ -2493,7 +2558,7 @@
         "frozen_eval"
       ]
     },
-    "__schema66": {
+    "__schema68": {
       "anyOf": [
         {
           "type": "object",
@@ -2503,21 +2568,21 @@
               "const": "openpond.taskSourceMapping.v1"
             },
             "exampleId": {
-              "$ref": "#/$defs/__schema67"
+              "$ref": "#/$defs/__schema69"
             },
             "attemptId": {
-              "$ref": "#/$defs/__schema67"
+              "$ref": "#/$defs/__schema69"
             },
             "occurredAt": {
-              "$ref": "#/$defs/__schema67"
+              "$ref": "#/$defs/__schema69"
             },
             "input": {
-              "$ref": "#/$defs/__schema67"
+              "$ref": "#/$defs/__schema69"
             },
             "observedOutput": {
               "anyOf": [
                 {
-                  "$ref": "#/$defs/__schema67"
+                  "$ref": "#/$defs/__schema69"
                 },
                 {
                   "type": "null"
@@ -2527,7 +2592,7 @@
             "expected": {
               "anyOf": [
                 {
-                  "$ref": "#/$defs/__schema67"
+                  "$ref": "#/$defs/__schema69"
                 },
                 {
                   "type": "null"
@@ -2537,7 +2602,7 @@
             "evaluatorContext": {
               "anyOf": [
                 {
-                  "$ref": "#/$defs/__schema67"
+                  "$ref": "#/$defs/__schema69"
                 },
                 {
                   "type": "null"
@@ -2545,10 +2610,10 @@
               ]
             },
             "familyKey": {
-              "$ref": "#/$defs/__schema67"
+              "$ref": "#/$defs/__schema69"
             },
             "split": {
-              "$ref": "#/$defs/__schema65"
+              "$ref": "#/$defs/__schema67"
             }
           },
           "required": [
@@ -2570,11 +2635,11 @@
         }
       ]
     },
-    "__schema67": {
+    "__schema69": {
       "type": "string",
       "maxLength": 2000
     },
-    "__schema68": {
+    "__schema70": {
       "anyOf": [
         {
           "type": "string",
@@ -2586,7 +2651,7 @@
         }
       ]
     },
-    "__schema69": {
+    "__schema71": {
       "type": "object",
       "properties": {
         "modelId": {
@@ -2614,33 +2679,33 @@
       ],
       "additionalProperties": false
     },
-    "__schema70": {
+    "__schema72": {
       "type": "string",
       "const": "policy"
     },
-    "__schema71": {
+    "__schema73": {
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema72"
+          "$ref": "#/$defs/__schema74"
         },
         "id": {
           "$ref": "#/$defs/__schema0"
         },
         "revision": {
-          "$ref": "#/$defs/__schema73"
+          "$ref": "#/$defs/__schema75"
         },
         "modelProjectId": {
           "$ref": "#/$defs/__schema0"
         },
         "executionOwner": {
-          "$ref": "#/$defs/__schema74"
+          "$ref": "#/$defs/__schema76"
         },
         "enabled": {
-          "$ref": "#/$defs/__schema75"
+          "$ref": "#/$defs/__schema77"
         },
         "sources": {
-          "$ref": "#/$defs/__schema76"
+          "$ref": "#/$defs/__schema78"
         },
         "taskDefinition": {
           "$ref": "#/$defs/__schema1"
@@ -2649,28 +2714,28 @@
           "$ref": "#/$defs/__schema1"
         },
         "admission": {
-          "$ref": "#/$defs/__schema77"
+          "$ref": "#/$defs/__schema79"
         },
         "trigger": {
-          "$ref": "#/$defs/__schema78"
+          "$ref": "#/$defs/__schema80"
         },
         "trainingParent": {
           "$ref": "#/$defs/__schema7"
         },
         "teacher": {
-          "$ref": "#/$defs/__schema79"
+          "$ref": "#/$defs/__schema81"
         },
         "training": {
-          "$ref": "#/$defs/__schema80"
+          "$ref": "#/$defs/__schema82"
         },
         "limits": {
-          "$ref": "#/$defs/__schema81"
+          "$ref": "#/$defs/__schema83"
         },
         "automation": {
-          "$ref": "#/$defs/__schema82"
+          "$ref": "#/$defs/__schema84"
         },
         "acceptance": {
-          "$ref": "#/$defs/__schema83"
+          "$ref": "#/$defs/__schema85"
         }
       },
       "required": [
@@ -2694,26 +2759,26 @@
       ],
       "additionalProperties": false
     },
-    "__schema72": {
+    "__schema74": {
       "type": "string",
       "const": "openpond.learningPolicy.v1"
     },
-    "__schema73": {
+    "__schema75": {
       "type": "integer",
       "exclusiveMinimum": 0,
       "maximum": 9007199254740991
     },
-    "__schema74": {
+    "__schema76": {
       "type": "string",
       "enum": [
         "local",
         "hosted"
       ]
     },
-    "__schema75": {
+    "__schema77": {
       "type": "boolean"
     },
-    "__schema76": {
+    "__schema78": {
       "minItems": 1,
       "maxItems": 100,
       "type": "array",
@@ -2721,7 +2786,7 @@
         "$ref": "#/$defs/__schema1"
       }
     },
-    "__schema77": {
+    "__schema79": {
       "type": "object",
       "properties": {
         "mode": {
@@ -2754,7 +2819,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema78": {
+    "__schema80": {
       "oneOf": [
         {
           "type": "object",
@@ -2820,7 +2885,7 @@
         }
       ]
     },
-    "__schema79": {
+    "__schema81": {
       "anyOf": [
         {
           "$ref": "#/$defs/__schema7"
@@ -2830,7 +2895,7 @@
         }
       ]
     },
-    "__schema80": {
+    "__schema82": {
       "type": "object",
       "properties": {
         "method": {
@@ -2868,7 +2933,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema81": {
+    "__schema83": {
       "type": "object",
       "properties": {
         "maxIterationSpendUsd": {
@@ -2912,7 +2977,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema82": {
+    "__schema84": {
       "type": "object",
       "properties": {
         "collect": {
@@ -2936,7 +3001,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema83": {
+    "__schema85": {
       "type": "object",
       "properties": {
         "minimumScore": {
@@ -2971,7 +3036,7 @@
       ],
       "additionalProperties": false
     },
-    "__schema84": {
+    "__schema86": {
       "anyOf": [
         {
           "type": "object",
@@ -3020,10 +3085,10 @@
               "$ref": "#/$defs/__schema12"
             },
             "kind": {
-              "$ref": "#/$defs/__schema37"
+              "$ref": "#/$defs/__schema39"
             },
             "content": {
-              "$ref": "#/$defs/__schema38"
+              "$ref": "#/$defs/__schema40"
             }
           },
           "required": [
@@ -3040,10 +3105,10 @@
               "$ref": "#/$defs/__schema12"
             },
             "kind": {
-              "$ref": "#/$defs/__schema47"
+              "$ref": "#/$defs/__schema49"
             },
             "content": {
-              "$ref": "#/$defs/__schema48"
+              "$ref": "#/$defs/__schema50"
             }
           },
           "required": [
@@ -3060,10 +3125,10 @@
               "$ref": "#/$defs/__schema12"
             },
             "kind": {
-              "$ref": "#/$defs/__schema57"
+              "$ref": "#/$defs/__schema59"
             },
             "content": {
-              "$ref": "#/$defs/__schema58"
+              "$ref": "#/$defs/__schema60"
             }
           },
           "required": [
@@ -3080,10 +3145,10 @@
               "$ref": "#/$defs/__schema12"
             },
             "kind": {
-              "$ref": "#/$defs/__schema70"
+              "$ref": "#/$defs/__schema72"
             },
             "content": {
-              "$ref": "#/$defs/__schema71"
+              "$ref": "#/$defs/__schema73"
             }
           },
           "required": [
@@ -3095,16 +3160,16 @@
         }
       ]
     },
-    "__schema85": {
+    "__schema87": {
       "type": "object",
       "propertyNames": {
         "type": "string"
       },
       "additionalProperties": {
-        "$ref": "#/$defs/__schema86"
+        "$ref": "#/$defs/__schema88"
       }
     },
-    "__schema86": {
+    "__schema88": {
       "anyOf": [
         {
           "type": "string"
@@ -3121,7 +3186,7 @@
         {
           "type": "array",
           "items": {
-            "$ref": "#/$defs/__schema86"
+            "$ref": "#/$defs/__schema88"
           }
         },
         {
@@ -3130,7 +3195,7 @@
             "type": "string"
           },
           "additionalProperties": {
-            "$ref": "#/$defs/__schema86"
+            "$ref": "#/$defs/__schema88"
           }
         }
       ]