@openpond/evals 0.11.16 → 0.11.17

package/schemas/learning/v1/decision.schema.json

@@ -253,6 +253,7 @@
               "type": "string",
               "enum": [
                 "scored",
+                "pending",
                 "unscorable",
                 "not_configured"
               ]
@@ -294,6 +295,7 @@
               "type": "string",
               "enum": [
                 "scored",
+                "pending",
                 "unscorable",
                 "not_configured"
               ]
@@ -423,6 +425,15 @@
               "minLength": 1,
               "maxLength": 100
             },
+            "status": {
+              "type": "string",
+              "enum": [
+                "scored",
+                "pending",
+                "unavailable",
+                "failed"
+              ]
+            },
             "score": {
               "anyOf": [
                 {

package/schemas/learning/v1/draft.schema.json

@@ -183,6 +183,13 @@
             "timeout": {
               "type": "string"
             },
+            "verifierRuntime": {
+              "type": "string",
+              "enum": [
+                "isolated_javascript",
+                "sandbox_process"
+              ]
+            },
             "rubric": {
               "type": "string"
             },
@@ -201,6 +208,10 @@
             "reviewerRole": {
               "type": "string"
             },
+            "humanForm": {
+              "type": "string",
+              "maxLength": 131072
+            },
             "learnedId": {
               "type": "string"
             },

package/schemas/learning/v1/grade.schema.json

@@ -79,6 +79,7 @@
                   "type": "string",
                   "enum": [
                     "scored",
+                    "pending",
                     "unscorable",
                     "not_configured"
                   ]
@@ -120,6 +121,7 @@
                   "type": "string",
                   "enum": [
                     "scored",
+                    "pending",
                     "unscorable",
                     "not_configured"
                   ]
@@ -400,6 +402,15 @@
               "minLength": 1,
               "maxLength": 100
             },
+            "status": {
+              "type": "string",
+              "enum": [
+                "scored",
+                "pending",
+                "unavailable",
+                "failed"
+              ]
+            },
             "score": {
               "anyOf": [
                 {

package/schemas/learning/v1/package.schema.json

@@ -185,22 +185,22 @@
       "type": "object",
       "properties": {
         "schemaVersion": {
-          "$ref": "#/$defs/__schema16"
+          "$ref": "#/$defs/__schema45"
         },
         "primaryMetric": {
-          "$ref": "#/$defs/__schema17"
+          "$ref": "#/$defs/__schema46"
         },
         "aggregation": {
-          "$ref": "#/$defs/__schema18"
+          "$ref": "#/$defs/__schema47"
         },
         "missingReward": {
-          "$ref": "#/$defs/__schema19"
+          "$ref": "#/$defs/__schema48"
         },
         "taskWeights": {
-          "$ref": "#/$defs/__schema20"
+          "$ref": "#/$defs/__schema49"
         },
         "customAggregator": {
-          "$ref": "#/$defs/__schema21"
+          "$ref": "#/$defs/__schema50"
         }
       },
       "required": [
@@ -768,6 +768,30 @@
               "type": "string",
               "minLength": 1,
               "maxLength": 500
+            },
+            "form": {
+              "type": "object",
+              "properties": {
+                "schemaVersion": {
+                  "$ref": "#/$defs/__schema16"
+                },
+                "mode": {
+                  "$ref": "#/$defs/__schema17"
+                },
+                "instructions": {
+                  "$ref": "#/$defs/__schema18"
+                },
+                "criteria": {
+                  "$ref": "#/$defs/__schema19"
+                }
+              },
+              "required": [
+                "schemaVersion",
+                "mode",
+                "instructions",
+                "criteria"
+              ],
+              "additionalProperties": false
             }
           },
           "required": [
@@ -810,14 +834,510 @@
     },
     "__schema16": {
       "type": "string",
-      "const": "openpond.tasksetMetricPolicy.v1"
+      "const": "openpond.humanForm.v1"
     },
     "__schema17": {
+      "type": "string",
+      "enum": [
+        "individual",
+        "pairwise"
+      ]
+    },
+    "__schema18": {
+      "type": "string",
+      "maxLength": 20000
+    },
+    "__schema19": {
+      "minItems": 1,
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema20"
+      }
+    },
+    "__schema20": {
+      "oneOf": [
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "label": {
+              "$ref": "#/$defs/__schema21"
+            },
+            "instructions": {
+              "$ref": "#/$defs/__schema22"
+            },
+            "required": {
+              "$ref": "#/$defs/__schema23"
+            },
+            "allowAbstain": {
+              "$ref": "#/$defs/__schema24"
+            },
+            "requireRationale": {
+              "$ref": "#/$defs/__schema25"
+            },
+            "anchors": {
+              "$ref": "#/$defs/__schema26"
+            },
+            "kind": {
+              "$ref": "#/$defs/__schema28"
+            },
+            "minimum": {
+              "$ref": "#/$defs/__schema29"
+            },
+            "maximum": {
+              "$ref": "#/$defs/__schema30"
+            },
+            "step": {
+              "$ref": "#/$defs/__schema31"
+            }
+          },
+          "required": [
+            "id",
+            "label",
+            "instructions",
+            "required",
+            "allowAbstain",
+            "kind",
+            "minimum",
+            "maximum",
+            "step"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "label": {
+              "$ref": "#/$defs/__schema32"
+            },
+            "instructions": {
+              "$ref": "#/$defs/__schema33"
+            },
+            "required": {
+              "$ref": "#/$defs/__schema34"
+            },
+            "allowAbstain": {
+              "$ref": "#/$defs/__schema35"
+            },
+            "requireRationale": {
+              "$ref": "#/$defs/__schema36"
+            },
+            "anchors": {
+              "$ref": "#/$defs/__schema37"
+            },
+            "kind": {
+              "$ref": "#/$defs/__schema39"
+            },
+            "options": {
+              "$ref": "#/$defs/__schema40"
+            }
+          },
+          "required": [
+            "id",
+            "label",
+            "instructions",
+            "required",
+            "allowAbstain",
+            "kind",
+            "options"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "label": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 500
+            },
+            "instructions": {
+              "type": "string",
+              "maxLength": 20000
+            },
+            "required": {
+              "type": "boolean"
+            },
+            "allowAbstain": {
+              "type": "boolean"
+            },
+            "requireRationale": {
+              "type": "boolean"
+            },
+            "anchors": {
+              "maxItems": 100,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema42"
+              }
+            },
+            "kind": {
+              "type": "string",
+              "const": "boolean"
+            }
+          },
+          "required": [
+            "id",
+            "label",
+            "instructions",
+            "required",
+            "allowAbstain",
+            "kind"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "label": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 500
+            },
+            "instructions": {
+              "type": "string",
+              "maxLength": 20000
+            },
+            "required": {
+              "type": "boolean"
+            },
+            "allowAbstain": {
+              "type": "boolean"
+            },
+            "requireRationale": {
+              "type": "boolean"
+            },
+            "anchors": {
+              "maxItems": 100,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema43"
+              }
+            },
+            "kind": {
+              "type": "string",
+              "const": "text"
+            },
+            "minimumLength": {
+              "type": "integer",
+              "minimum": 0,
+              "maximum": 20000
+            },
+            "maximumLength": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 20000
+            }
+          },
+          "required": [
+            "id",
+            "label",
+            "instructions",
+            "required",
+            "allowAbstain",
+            "kind",
+            "maximumLength"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "label": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 500
+            },
+            "instructions": {
+              "type": "string",
+              "maxLength": 20000
+            },
+            "required": {
+              "type": "boolean"
+            },
+            "allowAbstain": {
+              "type": "boolean"
+            },
+            "requireRationale": {
+              "type": "boolean"
+            },
+            "anchors": {
+              "maxItems": 100,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema44"
+              }
+            },
+            "kind": {
+              "type": "string",
+              "const": "preference"
+            },
+            "requireStrength": {
+              "type": "boolean"
+            },
+            "allowTie": {
+              "type": "boolean"
+            }
+          },
+          "required": [
+            "id",
+            "label",
+            "instructions",
+            "required",
+            "allowAbstain",
+            "kind",
+            "allowTie"
+          ],
+          "additionalProperties": false
+        }
+      ]
+    },
+    "__schema21": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 500
+    },
+    "__schema22": {
+      "type": "string",
+      "maxLength": 20000
+    },
+    "__schema23": {
+      "type": "boolean"
+    },
+    "__schema24": {
+      "type": "boolean"
+    },
+    "__schema25": {
+      "type": "boolean"
+    },
+    "__schema26": {
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema27"
+      }
+    },
+    "__schema27": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        }
+      },
+      "required": [
+        "value",
+        "description"
+      ],
+      "additionalProperties": false
+    },
+    "__schema28": {
+      "type": "string",
+      "const": "score"
+    },
+    "__schema29": {
+      "type": "number"
+    },
+    "__schema30": {
+      "type": "number"
+    },
+    "__schema31": {
+      "type": "number",
+      "exclusiveMinimum": 0
+    },
+    "__schema32": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 500
+    },
+    "__schema33": {
+      "type": "string",
+      "maxLength": 20000
+    },
+    "__schema34": {
+      "type": "boolean"
+    },
+    "__schema35": {
+      "type": "boolean"
+    },
+    "__schema36": {
+      "type": "boolean"
+    },
+    "__schema37": {
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema38"
+      }
+    },
+    "__schema38": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        }
+      },
+      "required": [
+        "value",
+        "description"
+      ],
+      "additionalProperties": false
+    },
+    "__schema39": {
+      "type": "string",
+      "const": "category"
+    },
+    "__schema40": {
+      "minItems": 2,
+      "maxItems": 100,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema41"
+      }
+    },
+    "__schema41": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 500
+    },
+    "__schema42": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        }
+      },
+      "required": [
+        "value",
+        "description"
+      ],
+      "additionalProperties": false
+    },
+    "__schema43": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        }
+      },
+      "required": [
+        "value",
+        "description"
+      ],
+      "additionalProperties": false
+    },
+    "__schema44": {
+      "type": "object",
+      "properties": {
+        "value": {
+          "anyOf": [
+            {
+              "type": "number"
+            },
+            {
+              "type": "string"
+            },
+            {
+              "type": "boolean"
+            }
+          ]
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        }
+      },
+      "required": [
+        "value",
+        "description"
+      ],
+      "additionalProperties": false
+    },
+    "__schema45": {
+      "type": "string",
+      "const": "openpond.tasksetMetricPolicy.v1"
+    },
+    "__schema46": {
       "type": "string",
       "minLength": 1,
       "maxLength": 240
     },
-    "__schema18": {
+    "__schema47": {
       "type": "string",
       "enum": [
         "mean_score",
@@ -826,17 +1346,17 @@
         "custom"
       ]
     },
-    "__schema19": {
+    "__schema48": {
       "type": "string",
       "enum": [
         "zero",
         "exclude"
       ]
     },
-    "__schema20": {
+    "__schema49": {
       "type": "object",
       "propertyNames": {
-        "$ref": "#/$defs/__schema17"
+        "$ref": "#/$defs/__schema46"
       },
       "additionalProperties": {
         "type": "number",
@@ -844,7 +1364,7 @@
         "maximum": 1000000
       }
     },
-    "__schema21": {
+    "__schema50": {
       "anyOf": [
         {
           "type": "object",