@openpond/evals 0.7.6 → 0.8.0

package/schemas/learning/v1/draft.schema.json

@@ -211,6 +211,13 @@
             },
             "maximum": {
               "type": "string"
+            },
+            "fixtures": {
+              "maxItems": 50,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema12"
+              }
             }
           },
           "required": [
@@ -555,6 +562,95 @@
       "type": "string",
       "format": "date-time",
       "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+    },
+    "__schema12": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "name": {
+          "type": "string",
+          "maxLength": 500
+        },
+        "input": {
+          "type": "string"
+        },
+        "output": {
+          "type": "string"
+        },
+        "expectedOutput": {
+          "type": "string"
+        },
+        "evaluatorContext": {
+          "type": "string"
+        },
+        "artifactRefs": {
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema13"
+          }
+        },
+        "runtimeEventRefs": {
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema14"
+          }
+        },
+        "infrastructureError": {
+          "type": "string",
+          "maxLength": 20000
+        },
+        "expectedStatus": {
+          "type": "string",
+          "enum": [
+            "scored",
+            "pending",
+            "unavailable",
+            "failed"
+          ]
+        },
+        "minimumScore": {
+          "type": "string"
+        },
+        "maximumScore": {
+          "type": "string"
+        },
+        "expectedPassed": {
+          "type": "string",
+          "enum": [
+            "true",
+            "false",
+            "any"
+          ]
+        }
+      },
+      "required": [
+        "id",
+        "name",
+        "input",
+        "output",
+        "expectedOutput",
+        "evaluatorContext",
+        "artifactRefs",
+        "runtimeEventRefs",
+        "infrastructureError",
+        "expectedStatus",
+        "minimumScore",
+        "maximumScore",
+        "expectedPassed"
+      ],
+      "additionalProperties": false
+    },
+    "__schema13": {
+      "type": "string",
+      "maxLength": 500
+    },
+    "__schema14": {
+      "type": "string",
+      "maxLength": 500
     }
   },
   "$id": "urn:openpond:learning:v1:draft",

package/schemas/learning/v1/read-request.schema.json

@@ -122,6 +122,7 @@
       "type": "string",
       "enum": [
         "draft",
+        "reward_check",
         "asset",
         "definition",
         "reward",

package/schemas/learning/v1/reward.schema.json

@@ -227,6 +227,9 @@
         "$ref": "#/$defs/__schema1"
       }
     },
+    "fixtureSetRef": {
+      "$ref": "#/$defs/__schema1"
+    },
     "contentHash": {
       "$ref": "#/$defs/__schema2"
     }

package/schemas/learning/v1/reward_check.schema.json

@@ -0,0 +1,343 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.rewardCheckRun.v1"
+    },
+    "id": {
+      "$ref": "#/$defs/__schema0"
+    },
+    "revision": {
+      "type": "integer",
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991
+    },
+    "draft": {
+      "$ref": "#/$defs/__schema1"
+    },
+    "reward": {
+      "$ref": "#/$defs/__schema1"
+    },
+    "snapshotHash": {
+      "$ref": "#/$defs/__schema2"
+    },
+    "fixtureRefs": {
+      "minItems": 1,
+      "maxItems": 50,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema3"
+      }
+    },
+    "status": {
+      "type": "string",
+      "enum": [
+        "queued",
+        "running",
+        "completed",
+        "failed",
+        "cancelling",
+        "cancelled"
+      ]
+    },
+    "runtime": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "packageVersion": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 100
+            },
+            "engine": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 200
+            }
+          },
+          "required": [
+            "id",
+            "packageVersion",
+            "engine"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "results": {
+      "maxItems": 50,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema4"
+      }
+    },
+    "matchesExpectations": {
+      "anyOf": [
+        {
+          "type": "boolean"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "failure": {
+      "anyOf": [
+        {
+          "type": "string",
+          "maxLength": 20000
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "timeoutMs": {
+      "type": "integer",
+      "minimum": 100,
+      "maximum": 300000
+    },
+    "maximumSpendUsd": {
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1000
+    },
+    "leaseOwner": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/__schema0"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "leaseExpiresAt": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/__schema5"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "attemptCount": {
+      "type": "integer",
+      "minimum": 0,
+      "maximum": 9007199254740991
+    },
+    "createdAt": {
+      "$ref": "#/$defs/__schema5"
+    },
+    "updatedAt": {
+      "$ref": "#/$defs/__schema5"
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "revision",
+    "draft",
+    "reward",
+    "snapshotHash",
+    "fixtureRefs",
+    "status",
+    "runtime",
+    "results",
+    "matchesExpectations",
+    "failure",
+    "timeoutMs",
+    "maximumSpendUsd",
+    "leaseOwner",
+    "leaseExpiresAt",
+    "attemptCount",
+    "createdAt",
+    "updatedAt"
+  ],
+  "additionalProperties": false,
+  "$defs": {
+    "__schema0": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "__schema1": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "contentHash": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "revision": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 9007199254740991
+        }
+      },
+      "required": [
+        "id",
+        "contentHash",
+        "revision"
+      ],
+      "additionalProperties": false
+    },
+    "__schema2": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "__schema3": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "contentHash": {
+          "$ref": "#/$defs/__schema2"
+        }
+      },
+      "required": [
+        "id",
+        "contentHash"
+      ],
+      "additionalProperties": false
+    },
+    "__schema4": {
+      "type": "object",
+      "properties": {
+        "fixture": {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "contentHash": {
+              "$ref": "#/$defs/__schema2"
+            }
+          },
+          "required": [
+            "id",
+            "contentHash"
+          ],
+          "additionalProperties": false
+        },
+        "result": {
+          "type": "object",
+          "properties": {
+            "graderId": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "reward": {
+              "$ref": "#/$defs/__schema1"
+            },
+            "role": {
+              "type": "string",
+              "enum": [
+                "training",
+                "evaluation"
+              ]
+            },
+            "status": {
+              "type": "string",
+              "enum": [
+                "scored",
+                "pending",
+                "unavailable",
+                "failed"
+              ]
+            },
+            "rawScore": {
+              "anyOf": [
+                {
+                  "type": "number"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "normalizedScore": {
+              "anyOf": [
+                {
+                  "type": "number",
+                  "minimum": 0,
+                  "maximum": 1
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "passed": {
+              "anyOf": [
+                {
+                  "type": "boolean"
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "evidenceHashes": {
+              "maxItems": 1000,
+              "type": "array",
+              "items": {
+                "$ref": "#/$defs/__schema2"
+              }
+            },
+            "message": {
+              "anyOf": [
+                {
+                  "type": "string",
+                  "maxLength": 20000
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "graderId",
+            "reward",
+            "role",
+            "status",
+            "rawScore",
+            "normalizedScore",
+            "passed",
+            "evidenceHashes",
+            "message"
+          ],
+          "additionalProperties": false
+        },
+        "matchesExpectation": {
+          "type": "boolean"
+        }
+      },
+      "required": [
+        "fixture",
+        "result",
+        "matchesExpectation"
+      ],
+      "additionalProperties": false
+    },
+    "__schema5": {
+      "type": "string",
+      "format": "date-time",
+      "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+    }
+  },
+  "$id": "urn:openpond:learning:v1:reward_check",
+  "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
+}