@openpond/evals 0.10.9 → 0.11.0

package/schemas/learning/v1/reward_check.schema.json

@@ -111,6 +111,16 @@
       "minimum": 0,
       "maximum": 1000
     },
+    "judgeCalls": {
+      "maxItems": 1000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema6"
+      }
+    },
+    "requestedBy": {
+      "$ref": "#/$defs/__schema0"
+    },
     "leaseOwner": {
       "anyOf": [
         {
@@ -124,7 +134,7 @@
     "leaseExpiresAt": {
       "anyOf": [
         {
-          "$ref": "#/$defs/__schema5"
+          "$ref": "#/$defs/__schema10"
         },
         {
           "type": "null"
@@ -137,10 +147,10 @@
       "maximum": 9007199254740991
     },
     "createdAt": {
-      "$ref": "#/$defs/__schema5"
+      "$ref": "#/$defs/__schema10"
     },
     "updatedAt": {
-      "$ref": "#/$defs/__schema5"
+      "$ref": "#/$defs/__schema10"
     }
   },
   "required": [
@@ -296,6 +306,194 @@
                 "$ref": "#/$defs/__schema2"
               }
             },
+            "graderEvidence": {
+              "type": "object",
+              "properties": {
+                "schemaVersion": {
+                  "type": "string",
+                  "const": "openpond.graderEvidence.v1"
+                },
+                "graderId": {
+                  "$ref": "#/$defs/__schema0"
+                },
+                "graderVersion": {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 100
+                },
+                "score": {
+                  "anyOf": [
+                    {
+                      "type": "number",
+                      "minimum": 0,
+                      "maximum": 1
+                    },
+                    {
+                      "type": "null"
+                    }
+                  ]
+                },
+                "passed": {
+                  "type": "boolean"
+                },
+                "rewardEligible": {
+                  "type": "boolean"
+                },
+                "failureClass": {
+                  "anyOf": [
+                    {
+                      "type": "string",
+                      "enum": [
+                        "policy_failure",
+                        "grader_failure",
+                        "environment_failure",
+                        "infrastructure_failure",
+                        "timeout",
+                        "cancelled"
+                      ]
+                    },
+                    {
+                      "type": "null"
+                    }
+                  ]
+                },
+                "feedback": {
+                  "maxItems": 1000,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema5"
+                  }
+                },
+                "visibleEvidenceRefs": {
+                  "maxItems": 10000,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema0"
+                  }
+                },
+                "privilegedEvidenceRefs": {
+                  "maxItems": 10000,
+                  "type": "array",
+                  "items": {
+                    "$ref": "#/$defs/__schema0"
+                  }
+                },
+                "modelJudgeReceipt": {
+                  "type": "object",
+                  "properties": {
+                    "schemaVersion": {
+                      "type": "string",
+                      "const": "openpond.modelJudgeReceipt.v1"
+                    },
+                    "providerId": {
+                      "type": "string",
+                      "minLength": 1,
+                      "maxLength": 200
+                    },
+                    "modelId": {
+                      "type": "string",
+                      "minLength": 1,
+                      "maxLength": 500
+                    },
+                    "modelRevision": {
+                      "anyOf": [
+                        {
+                          "type": "string",
+                          "minLength": 1,
+                          "maxLength": 500
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ]
+                    },
+                    "responseId": {
+                      "anyOf": [
+                        {
+                          "type": "string",
+                          "minLength": 1,
+                          "maxLength": 500
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ]
+                    },
+                    "requestHash": {
+                      "$ref": "#/$defs/__schema2"
+                    },
+                    "responseHash": {
+                      "$ref": "#/$defs/__schema2"
+                    },
+                    "inputTokens": {
+                      "anyOf": [
+                        {
+                          "type": "integer",
+                          "minimum": 0,
+                          "maximum": 9007199254740991
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ]
+                    },
+                    "outputTokens": {
+                      "anyOf": [
+                        {
+                          "type": "integer",
+                          "minimum": 0,
+                          "maximum": 9007199254740991
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ]
+                    },
+                    "costUsd": {
+                      "anyOf": [
+                        {
+                          "type": "number",
+                          "minimum": 0
+                        },
+                        {
+                          "type": "null"
+                        }
+                      ]
+                    }
+                  },
+                  "required": [
+                    "schemaVersion",
+                    "providerId",
+                    "modelId",
+                    "modelRevision",
+                    "responseId",
+                    "requestHash",
+                    "responseHash",
+                    "inputTokens",
+                    "outputTokens",
+                    "costUsd"
+                  ],
+                  "additionalProperties": false
+                },
+                "contentHash": {
+                  "$ref": "#/$defs/__schema2"
+                }
+              },
+              "required": [
+                "schemaVersion",
+                "graderId",
+                "graderVersion",
+                "score",
+                "passed",
+                "rewardEligible",
+                "failureClass",
+                "feedback",
+                "visibleEvidenceRefs",
+                "privilegedEvidenceRefs",
+                "contentHash"
+              ],
+              "additionalProperties": false
+            },
             "message": {
               "anyOf": [
                 {
@@ -333,6 +531,140 @@
       "additionalProperties": false
     },
     "__schema5": {
+      "type": "string",
+      "maxLength": 20000
+    },
+    "__schema6": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "requestHash": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "reservedUsd": {
+          "$ref": "#/$defs/__schema7"
+        },
+        "status": {
+          "$ref": "#/$defs/__schema8"
+        },
+        "response": {
+          "$ref": "#/$defs/__schema9"
+        }
+      },
+      "required": [
+        "id",
+        "requestHash",
+        "reservedUsd",
+        "status",
+        "response"
+      ],
+      "additionalProperties": false
+    },
+    "__schema7": {
+      "type": "number",
+      "exclusiveMinimum": 0
+    },
+    "__schema8": {
+      "type": "string",
+      "enum": [
+        "reserved",
+        "settled",
+        "not_dispatched"
+      ]
+    },
+    "__schema9": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "text": {
+              "type": "string",
+              "maxLength": 100000
+            },
+            "modelId": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 500
+            },
+            "modelRevision": {
+              "anyOf": [
+                {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 500
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "responseId": {
+              "anyOf": [
+                {
+                  "type": "string",
+                  "minLength": 1,
+                  "maxLength": 500
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "inputTokens": {
+              "anyOf": [
+                {
+                  "type": "integer",
+                  "minimum": 0,
+                  "maximum": 9007199254740991
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "outputTokens": {
+              "anyOf": [
+                {
+                  "type": "integer",
+                  "minimum": 0,
+                  "maximum": 9007199254740991
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            },
+            "costUsd": {
+              "anyOf": [
+                {
+                  "type": "number",
+                  "minimum": 0
+                },
+                {
+                  "type": "null"
+                }
+              ]
+            }
+          },
+          "required": [
+            "text",
+            "modelId",
+            "modelRevision",
+            "responseId",
+            "inputTokens",
+            "outputTokens",
+            "costUsd"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "__schema10": {
       "type": "string",
       "format": "date-time",
       "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"

package/schemas/learning/v1/schedule.schema.json

@@ -26,26 +26,29 @@
     "intervalSeconds": {
       "$ref": "#/$defs/__schema6"
     },
-    "nextRunAt": {
+    "calendar": {
       "$ref": "#/$defs/__schema7"
     },
+    "nextRunAt": {
+      "$ref": "#/$defs/__schema8"
+    },
     "lastFire": {
-      "$ref": "#/$defs/__schema9"
+      "$ref": "#/$defs/__schema10"
     },
     "consecutiveFailures": {
-      "$ref": "#/$defs/__schema10"
+      "$ref": "#/$defs/__schema11"
     },
     "nextAttemptAt": {
-      "$ref": "#/$defs/__schema11"
+      "$ref": "#/$defs/__schema12"
     },
     "lastError": {
-      "$ref": "#/$defs/__schema12"
+      "$ref": "#/$defs/__schema13"
     },
     "createdAt": {
-      "$ref": "#/$defs/__schema8"
+      "$ref": "#/$defs/__schema9"
     },
     "updatedAt": {
-      "$ref": "#/$defs/__schema8"
+      "$ref": "#/$defs/__schema9"
     }
   },
   "required": [
@@ -57,6 +60,7 @@
     "executionOwner",
     "state",
     "intervalSeconds",
+    "calendar",
     "nextRunAt",
     "lastFire",
     "consecutiveFailures",
@@ -132,9 +136,26 @@
       ]
     },
     "__schema7": {
+      "default": null,
       "anyOf": [
         {
-          "$ref": "#/$defs/__schema8"
+          "type": "object",
+          "properties": {
+            "localTime": {
+              "type": "string",
+              "pattern": "^(?:[01]\\d|2[0-3]):[0-5]\\d$"
+            },
+            "timeZone": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 200
+            }
+          },
+          "required": [
+            "localTime",
+            "timeZone"
+          ],
+          "additionalProperties": false
         },
         {
           "type": "null"
@@ -142,11 +163,21 @@
       ]
     },
     "__schema8": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/__schema9"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "__schema9": {
       "type": "string",
       "format": "date-time",
       "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
     },
-    "__schema9": {
+    "__schema10": {
       "anyOf": [
         {
           "$ref": "#/$defs/__schema3"
@@ -156,22 +187,22 @@
         }
       ]
     },
-    "__schema10": {
+    "__schema11": {
       "type": "integer",
       "minimum": 0,
       "maximum": 9007199254740991
     },
-    "__schema11": {
+    "__schema12": {
       "anyOf": [
         {
-          "$ref": "#/$defs/__schema8"
+          "$ref": "#/$defs/__schema9"
         },
         {
           "type": "null"
         }
       ]
     },
-    "__schema12": {
+    "__schema13": {
       "anyOf": [
         {
           "type": "string",

package/schemas/learning/v1/task-queue-inspection.schema.json

@@ -0,0 +1,72 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "modelProjectId": {
+      "anyOf": [
+        {
+          "$ref": "#/$defs/__schema0"
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "inspectedAt": {
+      "type": "string",
+      "format": "date-time",
+      "pattern": "^(?:(?:\\d\\d[2468][048]|\\d\\d[13579][26]|\\d\\d0[48]|[02468][048]00|[13579][26]00)-02-29|\\d{4}-(?:(?:0[13578]|1[02])-(?:0[1-9]|[12]\\d|3[01])|(?:0[469]|11)-(?:0[1-9]|[12]\\d|30)|(?:02)-(?:0[1-9]|1\\d|2[0-8])))T(?:(?:[01]\\d|2[0-3]):[0-5]\\d(?::[0-5]\\d(?:\\.\\d+)?)?(?:Z|([+-](?:[01]\\d|2[0-3]):[0-5]\\d)))$"
+    },
+    "pendingTrainingCount": {
+      "anyOf": [
+        {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 9007199254740991
+        },
+        {
+          "type": "null"
+        }
+      ]
+    },
+    "issues": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "properties": {
+          "policyId": {
+            "$ref": "#/$defs/__schema0"
+          },
+          "code": {
+            "type": "string"
+          },
+          "message": {
+            "type": "string"
+          }
+        },
+        "required": [
+          "policyId",
+          "code",
+          "message"
+        ],
+        "additionalProperties": false
+      }
+    }
+  },
+  "required": [
+    "modelProjectId",
+    "inspectedAt",
+    "pendingTrainingCount",
+    "issues"
+  ],
+  "additionalProperties": false,
+  "$defs": {
+    "__schema0": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    }
+  },
+  "$id": "urn:openpond:learning:v1:task-queue-inspection",
+  "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
+}