@openpond/evals 0.6.0 → 0.7.1

package/schemas/learning/v1/package.schema.json

@@ -0,0 +1,776 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "type": "object",
+  "properties": {
+    "schemaVersion": {
+      "type": "string",
+      "const": "openpond.tasksetRelease.v2"
+    },
+    "id": {
+      "$ref": "#/$defs/__schema0"
+    },
+    "revision": {
+      "type": "integer",
+      "exclusiveMinimum": 0,
+      "maximum": 9007199254740991
+    },
+    "policy": {
+      "type": "object",
+      "properties": {
+        "policyVisibleFields": {
+          "default": [],
+          "maxItems": 1000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
+        "privilegedFields": {
+          "default": [],
+          "maxItems": 1000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
+        "hiddenGraderRefs": {
+          "default": [],
+          "maxItems": 1000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        },
+        "connectedAppScopes": {
+          "default": [],
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        }
+      },
+      "required": [
+        "policyVisibleFields",
+        "privilegedFields",
+        "hiddenGraderRefs",
+        "connectedAppScopes"
+      ],
+      "additionalProperties": false
+    },
+    "environment": {
+      "type": "object",
+      "properties": {
+        "protocolVersion": {
+          "type": "string",
+          "const": "openpond.environment.v1"
+        },
+        "kind": {
+          "type": "string",
+          "enum": [
+            "text",
+            "agent",
+            "work",
+            "custom_program"
+          ]
+        },
+        "entrypoint": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 1000
+        },
+        "stateful": {
+          "type": "boolean"
+        },
+        "deterministicSeeds": {
+          "type": "boolean"
+        },
+        "lifecycle": {
+          "minItems": 5,
+          "maxItems": 5,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema1"
+          }
+        },
+        "networkPolicy": {
+          "type": "string",
+          "enum": [
+            "none",
+            "declared_read_only",
+            "declared_scoped"
+          ]
+        },
+        "defaultTimeoutMs": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 3600000
+        }
+      },
+      "required": [
+        "protocolVersion",
+        "kind",
+        "entrypoint",
+        "stateful",
+        "deterministicSeeds",
+        "lifecycle",
+        "networkPolicy",
+        "defaultTimeoutMs"
+      ],
+      "additionalProperties": false
+    },
+    "environmentRelease": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "contentHash": {
+          "$ref": "#/$defs/__schema2"
+        }
+      },
+      "required": [
+        "id",
+        "contentHash"
+      ],
+      "additionalProperties": false
+    },
+    "tools": {
+      "maxItems": 200,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema3"
+      }
+    },
+    "capabilities": {
+      "maxItems": 200,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema4"
+      }
+    },
+    "tasks": {
+      "minItems": 1,
+      "maxItems": 1000000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema6"
+      }
+    },
+    "graders": {
+      "minItems": 1,
+      "maxItems": 1000,
+      "type": "array",
+      "items": {
+        "$ref": "#/$defs/__schema10"
+      }
+    },
+    "verifierSetRelease": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "contentHash": {
+          "$ref": "#/$defs/__schema2"
+        }
+      },
+      "required": [
+        "id",
+        "contentHash"
+      ],
+      "additionalProperties": false
+    },
+    "metadata": {
+      "$ref": "#/$defs/__schema9"
+    },
+    "contentHash": {
+      "$ref": "#/$defs/__schema2"
+    }
+  },
+  "required": [
+    "schemaVersion",
+    "id",
+    "revision",
+    "policy",
+    "environment",
+    "tools",
+    "capabilities",
+    "tasks",
+    "graders",
+    "metadata",
+    "contentHash"
+  ],
+  "additionalProperties": false,
+  "$defs": {
+    "__schema0": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 240
+    },
+    "__schema1": {
+      "type": "string",
+      "enum": [
+        "create",
+        "reset",
+        "step",
+        "collect",
+        "destroy"
+      ]
+    },
+    "__schema2": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "__schema3": {
+      "type": "object",
+      "properties": {
+        "name": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 64,
+          "pattern": "^[a-zA-Z][a-zA-Z0-9_-]*$"
+        },
+        "description": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "inputSchema": {
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        },
+        "inputSchemaHash": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "sideEffect": {
+          "type": "string",
+          "enum": [
+            "read",
+            "write"
+          ]
+        },
+        "timeoutMs": {
+          "type": "integer",
+          "exclusiveMinimum": 0,
+          "maximum": 3600000
+        }
+      },
+      "required": [
+        "name",
+        "description",
+        "inputSchema",
+        "inputSchemaHash",
+        "sideEffect",
+        "timeoutMs"
+      ],
+      "additionalProperties": false
+    },
+    "__schema4": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "required": {
+          "type": "boolean"
+        },
+        "scopes": {
+          "default": [],
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema5"
+          }
+        },
+        "portability": {
+          "type": "string",
+          "enum": [
+            "portable",
+            "host_adapter",
+            "local_only",
+            "hosted_only"
+          ]
+        }
+      },
+      "required": [
+        "id",
+        "required",
+        "scopes",
+        "portability"
+      ],
+      "additionalProperties": false
+    },
+    "__schema5": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 500
+    },
+    "__schema6": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "clusterKey": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "split": {
+          "type": "string",
+          "enum": [
+            "train",
+            "validation",
+            "test",
+            "frozen_eval"
+          ]
+        },
+        "input": {
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        },
+        "expectedOutput": {
+          "anyOf": [
+            {
+              "type": "object",
+              "propertyNames": {
+                "type": "string"
+              },
+              "additionalProperties": {}
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "policyVisibleContext": {
+          "default": {},
+          "type": "object",
+          "propertyNames": {
+            "type": "string"
+          },
+          "additionalProperties": {}
+        },
+        "privilegedContextRef": {
+          "anyOf": [
+            {
+              "$ref": "#/$defs/__schema0"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "artifactRefs": {
+          "default": [],
+          "maxItems": 1000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema7"
+          }
+        },
+        "requiredOutputs": {
+          "maxItems": 1000,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema8"
+          }
+        },
+        "tags": {
+          "default": [],
+          "maxItems": 100,
+          "type": "array",
+          "items": {
+            "$ref": "#/$defs/__schema0"
+          }
+        }
+      },
+      "required": [
+        "id",
+        "clusterKey",
+        "split",
+        "input",
+        "expectedOutput",
+        "policyVisibleContext",
+        "privilegedContextRef",
+        "artifactRefs",
+        "tags"
+      ],
+      "additionalProperties": false
+    },
+    "__schema7": {
+      "type": "object",
+      "properties": {
+        "id": {
+          "$ref": "#/$defs/__schema0"
+        },
+        "path": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "contentHash": {
+          "$ref": "#/$defs/__schema2"
+        },
+        "sizeBytes": {
+          "type": "integer",
+          "minimum": 0,
+          "maximum": 250000000
+        },
+        "mediaType": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 200
+        },
+        "visibility": {
+          "type": "string",
+          "enum": [
+            "policy",
+            "verifier",
+            "host_private"
+          ]
+        }
+      },
+      "required": [
+        "id",
+        "path",
+        "contentHash",
+        "sizeBytes",
+        "mediaType",
+        "visibility"
+      ],
+      "additionalProperties": false
+    },
+    "__schema8": {
+      "type": "object",
+      "properties": {
+        "path": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 2000
+        },
+        "mediaType": {
+          "type": "string",
+          "minLength": 1,
+          "maxLength": 200
+        },
+        "schemaRef": {
+          "default": null,
+          "anyOf": [
+            {
+              "$ref": "#/$defs/__schema7"
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "maxBytes": {
+          "default": null,
+          "anyOf": [
+            {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 250000000
+            },
+            {
+              "type": "null"
+            }
+          ]
+        },
+        "metadata": {
+          "$ref": "#/$defs/__schema9"
+        }
+      },
+      "required": [
+        "path",
+        "mediaType",
+        "schemaRef",
+        "maxBytes",
+        "metadata"
+      ],
+      "additionalProperties": false
+    },
+    "__schema9": {
+      "default": {},
+      "type": "object",
+      "propertyNames": {
+        "type": "string"
+      },
+      "additionalProperties": {}
+    },
+    "__schema10": {
+      "anyOf": [
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "version": {
+              "$ref": "#/$defs/__schema11"
+            },
+            "weight": {
+              "$ref": "#/$defs/__schema12"
+            },
+            "hardGate": {
+              "$ref": "#/$defs/__schema13"
+            },
+            "rewardEligible": {
+              "$ref": "#/$defs/__schema14"
+            },
+            "privileged": {
+              "$ref": "#/$defs/__schema15"
+            },
+            "kind": {
+              "type": "string",
+              "enum": [
+                "content",
+                "schema",
+                "artifact",
+                "runtime_event",
+                "state"
+              ]
+            },
+            "config": {
+              "type": "object",
+              "propertyNames": {
+                "type": "string"
+              },
+              "additionalProperties": {}
+            }
+          },
+          "required": [
+            "id",
+            "version",
+            "weight",
+            "hardGate",
+            "rewardEligible",
+            "privileged",
+            "kind",
+            "config"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "version": {
+              "$ref": "#/$defs/__schema11"
+            },
+            "weight": {
+              "$ref": "#/$defs/__schema12"
+            },
+            "hardGate": {
+              "$ref": "#/$defs/__schema13"
+            },
+            "rewardEligible": {
+              "$ref": "#/$defs/__schema14"
+            },
+            "privileged": {
+              "$ref": "#/$defs/__schema15"
+            },
+            "kind": {
+              "type": "string",
+              "const": "model_judge"
+            },
+            "rubricRef": {
+              "$ref": "#/$defs/__schema7"
+            },
+            "calibrationStatus": {
+              "type": "string",
+              "enum": [
+                "pending",
+                "passed",
+                "failed"
+              ]
+            },
+            "model": {
+              "type": "object",
+              "properties": {
+                "providerId": {
+                  "$ref": "#/$defs/__schema0"
+                },
+                "modelId": {
+                  "$ref": "#/$defs/__schema0"
+                },
+                "revision": {
+                  "anyOf": [
+                    {
+                      "type": "string",
+                      "minLength": 1,
+                      "maxLength": 500
+                    },
+                    {
+                      "type": "null"
+                    }
+                  ]
+                }
+              },
+              "required": [
+                "providerId",
+                "modelId",
+                "revision"
+              ],
+              "additionalProperties": false
+            },
+            "temperature": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 2
+            }
+          },
+          "required": [
+            "id",
+            "version",
+            "weight",
+            "hardGate",
+            "rewardEligible",
+            "privileged",
+            "kind",
+            "rubricRef",
+            "calibrationStatus"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "version": {
+              "$ref": "#/$defs/__schema11"
+            },
+            "weight": {
+              "$ref": "#/$defs/__schema12"
+            },
+            "hardGate": {
+              "$ref": "#/$defs/__schema13"
+            },
+            "rewardEligible": {
+              "$ref": "#/$defs/__schema14"
+            },
+            "privileged": {
+              "$ref": "#/$defs/__schema15"
+            },
+            "kind": {
+              "type": "string",
+              "const": "custom_verifier"
+            },
+            "verifierRef": {
+              "$ref": "#/$defs/__schema7"
+            },
+            "exportName": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 240
+            },
+            "timeoutMs": {
+              "type": "integer",
+              "exclusiveMinimum": 0,
+              "maximum": 300000
+            },
+            "networkPolicy": {
+              "type": "string",
+              "const": "none"
+            }
+          },
+          "required": [
+            "id",
+            "version",
+            "weight",
+            "hardGate",
+            "rewardEligible",
+            "privileged",
+            "kind",
+            "verifierRef",
+            "timeoutMs",
+            "networkPolicy"
+          ],
+          "additionalProperties": false
+        },
+        {
+          "type": "object",
+          "properties": {
+            "id": {
+              "$ref": "#/$defs/__schema0"
+            },
+            "version": {
+              "$ref": "#/$defs/__schema11"
+            },
+            "weight": {
+              "$ref": "#/$defs/__schema12"
+            },
+            "hardGate": {
+              "$ref": "#/$defs/__schema13"
+            },
+            "rewardEligible": {
+              "$ref": "#/$defs/__schema14"
+            },
+            "privileged": {
+              "$ref": "#/$defs/__schema15"
+            },
+            "kind": {
+              "type": "string",
+              "const": "human"
+            },
+            "rubricRef": {
+              "$ref": "#/$defs/__schema7"
+            },
+            "reviewerRole": {
+              "type": "string",
+              "minLength": 1,
+              "maxLength": 500
+            }
+          },
+          "required": [
+            "id",
+            "version",
+            "weight",
+            "hardGate",
+            "rewardEligible",
+            "privileged",
+            "kind",
+            "rubricRef",
+            "reviewerRole"
+          ],
+          "additionalProperties": false
+        }
+      ]
+    },
+    "__schema11": {
+      "type": "string",
+      "minLength": 1,
+      "maxLength": 100
+    },
+    "__schema12": {
+      "default": 1,
+      "type": "number",
+      "minimum": 0,
+      "maximum": 1000
+    },
+    "__schema13": {
+      "default": false,
+      "type": "boolean"
+    },
+    "__schema14": {
+      "default": false,
+      "type": "boolean"
+    },
+    "__schema15": {
+      "default": false,
+      "type": "boolean"
+    }
+  },
+  "$id": "urn:openpond:learning:v1:package",
+  "$comment": "Generated from @openpond/evals. Cross-resource identity, authorization, idempotency, admission, grading and split constraints are enforced by the shared domain service in addition to this structural schema."
+}