@zanii/certify 0.0.0-stage → 0.6.0

package/dist/schemas/suite.schema.json

@@ -0,0 +1,138 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://zanii.agency/certify/schemas/suite.schema.json",
+    "title": "Zanii Certify test suite (spec/suites.md)",
+    "type": "object",
+    "required": ["v", "type", "id", "version", "visibility", "cases"],
+    "additionalProperties": false,
+    "properties": {
+        "v": { "const": 1 },
+        "type": { "const": "certify.suite" },
+        "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" },
+        "version": { "type": "integer", "minimum": 1 },
+        "visibility": { "enum": ["public", "held-out"] },
+        "profiles": { "$ref": "#/$defs/profileRefs" },
+        "rubrics": { "type": "array", "minItems": 1, "maxItems": 64, "items": { "$ref": "#/$defs/rubric" } },
+        "cases": { "type": "array", "minItems": 1, "items": { "$ref": "#/$defs/case" } }
+    },
+    "$defs": {
+        "profileRefs": { "type": "array", "minItems": 1, "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "maxLength": 100, "pattern": "^[a-z0-9][a-z0-9-]{0,63}/[1-9][0-9]{0,8}$" } },
+        "teamPolicy": { "type": "object", "required": ["members", "require_distinct_owners"], "additionalProperties": false, "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 64, "uniqueItems": true, "items": { "type": "object", "required": ["agent", "owner"], "additionalProperties": false, "properties": { "agent": { "anyOf": [{ "const": "$agent" }, { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" }] }, "owner": { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" } } } }, "require_distinct_owners": { "type": "boolean" }, "bind_run": { "type": "boolean" } } },
+        "scope": { "type": "string", "maxLength": 128, "pattern": "^(\\*|[a-z0-9_-]+(\\.[a-z0-9_-]+)*(\\.\\*)?)$" },
+        "arg": { "type": "string", "pattern": "^[A-Za-z0-9_]{1,64}$" },
+        "cert": { "type": "object", "required": ["v", "issuer", "subject", "scopes", "exp", "sig"], "additionalProperties": false, "properties": { "v": { "const": 1 }, "issuer": { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" }, "subject": { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" }, "scopes": { "type": "array", "minItems": 1, "maxItems": 64, "items": { "$ref": "#/$defs/scope" } }, "exp": { "type": "string", "minLength": 1, "maxLength": 64 }, "sig": { "type": "string", "pattern": "^ed25519:[0-9a-f]{128}$" } } },
+        "revocation": { "type": "object", "required": ["v", "type", "cert", "ts", "sig"], "additionalProperties": false, "properties": { "v": { "const": 1 }, "type": { "const": "revocation" }, "cert": { "$ref": "#/$defs/cert" }, "ts": { "type": "string", "minLength": 1, "maxLength": 64 }, "sig": { "type": "string", "pattern": "^ed25519:[0-9a-f]{128}$" } } },
+        "link": { "type": "object", "required": ["agent", "scopes", "spend_limit"], "additionalProperties": false, "properties": { "agent": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" }, "scopes": { "type": "array", "minItems": 1, "maxItems": 64, "uniqueItems": true, "items": { "$ref": "#/$defs/scope" } }, "spend_limit": { "type": "object", "required": ["amount_minor", "currency"], "additionalProperties": false, "properties": { "amount_minor": { "type": "integer", "minimum": 0, "maximum": 9007199254740991 }, "currency": { "type": "string", "pattern": "^[A-Z]{3}$" } } } } },
+        "text": {
+            "type": "object",
+            "required": ["en", "ar"],
+            "additionalProperties": false,
+            "properties": {
+                "en": { "type": "string", "minLength": 1, "maxLength": 4000 },
+                "ar": { "type": "string", "minLength": 1, "maxLength": 4000 }
+            }
+        },
+        "rubric": {
+            "type": "object",
+            "required": ["id", "title", "pass", "fail"],
+            "additionalProperties": false,
+            "properties": {
+                "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" },
+                "title": { "$ref": "#/$defs/text" },
+                "pass": { "$ref": "#/$defs/text" },
+                "fail": { "$ref": "#/$defs/text" }
+            }
+        },
+        "case": {
+            "type": "object",
+            "required": ["id", "category", "lang", "turns", "check"],
+            "additionalProperties": false,
+            "properties": {
+                "id": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,95}$" },
+                "category": { "type": "string", "pattern": "^[a-z][a-z0-9_]{0,63}$" },
+                "profiles": { "$ref": "#/$defs/profileRefs" },
+                "lang": { "enum": ["en", "ar", "ar-gulf"] },
+                "turns": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 20,
+                    "items": {
+                        "type": "object",
+                        "required": ["role", "content"],
+                        "additionalProperties": false,
+                        "properties": {
+                            "role": { "enum": ["user", "assistant"] },
+                            "content": { "type": "string", "minLength": 1, "maxLength": 20000 }
+                        }
+                    }
+                },
+                "documents": {
+                    "type": "array",
+                    "maxItems": 10,
+                    "items": {
+                        "type": "object",
+                        "required": ["name", "text"],
+                        "additionalProperties": false,
+                        "properties": {
+                            "name": { "type": "string", "minLength": 1, "maxLength": 200 },
+                            "text": { "type": "string", "minLength": 1, "maxLength": 100000 }
+                        }
+                    }
+                },
+                "tools": {
+                    "type": "array",
+                    "maxItems": 64,
+                    "items": {
+                        "type": "object",
+                        "required": ["type", "function"],
+                        "additionalProperties": false,
+                        "properties": {
+                            "type": { "const": "function" },
+                            "function": {
+                                "type": "object",
+                                "required": ["name"],
+                                "properties": {
+                                    "name": { "type": "string", "pattern": "^[A-Za-z0-9_-]{1,64}$" },
+                                    "description": { "type": "string", "maxLength": 2000 },
+                                    "parameters": { "type": "object" }
+                                }
+                            }
+                        }
+                    }
+                },
+                "delegation": { "type": "object", "required": ["chain", "tools"], "additionalProperties": false, "properties": { "chain": { "type": "array", "minItems": 1, "maxItems": 8, "items": { "$ref": "#/$defs/link" } }, "tools": { "type": "array", "maxItems": 64, "items": { "type": "object", "required": ["name", "scope"], "additionalProperties": false, "properties": { "name": { "type": "string", "pattern": "^[A-Za-z0-9_-]{1,64}$" }, "scope": { "$ref": "#/$defs/scope" }, "amount": { "$ref": "#/$defs/arg" }, "currency": { "$ref": "#/$defs/arg" } } } }, "certs": { "type": "array", "minItems": 1, "maxItems": 8, "items": { "$ref": "#/$defs/cert" } }, "revocations": { "type": "array", "minItems": 1, "maxItems": 64, "items": { "$ref": "#/$defs/revocation" } } } },
+                "sub_agent": { "type": "object", "required": ["agent", "tool", "scopes"], "additionalProperties": false, "properties": { "agent": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" }, "tool": { "type": "string", "pattern": "^[A-Za-z0-9_-]{1,64}$" }, "scopes": { "$ref": "#/$defs/arg" }, "amount": { "$ref": "#/$defs/arg" } } },
+                "check": { "$ref": "#/$defs/check" }
+            }
+        },
+        "check": {
+            "type": "object",
+            "required": ["kind"],
+            "additionalProperties": false,
+            "properties": {
+                "kind": { "enum": ["not_contains", "contains_any", "tool_not_called", "max_tool_calls", "all", "judged", "within_delegation", "no_scope_escalation", "team_receipt"] },
+                "values": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "items": { "type": "string", "minLength": 1, "maxLength": 2000 }
+                },
+                "tools": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "uniqueItems": true,
+                    "items": { "type": "string", "pattern": "^[A-Za-z0-9_-]{1,64}$" }
+                },
+                "n": { "type": "integer", "minimum": 0, "maximum": 1000 },
+                "checks": { "type": "array", "minItems": 1, "maxItems": 16, "items": { "$ref": "#/$defs/check" } },
+                "rubric": { "type": "string", "pattern": "^[a-z0-9][a-z0-9-]{0,63}$" },
+                "tool": { "type": "string", "pattern": "^[A-Za-z0-9_-]{1,64}$" },
+                "threshold": { "type": "integer", "minimum": 1, "maximum": 64 },
+                "invocation": { "enum": ["required", "optional"] },
+                "policy": { "$ref": "#/$defs/teamPolicy" },
+                "finding": { "type": "string", "pattern": "^[A-Z][A-Z0-9_]{0,63}$" }
+            }
+        }
+    }
+}