@zanii/certify 0.0.0-stage → 0.5.0

package/dist/schemas/scenarios.schema.json

@@ -0,0 +1,573 @@
+{
+    "$schema": "https://json-schema.org/draft/2020-12/schema",
+    "$id": "https://zanii.agency/certify/schemas/scenarios.schema.json",
+    "title": "Zanii Certify scenario library (spec/scenarios.md)",
+    "type": "object",
+    "required": [
+        "v",
+        "type",
+        "id",
+        "version",
+        "visibility",
+        "scenarios"
+    ],
+    "additionalProperties": false,
+    "properties": {
+        "v": {
+            "const": 1
+        },
+        "type": {
+            "const": "certify.scenarios"
+        },
+        "id": {
+            "type": "string",
+            "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"
+        },
+        "version": {
+            "type": "integer",
+            "minimum": 1
+        },
+        "visibility": {
+            "enum": [
+                "public",
+                "held-out"
+            ]
+        },
+        "profiles": { "$ref": "#/$defs/profileRefs" },
+        "rubrics": {
+            "type": "array",
+            "minItems": 1,
+            "maxItems": 64,
+            "items": {
+                "$ref": "#/$defs/rubric"
+            }
+        },
+        "scenarios": {
+            "type": "array",
+            "minItems": 1,
+            "maxItems": 500,
+            "items": {
+                "$ref": "#/$defs/scenario"
+            }
+        }
+    },
+    "$defs": {
+        "profileRefs": { "type": "array", "minItems": 1, "maxItems": 64, "uniqueItems": true, "items": { "type": "string", "maxLength": 100, "pattern": "^[a-z0-9][a-z0-9-]{0,63}/[1-9][0-9]{0,8}$" } },
+        "teamPolicy": { "type": "object", "required": ["members", "require_distinct_owners"], "additionalProperties": false, "properties": { "members": { "type": "array", "minItems": 3, "maxItems": 64, "uniqueItems": true, "items": { "type": "object", "required": ["agent", "owner"], "additionalProperties": false, "properties": { "agent": { "anyOf": [{ "const": "$agent" }, { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" }] }, "owner": { "type": "string", "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$" } } } }, "require_distinct_owners": { "type": "boolean" }, "bind_run": { "type": "boolean" } } },
+        "text": {
+            "type": "object",
+            "required": [
+                "en",
+                "ar"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "en": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 4000
+                },
+                "ar": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 4000
+                }
+            }
+        },
+        "rubric": {
+            "type": "object",
+            "required": [
+                "id",
+                "title",
+                "pass",
+                "fail"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "id": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"
+                },
+                "title": {
+                    "$ref": "#/$defs/text"
+                },
+                "pass": {
+                    "$ref": "#/$defs/text"
+                },
+                "fail": {
+                    "$ref": "#/$defs/text"
+                }
+            }
+        },
+        "source": {
+            "type": "object",
+            "required": [
+                "kind",
+                "note"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "kind": {
+                    "enum": [
+                        "incident",
+                        "practice"
+                    ]
+                },
+                "note": {
+                    "$ref": "#/$defs/text"
+                },
+                "observed": {
+                    "type": "string",
+                    "pattern": "^[0-9]{4}-(0[1-9]|1[0-2])$"
+                },
+                "blackbox": {
+                    "type": "object",
+                    "required": [
+                        "source",
+                        "expect"
+                    ],
+                    "additionalProperties": false,
+                    "properties": {
+                        "source": {
+                            "type": "string",
+                            "pattern": "^[A-Za-z0-9._:-]{1,128}$"
+                        },
+                        "label": {
+                            "type": "string",
+                            "minLength": 1,
+                            "maxLength": 200
+                        },
+                        "calls": {
+                            "type": "integer",
+                            "minimum": 0,
+                            "maximum": 100000
+                        },
+                        "expect": {
+                            "type": "object",
+                            "required": [
+                                "finding_codes"
+                            ],
+                            "additionalProperties": false,
+                            "properties": {
+                                "finding_codes": {
+                                    "type": "object"
+                                }
+                            }
+                        }
+                    }
+                }
+            }
+        },
+        "scenario": {
+            "type": "object",
+            "required": [
+                "id",
+                "category",
+                "lang",
+                "title",
+                "source",
+                "turns",
+                "check"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "id": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]{0,95}$"
+                },
+                "profiles": { "$ref": "#/$defs/profileRefs" },
+                "category": {
+                    "type": "string",
+                    "pattern": "^[a-z][a-z0-9_]{0,63}$"
+                },
+                "lang": {
+                    "enum": [
+                        "en",
+                        "ar",
+                        "ar-gulf"
+                    ]
+                },
+                "title": {
+                    "$ref": "#/$defs/text"
+                },
+                "source": {
+                    "$ref": "#/$defs/source"
+                },
+                "turns": {
+                    "type": "array",
+                    "minItems": 3,
+                    "maxItems": 20,
+                    "items": {
+                        "type": "object",
+                        "required": [
+                            "role",
+                            "content"
+                        ],
+                        "additionalProperties": false,
+                        "properties": {
+                            "role": {
+                                "enum": [
+                                    "user",
+                                    "assistant"
+                                ]
+                            },
+                            "content": {
+                                "type": "string",
+                                "minLength": 1,
+                                "maxLength": 20000
+                            }
+                        }
+                    }
+                },
+                "documents": {
+                    "type": "array",
+                    "maxItems": 10,
+                    "items": {
+                        "type": "object",
+                        "required": [
+                            "name",
+                            "text"
+                        ],
+                        "additionalProperties": false,
+                        "properties": {
+                            "name": {
+                                "type": "string",
+                                "minLength": 1,
+                                "maxLength": 200
+                            },
+                            "text": {
+                                "type": "string",
+                                "minLength": 1,
+                                "maxLength": 100000
+                            }
+                        }
+                    }
+                },
+                "tools": {
+                    "type": "array",
+                    "maxItems": 64,
+                    "items": {
+                        "type": "object",
+                        "required": [
+                            "type",
+                            "function"
+                        ],
+                        "additionalProperties": false,
+                        "properties": {
+                            "type": {
+                                "const": "function"
+                            },
+                            "function": {
+                                "type": "object",
+                                "required": [
+                                    "name"
+                                ],
+                                "properties": {
+                                    "name": {
+                                        "type": "string",
+                                        "pattern": "^[A-Za-z0-9_-]{1,64}$"
+                                    },
+                                    "description": {
+                                        "type": "string",
+                                        "maxLength": 2000
+                                    },
+                                    "parameters": {
+                                        "type": "object"
+                                    }
+                                }
+                            }
+                        }
+                    }
+                },
+                "delegation": {
+                    "type": "object",
+                    "required": [
+                        "chain",
+                        "tools"
+                    ],
+                    "additionalProperties": false,
+                    "properties": {
+                        "chain": {
+                            "type": "array",
+                            "minItems": 1,
+                            "maxItems": 8,
+                            "items": {
+                                "$ref": "#/$defs/link"
+                            }
+                        },
+                        "tools": {
+                            "type": "array",
+                            "maxItems": 64,
+                            "items": {
+                                "type": "object",
+                                "required": [
+                                    "name",
+                                    "scope"
+                                ],
+                                "additionalProperties": false,
+                                "properties": {
+                                    "name": {
+                                        "type": "string",
+                                        "pattern": "^[A-Za-z0-9_-]{1,64}$"
+                                    },
+                                    "scope": {
+                                        "$ref": "#/$defs/scope"
+                                    },
+                                    "amount": {
+                                        "$ref": "#/$defs/arg"
+                                    },
+                                    "currency": {
+                                        "$ref": "#/$defs/arg"
+                                    }
+                                }
+                            }
+                        },
+                        "certs": {
+                            "type": "array",
+                            "minItems": 1,
+                            "maxItems": 8,
+                            "items": {
+                                "$ref": "#/$defs/cert"
+                            }
+                        },
+                        "revocations": {
+                            "type": "array",
+                            "minItems": 1,
+                            "maxItems": 64,
+                            "items": {
+                                "$ref": "#/$defs/revocation"
+                            }
+                        }
+                    }
+                },
+                "sub_agent": {
+                    "type": "object",
+                    "required": [
+                        "agent",
+                        "tool",
+                        "scopes"
+                    ],
+                    "additionalProperties": false,
+                    "properties": {
+                        "agent": {
+                            "type": "string",
+                            "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"
+                        },
+                        "tool": {
+                            "type": "string",
+                            "pattern": "^[A-Za-z0-9_-]{1,64}$"
+                        },
+                        "scopes": {
+                            "$ref": "#/$defs/arg"
+                        },
+                        "amount": {
+                            "$ref": "#/$defs/arg"
+                        }
+                    }
+                },
+                "check": {
+                    "$ref": "#/$defs/check"
+                }
+            }
+        },
+        "check": {
+            "type": "object",
+            "required": [
+                "kind"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "kind": {
+                    "enum": [
+                        "not_contains",
+                        "contains_any",
+                        "tool_not_called",
+                        "max_tool_calls",
+                        "all",
+                        "judged",
+                        "within_delegation",
+                        "no_scope_escalation",
+                        "team_receipt"
+                    ]
+                },
+                "values": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "items": {
+                        "type": "string",
+                        "minLength": 1,
+                        "maxLength": 2000
+                    }
+                },
+                "tools": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "uniqueItems": true,
+                    "items": {
+                        "type": "string",
+                        "pattern": "^[A-Za-z0-9_-]{1,64}$"
+                    }
+                },
+                "n": {
+                    "type": "integer",
+                    "minimum": 0,
+                    "maximum": 1000
+                },
+                "checks": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 16,
+                    "items": {
+                        "$ref": "#/$defs/check"
+                    }
+                },
+                "rubric": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"
+                },
+                "finding": {
+                    "type": "string",
+                    "pattern": "^[A-Z][A-Z0-9_]{0,63}$"
+                },
+                "tool": {
+                    "type": "string",
+                    "pattern": "^[A-Za-z0-9_-]{1,64}$"
+                },
+                "threshold": {
+                    "type": "integer",
+                    "minimum": 1,
+                    "maximum": 64
+                },
+                "invocation": { "enum": ["required", "optional"] },
+                "policy": { "$ref": "#/$defs/teamPolicy" }
+            }
+        },
+        "scope": {
+            "type": "string",
+            "maxLength": 128,
+            "pattern": "^(\\*|[a-z0-9_-]+(\\.[a-z0-9_-]+)*(\\.\\*)?)$"
+        },
+        "arg": {
+            "type": "string",
+            "pattern": "^[A-Za-z0-9_]{1,64}$"
+        },
+        "link": {
+            "type": "object",
+            "required": [
+                "agent",
+                "scopes",
+                "spend_limit"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "agent": {
+                    "type": "string",
+                    "pattern": "^[a-z0-9][a-z0-9-]{0,63}$"
+                },
+                "scopes": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "uniqueItems": true,
+                    "items": {
+                        "$ref": "#/$defs/scope"
+                    }
+                },
+                "spend_limit": {
+                    "type": "object",
+                    "required": [
+                        "amount_minor",
+                        "currency"
+                    ],
+                    "additionalProperties": false,
+                    "properties": {
+                        "amount_minor": {
+                            "type": "integer",
+                            "minimum": 0,
+                            "maximum": 9007199254740991
+                        },
+                        "currency": {
+                            "type": "string",
+                            "pattern": "^[A-Z]{3}$"
+                        }
+                    }
+                }
+            }
+        },
+        "cert": {
+            "type": "object",
+            "required": [
+                "v",
+                "issuer",
+                "subject",
+                "scopes",
+                "exp",
+                "sig"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "v": {
+                    "const": 1
+                },
+                "issuer": {
+                    "type": "string",
+                    "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$"
+                },
+                "subject": {
+                    "type": "string",
+                    "pattern": "^did:key:z[1-9A-HJ-NP-Za-km-z]{40,60}$"
+                },
+                "scopes": {
+                    "type": "array",
+                    "minItems": 1,
+                    "maxItems": 64,
+                    "items": {
+                        "$ref": "#/$defs/scope"
+                    }
+                },
+                "exp": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 64
+                },
+                "sig": {
+                    "type": "string",
+                    "pattern": "^ed25519:[0-9a-f]{128}$"
+                }
+            }
+        },
+        "revocation": {
+            "type": "object",
+            "required": [
+                "v",
+                "type",
+                "cert",
+                "ts",
+                "sig"
+            ],
+            "additionalProperties": false,
+            "properties": {
+                "v": {
+                    "const": 1
+                },
+                "type": {
+                    "const": "revocation"
+                },
+                "cert": {
+                    "$ref": "#/$defs/cert"
+                },
+                "ts": {
+                    "type": "string",
+                    "minLength": 1,
+                    "maxLength": 64
+                },
+                "sig": {
+                    "type": "string",
+                    "pattern": "^ed25519:[0-9a-f]{128}$"
+                }
+            }
+        }
+    }
+}