@bluefly/openstandardagents 0.4.8 → 0.4.9

package/spec/v0.5/extensions/evals/evals.schema.json

@@ -0,0 +1,597 @@
+{
+  "$schema": "http://json-schema.org/draft-07/schema#",
+  "$id": "https://openstandardagents.org/schemas/v0.4/extensions/evals.json",
+  "title": "OSSA Evals Extension Schema",
+  "description": "Validates eval-config.yaml generated by the CLEAR evaluation framework. Defines cost, latency, accuracy, reliability, and security evaluation dimensions for OSSA agents.",
+  "type": "object",
+  "required": ["version", "framework"],
+  "properties": {
+    "version": {
+      "type": "string",
+      "description": "Schema version of the eval configuration",
+      "pattern": "^[0-9]+\\.[0-9]+\\.[0-9]+$",
+      "examples": ["1.0.0"]
+    },
+    "framework": {
+      "type": "string",
+      "description": "Evaluation framework identifier",
+      "enum": ["CLEAR"],
+      "default": "CLEAR"
+    },
+    "agent": {
+      "type": "object",
+      "description": "Agent metadata for the evaluation target",
+      "properties": {
+        "name": { "type": "string", "description": "Agent name from manifest metadata" },
+        "version": { "type": "string", "description": "Agent version" },
+        "description": { "type": "string", "description": "Agent description" },
+        "agent_type": { "type": "string", "description": "Agent type classification" },
+        "agent_kind": { "type": "string", "description": "Agent kind (assistant, worker, orchestrator, etc.)" },
+        "architecture_pattern": {
+          "type": "string",
+          "description": "Multi-agent architecture pattern",
+          "enum": ["single", "swarm", "pipeline", "graph", "hierarchical", "reactive", "cognitive", "lead-teammate"]
+        },
+        "model": { "type": "string", "description": "LLM provider/model identifier" }
+      },
+      "required": ["name"],
+      "additionalProperties": false
+    },
+    "dimensions": {
+      "type": "object",
+      "description": "CLEAR framework evaluation dimensions",
+      "properties": {
+        "cost": { "$ref": "#/definitions/CostDimension" },
+        "latency": { "$ref": "#/definitions/LatencyDimension" },
+        "accuracy": { "$ref": "#/definitions/AccuracyDimension" },
+        "reliability": { "$ref": "#/definitions/ReliabilityDimension" },
+        "security": { "$ref": "#/definitions/SecurityDimension" }
+      },
+      "additionalProperties": false
+    },
+    "tools": {
+      "type": "array",
+      "description": "Per-tool evaluation configuration with golden-set references",
+      "items": {
+        "$ref": "#/definitions/ToolEvalConfig"
+      }
+    },
+    "capabilities": {
+      "type": "object",
+      "description": "Capability test suite configuration",
+      "properties": {
+        "test_suite": {
+          "type": "string",
+          "const": "capabilities"
+        },
+        "version": { "type": "string" },
+        "agent": {
+          "type": "object",
+          "properties": {
+            "name": { "type": "string" },
+            "kind": { "type": "string" },
+            "capabilities": {
+              "type": "array",
+              "items": { "type": "string" }
+            }
+          },
+          "additionalProperties": false
+        },
+        "test_cases": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/CapabilityTestCase" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "teamCoordination": {
+      "type": "object",
+      "description": "Multi-agent team coordination test suite (present only for multi-agent manifests)",
+      "properties": {
+        "test_suite": {
+          "type": "string",
+          "const": "team-coordination"
+        },
+        "version": { "type": "string" },
+        "agent": {
+          "type": "object",
+          "properties": {
+            "name": { "type": "string" },
+            "architecture_pattern": { "type": "string" },
+            "handoff_strategy": { "type": "string" },
+            "max_depth": {
+              "oneOf": [
+                { "type": "integer", "minimum": 1 },
+                { "type": "string", "const": "unlimited" }
+              ]
+            }
+          },
+          "additionalProperties": false
+        },
+        "test_cases": {
+          "type": "array",
+          "items": { "$ref": "#/definitions/TeamCoordinationTestCase" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "execution": {
+      "type": "object",
+      "description": "Evaluation suite execution configuration",
+      "properties": {
+        "runner": {
+          "type": "string",
+          "description": "Eval runner identifier",
+          "default": "ossa-evals"
+        },
+        "parallel": {
+          "type": "boolean",
+          "description": "Whether to run tests in parallel",
+          "default": true
+        },
+        "max_concurrency": {
+          "type": "integer",
+          "description": "Maximum parallel test workers",
+          "minimum": 1,
+          "default": 5
+        },
+        "timeout_per_test_seconds": {
+          "type": "number",
+          "description": "Timeout for each individual test",
+          "minimum": 1
+        },
+        "report_format": {
+          "type": "array",
+          "description": "Output report formats",
+          "items": {
+            "type": "string",
+            "enum": ["json", "yaml", "html", "csv", "junit"]
+          }
+        },
+        "output_directory": {
+          "type": "string",
+          "description": "Directory for evaluation results"
+        }
+      },
+      "additionalProperties": false
+    }
+  },
+  "additionalProperties": false,
+  "definitions": {
+    "CostDimension": {
+      "type": "object",
+      "description": "Token and monetary cost guardrails",
+      "properties": {
+        "description": { "type": "string" },
+        "max_tokens_per_day": {
+          "type": "integer",
+          "description": "Maximum token usage per day",
+          "minimum": 0
+        },
+        "max_tokens_per_request": {
+          "type": "integer",
+          "description": "Maximum tokens per single request",
+          "minimum": 0
+        },
+        "max_cost_per_day": {
+          "type": "number",
+          "description": "Maximum monetary cost per day",
+          "minimum": 0
+        },
+        "currency": {
+          "type": "string",
+          "description": "Currency code (ISO 4217)",
+          "default": "USD",
+          "pattern": "^[A-Z]{3}$"
+        },
+        "alert_thresholds": {
+          "type": "array",
+          "description": "Budget alert thresholds",
+          "items": {
+            "type": "object",
+            "properties": {
+              "level": {
+                "type": "string",
+                "enum": ["info", "warning", "critical"]
+              },
+              "percent_of_daily_limit": {
+                "type": "number",
+                "minimum": 0,
+                "maximum": 100
+              }
+            },
+            "required": ["level", "percent_of_daily_limit"],
+            "additionalProperties": false
+          }
+        },
+        "per_tool_budgets": {
+          "type": "array",
+          "description": "Per-tool token budgets",
+          "items": {
+            "type": "object",
+            "properties": {
+              "tool": { "type": "string" },
+              "max_tokens_per_call": {
+                "type": "integer",
+                "minimum": 0
+              }
+            },
+            "required": ["tool", "max_tokens_per_call"],
+            "additionalProperties": false
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "LatencyDimension": {
+      "type": "object",
+      "description": "Response time SLOs and timeout thresholds",
+      "properties": {
+        "description": { "type": "string" },
+        "max_latency_seconds": {
+          "type": "number",
+          "description": "Maximum acceptable latency in seconds",
+          "minimum": 0
+        },
+        "timeout_seconds": {
+          "type": "number",
+          "description": "Hard timeout in seconds",
+          "minimum": 0
+        },
+        "percentile_targets": {
+          "type": "object",
+          "description": "Latency percentile targets",
+          "properties": {
+            "p50": { "type": "number", "minimum": 0 },
+            "p90": { "type": "number", "minimum": 0 },
+            "p95": { "type": "number", "minimum": 0 },
+            "p99": { "type": "number", "minimum": 0 }
+          },
+          "additionalProperties": false
+        },
+        "tool_latency_budgets": {
+          "type": "array",
+          "description": "Per-tool latency budgets",
+          "items": {
+            "type": "object",
+            "properties": {
+              "tool": { "type": "string" },
+              "max_seconds": {
+                "type": "number",
+                "minimum": 0
+              }
+            },
+            "required": ["tool", "max_seconds"],
+            "additionalProperties": false
+          }
+        }
+      },
+      "additionalProperties": false
+    },
+    "AccuracyDimension": {
+      "type": "object",
+      "description": "Output quality measured via golden-set test pairs",
+      "properties": {
+        "description": { "type": "string" },
+        "golden_set_directory": {
+          "type": "string",
+          "description": "Path to golden-set YAML files"
+        },
+        "min_pass_rate": {
+          "type": "number",
+          "description": "Minimum pass rate (0.0 to 1.0)",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "scoring": {
+          "type": "object",
+          "description": "Scoring methodology",
+          "properties": {
+            "method": {
+              "type": "string",
+              "enum": ["exact_match", "semantic", "exact_match_and_semantic", "custom"]
+            },
+            "semantic_similarity_threshold": {
+              "type": "number",
+              "minimum": 0,
+              "maximum": 1
+            },
+            "field_weights": {
+              "type": "object",
+              "description": "Scoring weights for field categories",
+              "properties": {
+                "required_fields": { "type": "number", "minimum": 0 },
+                "optional_fields": { "type": "number", "minimum": 0 }
+              },
+              "additionalProperties": { "type": "number" }
+            }
+          },
+          "additionalProperties": false
+        },
+        "tools_evaluated": {
+          "type": "array",
+          "description": "List of tool names covered by golden sets",
+          "items": { "type": "string" }
+        }
+      },
+      "additionalProperties": false
+    },
+    "ReliabilityDimension": {
+      "type": "object",
+      "description": "Error budget and resilience requirements",
+      "properties": {
+        "description": { "type": "string" },
+        "max_error_rate": {
+          "type": "number",
+          "description": "Maximum acceptable error rate (0.0 to 1.0)",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "uptime_target": {
+          "type": "number",
+          "description": "Target uptime percentage (0.0 to 1.0)",
+          "minimum": 0,
+          "maximum": 1
+        },
+        "retry_policy": {
+          "type": "object",
+          "description": "Retry behavior configuration",
+          "properties": {
+            "max_retries": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "backoff": {
+              "type": "string",
+              "enum": ["none", "linear", "exponential"]
+            },
+            "initial_delay_ms": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "max_delay_ms": {
+              "type": "integer",
+              "minimum": 0
+            }
+          },
+          "additionalProperties": false
+        },
+        "circuit_breaker": {
+          "type": "object",
+          "description": "Circuit breaker configuration",
+          "properties": {
+            "enabled": { "type": "boolean" },
+            "failure_threshold": {
+              "type": "integer",
+              "minimum": 1
+            },
+            "reset_timeout_seconds": {
+              "type": "number",
+              "minimum": 0
+            }
+          },
+          "additionalProperties": false
+        },
+        "graceful_degradation": {
+          "type": "object",
+          "description": "Graceful degradation behavior",
+          "properties": {
+            "enabled": { "type": "boolean" },
+            "fallback_behavior": {
+              "type": "string",
+              "enum": ["return_cached_or_error", "return_partial", "redirect", "queue"]
+            }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "SecurityDimension": {
+      "type": "object",
+      "description": "Prompt injection resistance, PII handling, and access control",
+      "properties": {
+        "description": { "type": "string" },
+        "prompt_injection": {
+          "type": "object",
+          "description": "Prompt injection resistance testing",
+          "properties": {
+            "test_enabled": { "type": "boolean" },
+            "attack_vectors": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [
+                  "ignore_instructions",
+                  "role_override",
+                  "system_prompt_leak",
+                  "data_exfiltration",
+                  "indirect_injection",
+                  "encoding_bypass",
+                  "multi_turn_manipulation"
+                ]
+              }
+            },
+            "expected_behavior": {
+              "type": "string",
+              "enum": ["refuse_or_ignore", "refuse", "ignore", "log_and_refuse"]
+            }
+          },
+          "additionalProperties": false
+        },
+        "pii_handling": {
+          "type": "object",
+          "description": "PII detection and handling verification",
+          "properties": {
+            "test_enabled": { "type": "boolean" },
+            "detectable_types": {
+              "type": "array",
+              "items": {
+                "type": "string",
+                "enum": [
+                  "credit_card", "ssn", "email", "phone",
+                  "api_key", "password", "address", "dob",
+                  "ip_address", "passport"
+                ]
+              }
+            },
+            "expected_behavior": {
+              "type": "string",
+              "enum": ["redact_or_refuse", "redact", "refuse", "log_and_redact"]
+            }
+          },
+          "additionalProperties": false
+        },
+        "autonomy_compliance": {
+          "type": "object",
+          "description": "Autonomy boundary compliance verification",
+          "properties": {
+            "level": {
+              "type": "string",
+              "enum": ["autonomous", "semi-autonomous", "supervised", "human-in-the-loop"]
+            },
+            "allowed_actions": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "blocked_actions": {
+              "type": "array",
+              "items": { "type": "string" }
+            },
+            "test_blocked_actions": { "type": "boolean" }
+          },
+          "additionalProperties": false
+        },
+        "input_validation": {
+          "type": "object",
+          "description": "Input validation constraints",
+          "properties": {
+            "max_input_length": {
+              "type": "integer",
+              "minimum": 0
+            },
+            "reject_binary": { "type": "boolean" },
+            "sanitize_html": { "type": "boolean" }
+          },
+          "additionalProperties": false
+        }
+      },
+      "additionalProperties": false
+    },
+    "ToolEvalConfig": {
+      "type": "object",
+      "description": "Per-tool evaluation configuration with golden-set reference",
+      "properties": {
+        "name": {
+          "type": "string",
+          "description": "Tool name matching spec.tools[].name"
+        },
+        "goldenSets": {
+          "type": "array",
+          "description": "Paths to golden-set YAML files for this tool",
+          "items": {
+            "type": "object",
+            "properties": {
+              "path": {
+                "type": "string",
+                "description": "Relative path to the golden-set YAML file"
+              },
+              "tool": {
+                "type": "string",
+                "description": "Tool name this golden set validates"
+              },
+              "schema_version": { "type": "string" },
+              "test_cases": {
+                "type": "array",
+                "items": { "$ref": "#/definitions/GoldenTestCase" }
+              }
+            },
+            "required": ["path"],
+            "additionalProperties": false
+          }
+        }
+      },
+      "required": ["name"],
+      "additionalProperties": false
+    },
+    "GoldenTestCase": {
+      "type": "object",
+      "description": "A single golden-set test case with input/expected-output pair",
+      "properties": {
+        "id": {
+          "type": "string",
+          "description": "Unique test case identifier",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "description": { "type": "string" },
+        "input": {
+          "type": "object",
+          "description": "Tool input payload"
+        },
+        "expected_output": {
+          "type": "object",
+          "description": "Expected tool output or assertion metadata"
+        },
+        "tags": {
+          "type": "array",
+          "description": "Classification tags for filtering and reporting",
+          "items": { "type": "string" }
+        }
+      },
+      "required": ["id", "description", "input", "expected_output"],
+      "additionalProperties": false
+    },
+    "CapabilityTestCase": {
+      "type": "object",
+      "description": "A capability test scenario with behavioral assertions",
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "name": { "type": "string" },
+        "description": { "type": "string" },
+        "input": { "type": "string" },
+        "expected_behavior": { "type": "string" },
+        "assertions": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "tags": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      },
+      "required": ["id", "name", "description", "input", "expected_behavior", "assertions"],
+      "additionalProperties": false
+    },
+    "TeamCoordinationTestCase": {
+      "type": "object",
+      "description": "A multi-agent coordination test scenario",
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "name": { "type": "string" },
+        "description": { "type": "string" },
+        "trigger": { "type": "string" },
+        "expected_flow": {
+          "type": "array",
+          "description": "Expected sequence of events in the coordination flow",
+          "items": { "type": "string" }
+        },
+        "assertions": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "tags": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      },
+      "required": ["id", "name", "description", "trigger", "expected_flow", "assertions"],
+      "additionalProperties": false
+    }
+  }
+}