ai-capability-framework 1.0.0-rc.1

package/schemas/capability-manifest.schema.json

@@ -0,0 +1,250 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/capability-manifest.schema.json",
+  "title": "AICF Capability Manifest",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "id",
+    "version",
+    "status",
+    "name",
+    "summary",
+    "model_description",
+    "capability_type",
+    "autonomy_tier",
+    "risk_tier",
+    "input_schema",
+    "output_schema",
+    "side_effects",
+    "authorization",
+    "policy",
+    "lifecycle",
+    "observability"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*){2,}$"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-[0-9A-Za-z-.]+)?(?:\\+[0-9A-Za-z-.]+)?$"
+    },
+    "status": {
+      "enum": ["draft", "experimental", "active", "deprecated", "disabled"]
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1
+    },
+    "summary": {
+      "type": "string",
+      "minLength": 1
+    },
+    "model_description": {
+      "type": "string",
+      "minLength": 1
+    },
+    "domain": {
+      "type": "string",
+      "minLength": 1
+    },
+    "owner": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "team": { "type": "string" },
+        "contact": { "type": "string" }
+      }
+    },
+    "capability_type": {
+      "enum": [
+        "read_data",
+        "retrieve_documents",
+        "compute",
+        "write_prepare_only",
+        "write_commit",
+        "external_message_prepare",
+        "external_message_send",
+        "workflow_start",
+        "workflow_step",
+        "human_handoff"
+      ]
+    },
+    "autonomy_tier": {
+      "enum": ["A0", "A1", "A2", "A3", "A4", "A5"]
+    },
+    "risk_tier": {
+      "enum": ["none", "low", "medium", "high", "critical"]
+    },
+    "when_to_use": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "when_not_to_use": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "input_schema": { "$ref": "#/$defs/json_schema_object" },
+    "output_schema": { "$ref": "#/$defs/json_schema_object" },
+    "side_effects": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "reads_data",
+        "writes_data",
+        "creates_records",
+        "updates_records",
+        "deletes_records",
+        "sends_external_messages",
+        "charges_money",
+        "refunds_money",
+        "changes_permissions",
+        "triggers_external_workflow",
+        "irreversible"
+      ],
+      "properties": {
+        "reads_data": { "type": "boolean" },
+        "writes_data": { "type": "boolean" },
+        "creates_records": { "type": "boolean" },
+        "updates_records": { "type": "boolean" },
+        "deletes_records": { "type": "boolean" },
+        "sends_external_messages": { "type": "boolean" },
+        "charges_money": { "type": "boolean" },
+        "refunds_money": { "type": "boolean" },
+        "changes_permissions": { "type": "boolean" },
+        "triggers_external_workflow": { "type": "boolean" },
+        "irreversible": { "type": "boolean" }
+      }
+    },
+    "authorization": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["permissions", "tenant_scoped", "requires_user_context"],
+      "properties": {
+        "permissions": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "tenant_scoped": { "type": "boolean" },
+        "requires_user_context": { "type": "boolean" },
+        "data_scope": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "policy": {
+      "type": "object",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "approval_required": { "type": "boolean" },
+        "approval_required_if": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/policy_rule" }
+        },
+        "deny_if": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/policy_rule" }
+        },
+        "max_autonomy_tier": {
+          "enum": ["A0", "A1", "A2", "A3", "A4", "A5"]
+        }
+      }
+    },
+    "lifecycle": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["prepare", "preview", "approve", "commit", "verify", "audit"],
+      "properties": {
+        "prepare": { "type": "boolean" },
+        "preview": { "type": "boolean" },
+        "approve": { "type": "boolean" },
+        "commit": { "type": "boolean" },
+        "verify": { "type": "boolean" },
+        "audit": { "type": "boolean" }
+      }
+    },
+    "idempotency": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["required"],
+      "properties": {
+        "required": { "type": "boolean" },
+        "key_fields": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "observability": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["log_inputs", "log_outputs"],
+      "properties": {
+        "log_inputs": {
+          "enum": ["none", "summary", "redacted"]
+        },
+        "log_outputs": {
+          "enum": ["none", "summary", "redacted"]
+        },
+        "trace_attributes": {
+          "type": "object",
+          "additionalProperties": {
+            "type": ["string", "number", "boolean", "null"]
+          }
+        }
+      }
+    },
+    "evals": {
+      "type": "object",
+      "additionalProperties": false,
+      "properties": {
+        "golden": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "red_team": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "$defs": {
+    "json_schema_object": {
+      "type": "object",
+      "required": ["type"],
+      "additionalProperties": true,
+      "properties": {
+        "type": {
+          "type": ["string", "array"]
+        }
+      }
+    },
+    "policy_rule": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["rule", "reason"],
+      "properties": {
+        "rule": { "type": "string", "minLength": 1 },
+        "reason": { "type": "string", "minLength": 1 },
+        "field": { "type": "string" },
+        "operator": {
+          "enum": ["eq", "neq", "gt", "gte", "lt", "lte", "in", "not_in", "exists"]
+        },
+        "value": {}
+      }
+    }
+  }
+}

package/schemas/entity-manifest.schema.json

@@ -0,0 +1,97 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/entity-manifest.schema.json",
+  "title": "AICF Entity Manifest",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schema_version",
+    "id",
+    "version",
+    "name",
+    "summary",
+    "canonical_id",
+    "data_classification",
+    "lookup",
+    "allowed_actions",
+    "model_guidance"
+  ],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "id": {
+      "type": "string",
+      "pattern": "^[A-Z][A-Za-z0-9_]*$"
+    },
+    "version": {
+      "type": "string",
+      "pattern": "^(0|[1-9]\\d*)\\.(0|[1-9]\\d*)\\.(0|[1-9]\\d*)(?:-[0-9A-Za-z-.]+)?(?:\\+[0-9A-Za-z-.]+)?$"
+    },
+    "name": {
+      "type": "string",
+      "minLength": 1
+    },
+    "summary": {
+      "type": "string",
+      "minLength": 1
+    },
+    "canonical_id": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["field"],
+      "properties": {
+        "field": { "type": "string", "minLength": 1 },
+        "pattern": { "type": "string" }
+      }
+    },
+    "data_classification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["default"],
+      "properties": {
+        "default": { "type": "string", "minLength": 1 },
+        "fields": {
+          "type": "object",
+          "additionalProperties": { "type": "string" }
+        }
+      }
+    },
+    "relationships": {
+      "type": "array",
+      "items": {
+        "type": "object",
+        "additionalProperties": false,
+        "required": ["name", "target_entity", "cardinality"],
+        "properties": {
+          "name": { "type": "string", "minLength": 1 },
+          "target_entity": { "type": "string", "minLength": 1 },
+          "cardinality": {
+            "enum": ["one_to_one", "one_to_many", "many_to_one", "many_to_many"]
+          },
+          "lookup_capability": { "type": "string" }
+        }
+      }
+    },
+    "lookup": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["primary_capability"],
+      "properties": {
+        "primary_capability": { "type": "string" }
+      }
+    },
+    "allowed_actions": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "model_guidance": {
+      "type": "string",
+      "minLength": 1
+    },
+    "extensions": {
+      "type": "object"
+    }
+  }
+}

package/schemas/eval-case.schema.json

@@ -0,0 +1,134 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/eval-case.schema.json",
+  "title": "AICF Eval Case",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "id", "input", "expected", "scorers"],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "id": {
+      "type": "string",
+      "pattern": "^[a-z][a-z0-9_]*(\\.[a-z][a-z0-9_]*)*$"
+    },
+    "name": {
+      "type": "string"
+    },
+    "capability_under_test": {
+      "type": "string"
+    },
+    "tags": {
+      "type": "array",
+      "items": { "type": "string" }
+    },
+    "input": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["user_message"],
+      "properties": {
+        "user_message": { "type": "string", "minLength": 1 },
+        "conversation": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["role", "content"],
+            "properties": {
+              "role": { "enum": ["user", "assistant", "system", "tool"] },
+              "content": { "type": "string" }
+            }
+          }
+        }
+      }
+    },
+    "context": {
+      "type": "object"
+    },
+    "expected": { "$ref": "#/$defs/expected_behavior" },
+    "scorers": {
+      "type": "array",
+      "minItems": 1,
+      "items": {
+        "type": "object",
+        "additionalProperties": true,
+        "required": ["type"],
+        "properties": {
+          "type": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "$defs": {
+    "expected_behavior": {
+      "type": "object",
+      "additionalProperties": false,
+      "minProperties": 1,
+      "properties": {
+        "selected_capabilities": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "includes": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 }
+            },
+            "excludes": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 }
+            }
+          }
+        },
+        "tool_calls": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["capability_id"],
+            "properties": {
+              "capability_id": { "type": "string", "minLength": 1 },
+              "args_match": { "type": "object" }
+            }
+          }
+        },
+        "policy_decision": {
+          "enum": ["allowed", "approval_required", "denied"]
+        },
+        "action_state": {
+          "enum": ["none", "prepared", "approval_required", "committed", "denied", "refused"]
+        },
+        "no_commit": { "type": "boolean" },
+        "refusal": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["required"],
+          "properties": {
+            "required": { "type": "boolean" },
+            "reason_contains": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 }
+            }
+          }
+        },
+        "response": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "must_include": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 }
+            },
+            "must_not_include": {
+              "type": "array",
+              "items": { "type": "string", "minLength": 1 }
+            }
+          }
+        }
+      }
+    }
+  }
+}

package/schemas/eval-result.schema.json

@@ -0,0 +1,77 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://raw.githubusercontent.com/xennon101/ai-capability-framework/main/schemas/eval-result.schema.json",
+  "title": "AICF Eval Result Fixture",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schema_version", "results"],
+  "properties": {
+    "schema_version": {
+      "const": "1.0"
+    },
+    "results": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/eval_candidate_result" }
+    },
+    "extensions": {
+      "type": "object"
+    }
+  },
+  "$defs": {
+    "eval_candidate_result": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["eval_id"],
+      "properties": {
+        "eval_id": {
+          "type": "string",
+          "minLength": 1
+        },
+        "selected_capabilities": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "tool_calls": {
+          "type": "array",
+          "items": {
+            "type": "object",
+            "additionalProperties": false,
+            "required": ["capability_id"],
+            "properties": {
+              "capability_id": { "type": "string", "minLength": 1 },
+              "args": { "type": "object" }
+            }
+          }
+        },
+        "policy_decision": {
+          "enum": ["allowed", "approval_required", "denied"]
+        },
+        "action_state": {
+          "enum": ["none", "prepared", "approval_required", "committed", "denied", "refused"]
+        },
+        "committed_capabilities": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "refusal": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "present": { "type": "boolean" },
+            "reason": { "type": "string" }
+          }
+        },
+        "response": {
+          "type": "object",
+          "additionalProperties": false,
+          "properties": {
+            "text": { "type": "string" }
+          }
+        },
+        "extensions": {
+          "type": "object"
+        }
+      }
+    }
+  }
+}