audrey 0.23.1 → 1.0.0

package/benchmarks/schemas/guardbench-external-run.schema.json

@@ -0,0 +1,160 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-external-run.schema.json",
+  "title": "GuardBench External Run Metadata",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "suite",
+    "startedAt",
+    "adapter",
+    "adapterPath",
+    "outDir",
+    "requiredEnv",
+    "missingEnv",
+    "command",
+    "validationCommand",
+    "dryRun",
+    "status"
+  ],
+  "properties": {
+    "suite": { "const": "GuardBench external adapter run" },
+    "startedAt": { "type": "string", "minLength": 1 },
+    "completedAt": { "type": "string", "minLength": 1 },
+    "adapter": { "type": "string", "minLength": 1 },
+    "adapterPath": { "type": "string", "minLength": 1 },
+    "outDir": { "type": "string", "minLength": 1 },
+    "requiredEnv": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "missingEnv": {
+      "type": "array",
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "command": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "validationCommand": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "dryRun": { "type": "boolean" },
+    "status": {
+      "enum": [
+        "blocked",
+        "dry-run-missing-env",
+        "dry-run-ready",
+        "failed",
+        "passed",
+        "running"
+      ]
+    },
+    "blockReason": { "type": "string", "minLength": 1 },
+    "exitCode": {
+      "anyOf": [
+        { "type": "integer" },
+        { "type": "null" }
+      ]
+    },
+    "signal": {
+      "anyOf": [
+        { "type": "string", "minLength": 1 },
+        { "type": "null" }
+      ]
+    },
+    "artifactHashes": { "$ref": "#/$defs/artifactHashes" },
+    "artifactValidation": { "$ref": "#/$defs/artifactValidation" },
+    "adapterConformance": { "$ref": "#/$defs/adapterConformance" }
+  },
+  "$defs": {
+    "artifactHashes": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "guardbench-manifest.json",
+        "guardbench-summary.json",
+        "guardbench-raw.json"
+      ],
+      "properties": {
+        "guardbench-manifest.json": { "$ref": "#/$defs/sha256" },
+        "guardbench-summary.json": { "$ref": "#/$defs/sha256" },
+        "guardbench-raw.json": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "sha256": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    },
+    "artifactValidation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "dir", "schemasDir", "files", "failures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "dir": { "type": "string", "minLength": 1 },
+        "schemasDir": { "type": "string", "minLength": 1 },
+        "files": {
+          "type": "array",
+          "minItems": 3,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "optionalFiles": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "adapterConformance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "ok",
+        "adapter",
+        "requestedAdapter",
+        "scenarios",
+        "expectedScenarios",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "redactionLeaks",
+        "failures"
+      ],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "adapter": { "type": "string", "minLength": 1 },
+        "requestedAdapter": { "type": "string", "minLength": 1 },
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "expectedScenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassRate": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "number", "minimum": 0, "maximum": 1 }
+          ]
+        },
+        "decisionAccuracy": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "number", "minimum": 0, "maximum": 1 }
+          ]
+        },
+        "redactionLeaks": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "integer", "minimum": 0 }
+          ]
+        },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    }
+  }
+}

package/benchmarks/schemas/guardbench-leaderboard.schema.json

@@ -0,0 +1,179 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-leaderboard.schema.json",
+  "title": "GuardBench Leaderboard",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "suite", "generatedAt", "ranking", "rows", "failures"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench leaderboard" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "ranking": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "rows": {
+      "type": "array",
+      "items": { "$ref": "#/$defs/row" }
+    },
+    "failures": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "$defs": {
+    "row": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "rank",
+        "subject",
+        "score",
+        "conformance",
+        "source",
+        "verification"
+      ],
+      "properties": {
+        "rank": { "type": "integer", "minimum": 1 },
+        "subject": { "$ref": "#/$defs/subject" },
+        "score": { "$ref": "#/$defs/score" },
+        "conformance": { "$ref": "#/$defs/conformance" },
+        "source": { "$ref": "#/$defs/source" },
+        "verification": { "$ref": "#/$defs/verification" }
+      }
+    },
+    "subject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "requestedAdapter", "external"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "requestedAdapter": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "external": { "type": "boolean" }
+      }
+    },
+    "score": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "scenarios",
+        "fullContractPassed",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "evidenceRecall",
+        "redactionLeaks",
+        "latency"
+      ],
+      "properties": {
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassed": { "$ref": "#/$defs/nullableNumber" },
+        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
+        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
+        "evidenceRecall": { "$ref": "#/$defs/nullableRate" },
+        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
+        "latency": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["p50Ms", "p95Ms", "maxMs"],
+              "properties": {
+                "p50Ms": { "type": "number", "minimum": 0 },
+                "p95Ms": { "type": "number", "minimum": 0 },
+                "maxMs": { "type": "number", "minimum": 0 }
+              }
+            },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "conformance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "failures", "artifactValidationOk", "artifactValidationFailures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "artifactValidationOk": { "type": "boolean" },
+        "artifactValidationFailures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "source": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["dir", "manifestGeneratedAt", "fileCount"],
+      "properties": {
+        "dir": { "type": "string", "minLength": 1 },
+        "manifestGeneratedAt": { "type": "string", "minLength": 1 },
+        "fileCount": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "verification": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "dir", "subject", "files", "artifactValidation", "failures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "dir": { "type": "string", "minLength": 1 },
+        "subject": { "$ref": "#/$defs/subject" },
+        "files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "artifactValidation": { "$ref": "#/$defs/artifactValidation" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "artifactValidation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "dir", "schemasDir", "files", "optionalFiles", "failures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "dir": { "type": "string", "minLength": 1 },
+        "schemasDir": { "type": "string", "minLength": 1 },
+        "files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "optionalFiles": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "nullableNumber": {
+      "anyOf": [
+        { "type": "number", "minimum": 0 },
+        { "type": "null" }
+      ]
+    },
+    "nullableRate": {
+      "anyOf": [
+        { "type": "number", "minimum": 0, "maximum": 1 },
+        { "type": "null" }
+      ]
+    }
+  }
+}

package/benchmarks/schemas/guardbench-manifest.schema.json

@@ -0,0 +1,213 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-manifest.schema.json",
+  "title": "GuardBench Manifest",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "manifestVersion",
+    "suiteId",
+    "suiteName",
+    "generatedBy",
+    "decisionVocabulary",
+    "subjects",
+    "metrics",
+    "contract",
+    "scenarios"
+  ],
+  "properties": {
+    "manifestVersion": { "type": "string", "minLength": 1 },
+    "suiteId": { "type": "string", "minLength": 1 },
+    "suiteName": { "type": "string", "minLength": 1 },
+    "generatedBy": { "type": "string", "minLength": 1 },
+    "decisionVocabulary": {
+      "type": "array",
+      "minItems": 3,
+      "items": { "enum": ["allow", "warn", "block"] },
+      "contains": { "const": "allow" }
+    },
+    "subjects": {
+      "type": "array",
+      "minItems": 5,
+      "items": { "$ref": "#/$defs/subject" }
+    },
+    "metrics": {
+      "type": "array",
+      "minItems": 9,
+      "items": {
+        "enum": [
+          "fullContractPassRate",
+          "decisionAccuracy",
+          "preventionRate",
+          "falseBlockRate",
+          "evidenceRecall",
+          "redactionLeaks",
+          "recallDegradationDetectionRate",
+          "latencyP50Ms",
+          "latencyP95Ms"
+        ]
+      }
+    },
+    "contract": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["fullContractPass", "externalAdapterRule"],
+      "properties": {
+        "fullContractPass": { "type": "string", "minLength": 1 },
+        "externalAdapterRule": { "type": "string", "minLength": 1 }
+      }
+    },
+    "scenarios": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/scenario" }
+    }
+  },
+  "$defs": {
+    "subject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "name", "description", "external"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "name": { "type": "string", "minLength": 1 },
+        "description": { "type": "string", "minLength": 1 },
+        "external": { "type": "boolean" }
+      }
+    },
+    "scenario": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "name",
+        "expectedDecision",
+        "action",
+        "seed",
+        "expectedEvidenceClass",
+        "requiredEvidence",
+        "redactionSecretsSeeded"
+      ],
+      "properties": {
+        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
+        "name": { "type": "string", "minLength": 1 },
+        "expectedDecision": { "enum": ["allow", "warn", "block"] },
+        "action": { "$ref": "#/$defs/action" },
+        "seed": { "$ref": "#/$defs/seed" },
+        "expectedEvidenceClass": { "type": "string", "minLength": 1 },
+        "requiredEvidence": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "redactionSecretsSeeded": { "type": "integer", "minimum": 0 }
+      }
+    },
+    "action": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["tool", "action"],
+      "properties": {
+        "tool": { "type": "string", "minLength": 1 },
+        "action": { "type": "string", "minLength": 1 },
+        "command": { "type": "string", "minLength": 1 },
+        "cwd": { "type": "string" },
+        "files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        }
+      }
+    },
+    "seed": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "seededMemories",
+        "seededToolEvents",
+        "seededNoise",
+        "faultInjection",
+        "seededSecretRefs"
+      ],
+      "properties": {
+        "seededMemories": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/memorySeed" }
+        },
+        "seededToolEvents": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/toolEventSeed" }
+        },
+        "seededNoise": {
+          "anyOf": [
+            { "type": "null" },
+            { "$ref": "#/$defs/noiseSeed" }
+          ]
+        },
+        "faultInjection": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "string", "minLength": 1 }
+          ]
+        },
+        "seededSecretRefs": {
+          "type": "array",
+          "items": { "$ref": "#/$defs/secretRef" }
+        }
+      }
+    },
+    "memorySeed": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["content", "source", "tags", "salience"],
+      "properties": {
+        "content": { "type": "string", "minLength": 1 },
+        "source": { "type": "string", "minLength": 1 },
+        "tags": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "salience": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "toolEventSeed": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["tool"],
+      "properties": {
+        "tool": { "type": "string", "minLength": 1 },
+        "action": { "type": "string", "minLength": 1 },
+        "files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "outcome": { "type": "string", "minLength": 1 },
+        "errorSummary": { "type": "string", "minLength": 1 },
+        "errorSummaryPattern": { "type": "string", "minLength": 1 },
+        "output": { "type": "string", "minLength": 1 }
+      }
+    },
+    "noiseSeed": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["count", "source", "tags", "salience"],
+      "properties": {
+        "count": { "type": "integer", "minimum": 0 },
+        "source": { "type": "string", "minLength": 1 },
+        "tags": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "salience": { "type": "number", "minimum": 0, "maximum": 1 }
+      }
+    },
+    "secretRef": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "fingerprint", "length"],
+      "properties": {
+        "id": { "type": "string", "minLength": 1 },
+        "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{16}$" },
+        "length": { "type": "integer", "minimum": 1 }
+      }
+    }
+  }
+}

package/benchmarks/schemas/guardbench-publication-verification.schema.json

@@ -0,0 +1,47 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-publication-verification.schema.json",
+  "title": "GuardBench Publication Artifact Verification",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "suite", "generatedAt", "ok", "checks", "failures"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench publication artifact verification" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "ok": { "type": "boolean" },
+    "checks": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["registry", "adapterModule", "selfTest", "artifacts", "bundle", "externalDryRun", "externalEvidence", "leaderboard", "localPaths"],
+      "properties": {
+        "registry": { "$ref": "#/$defs/basicCheck" },
+        "adapterModule": { "$ref": "#/$defs/basicCheck" },
+        "selfTest": { "$ref": "#/$defs/basicCheck" },
+        "artifacts": { "$ref": "#/$defs/basicCheck" },
+        "bundle": { "$ref": "#/$defs/basicCheck" },
+        "externalDryRun": { "$ref": "#/$defs/basicCheck" },
+        "externalEvidence": { "$ref": "#/$defs/basicCheck" },
+        "leaderboard": { "$ref": "#/$defs/basicCheck" },
+        "localPaths": { "$ref": "#/$defs/basicCheck" }
+      }
+    },
+    "failures": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "$defs": {
+    "basicCheck": {
+      "type": "object",
+      "required": ["ok", "failures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    }
+  }
+}