audrey 1.1.2 → 1.2.0

package/benchmarks/output/submission-bundle/schemas/guardbench-external-run.schema.json

@@ -1,160 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://audrey-memory.org/schemas/guardbench-external-run.schema.json",
-  "title": "GuardBench External Run Metadata",
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "suite",
-    "startedAt",
-    "adapter",
-    "adapterPath",
-    "outDir",
-    "requiredEnv",
-    "missingEnv",
-    "command",
-    "validationCommand",
-    "dryRun",
-    "status"
-  ],
-  "properties": {
-    "suite": { "const": "GuardBench external adapter run" },
-    "startedAt": { "type": "string", "minLength": 1 },
-    "completedAt": { "type": "string", "minLength": 1 },
-    "adapter": { "type": "string", "minLength": 1 },
-    "adapterPath": { "type": "string", "minLength": 1 },
-    "outDir": { "type": "string", "minLength": 1 },
-    "requiredEnv": {
-      "type": "array",
-      "items": { "type": "string", "minLength": 1 }
-    },
-    "missingEnv": {
-      "type": "array",
-      "items": { "type": "string", "minLength": 1 }
-    },
-    "command": {
-      "type": "array",
-      "minItems": 1,
-      "items": { "type": "string", "minLength": 1 }
-    },
-    "validationCommand": {
-      "type": "array",
-      "minItems": 1,
-      "items": { "type": "string", "minLength": 1 }
-    },
-    "dryRun": { "type": "boolean" },
-    "status": {
-      "enum": [
-        "blocked",
-        "dry-run-missing-env",
-        "dry-run-ready",
-        "failed",
-        "passed",
-        "running"
-      ]
-    },
-    "blockReason": { "type": "string", "minLength": 1 },
-    "exitCode": {
-      "anyOf": [
-        { "type": "integer" },
-        { "type": "null" }
-      ]
-    },
-    "signal": {
-      "anyOf": [
-        { "type": "string", "minLength": 1 },
-        { "type": "null" }
-      ]
-    },
-    "artifactHashes": { "$ref": "#/$defs/artifactHashes" },
-    "artifactValidation": { "$ref": "#/$defs/artifactValidation" },
-    "adapterConformance": { "$ref": "#/$defs/adapterConformance" }
-  },
-  "$defs": {
-    "artifactHashes": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "guardbench-manifest.json",
-        "guardbench-summary.json",
-        "guardbench-raw.json"
-      ],
-      "properties": {
-        "guardbench-manifest.json": { "$ref": "#/$defs/sha256" },
-        "guardbench-summary.json": { "$ref": "#/$defs/sha256" },
-        "guardbench-raw.json": { "$ref": "#/$defs/sha256" }
-      }
-    },
-    "sha256": {
-      "type": "string",
-      "pattern": "^[a-f0-9]{64}$"
-    },
-    "artifactValidation": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["ok", "dir", "schemasDir", "files", "failures"],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "dir": { "type": "string", "minLength": 1 },
-        "schemasDir": { "type": "string", "minLength": 1 },
-        "files": {
-          "type": "array",
-          "minItems": 3,
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "optionalFiles": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    },
-    "adapterConformance": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "ok",
-        "adapter",
-        "requestedAdapter",
-        "scenarios",
-        "expectedScenarios",
-        "fullContractPassRate",
-        "decisionAccuracy",
-        "redactionLeaks",
-        "failures"
-      ],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "adapter": { "type": "string", "minLength": 1 },
-        "requestedAdapter": { "type": "string", "minLength": 1 },
-        "scenarios": { "type": "integer", "minimum": 0 },
-        "expectedScenarios": { "type": "integer", "minimum": 0 },
-        "fullContractPassRate": {
-          "anyOf": [
-            { "type": "null" },
-            { "type": "number", "minimum": 0, "maximum": 1 }
-          ]
-        },
-        "decisionAccuracy": {
-          "anyOf": [
-            { "type": "null" },
-            { "type": "number", "minimum": 0, "maximum": 1 }
-          ]
-        },
-        "redactionLeaks": {
-          "anyOf": [
-            { "type": "null" },
-            { "type": "integer", "minimum": 0 }
-          ]
-        },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    }
-  }
-}

package/benchmarks/output/submission-bundle/schemas/guardbench-leaderboard.schema.json

@@ -1,179 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://audrey-memory.org/schemas/guardbench-leaderboard.schema.json",
-  "title": "GuardBench Leaderboard",
-  "type": "object",
-  "additionalProperties": false,
-  "required": ["schemaVersion", "suite", "generatedAt", "ranking", "rows", "failures"],
-  "properties": {
-    "schemaVersion": { "const": "1.0.0" },
-    "suite": { "const": "GuardBench leaderboard" },
-    "generatedAt": { "type": "string", "minLength": 1 },
-    "ranking": {
-      "type": "array",
-      "minItems": 1,
-      "items": { "type": "string", "minLength": 1 }
-    },
-    "rows": {
-      "type": "array",
-      "items": { "$ref": "#/$defs/row" }
-    },
-    "failures": {
-      "type": "array",
-      "items": { "type": "string" }
-    }
-  },
-  "$defs": {
-    "row": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "rank",
-        "subject",
-        "score",
-        "conformance",
-        "source",
-        "verification"
-      ],
-      "properties": {
-        "rank": { "type": "integer", "minimum": 1 },
-        "subject": { "$ref": "#/$defs/subject" },
-        "score": { "$ref": "#/$defs/score" },
-        "conformance": { "$ref": "#/$defs/conformance" },
-        "source": { "$ref": "#/$defs/source" },
-        "verification": { "$ref": "#/$defs/verification" }
-      }
-    },
-    "subject": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["name", "requestedAdapter", "external"],
-      "properties": {
-        "name": { "type": "string", "minLength": 1 },
-        "requestedAdapter": {
-          "anyOf": [
-            { "type": "string", "minLength": 1 },
-            { "type": "null" }
-          ]
-        },
-        "external": { "type": "boolean" }
-      }
-    },
-    "score": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "scenarios",
-        "fullContractPassed",
-        "fullContractPassRate",
-        "decisionAccuracy",
-        "evidenceRecall",
-        "redactionLeaks",
-        "latency"
-      ],
-      "properties": {
-        "scenarios": { "type": "integer", "minimum": 0 },
-        "fullContractPassed": { "$ref": "#/$defs/nullableNumber" },
-        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
-        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
-        "evidenceRecall": { "$ref": "#/$defs/nullableRate" },
-        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
-        "latency": {
-          "anyOf": [
-            {
-              "type": "object",
-              "additionalProperties": false,
-              "required": ["p50Ms", "p95Ms", "maxMs"],
-              "properties": {
-                "p50Ms": { "type": "number", "minimum": 0 },
-                "p95Ms": { "type": "number", "minimum": 0 },
-                "maxMs": { "type": "number", "minimum": 0 }
-              }
-            },
-            { "type": "null" }
-          ]
-        }
-      }
-    },
-    "conformance": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["ok", "failures", "artifactValidationOk", "artifactValidationFailures"],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        },
-        "artifactValidationOk": { "type": "boolean" },
-        "artifactValidationFailures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    },
-    "source": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["dir", "manifestGeneratedAt", "fileCount"],
-      "properties": {
-        "dir": { "type": "string", "minLength": 1 },
-        "manifestGeneratedAt": { "type": "string", "minLength": 1 },
-        "fileCount": { "type": "integer", "minimum": 0 }
-      }
-    },
-    "verification": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["ok", "dir", "subject", "files", "artifactValidation", "failures"],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "dir": { "type": "string", "minLength": 1 },
-        "subject": { "$ref": "#/$defs/subject" },
-        "files": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "artifactValidation": { "$ref": "#/$defs/artifactValidation" },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    },
-    "artifactValidation": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["ok", "dir", "schemasDir", "files", "optionalFiles", "failures"],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "dir": { "type": "string", "minLength": 1 },
-        "schemasDir": { "type": "string", "minLength": 1 },
-        "files": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "optionalFiles": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    },
-    "nullableNumber": {
-      "anyOf": [
-        { "type": "number", "minimum": 0 },
-        { "type": "null" }
-      ]
-    },
-    "nullableRate": {
-      "anyOf": [
-        { "type": "number", "minimum": 0, "maximum": 1 },
-        { "type": "null" }
-      ]
-    }
-  }
-}

package/benchmarks/output/submission-bundle/schemas/guardbench-manifest.schema.json

@@ -1,213 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://audrey-memory.org/schemas/guardbench-manifest.schema.json",
-  "title": "GuardBench Manifest",
-  "type": "object",
-  "additionalProperties": false,
-  "required": [
-    "manifestVersion",
-    "suiteId",
-    "suiteName",
-    "generatedBy",
-    "decisionVocabulary",
-    "subjects",
-    "metrics",
-    "contract",
-    "scenarios"
-  ],
-  "properties": {
-    "manifestVersion": { "type": "string", "minLength": 1 },
-    "suiteId": { "type": "string", "minLength": 1 },
-    "suiteName": { "type": "string", "minLength": 1 },
-    "generatedBy": { "type": "string", "minLength": 1 },
-    "decisionVocabulary": {
-      "type": "array",
-      "minItems": 3,
-      "items": { "enum": ["allow", "warn", "block"] },
-      "contains": { "const": "allow" }
-    },
-    "subjects": {
-      "type": "array",
-      "minItems": 5,
-      "items": { "$ref": "#/$defs/subject" }
-    },
-    "metrics": {
-      "type": "array",
-      "minItems": 9,
-      "items": {
-        "enum": [
-          "fullContractPassRate",
-          "decisionAccuracy",
-          "preventionRate",
-          "falseBlockRate",
-          "evidenceRecall",
-          "redactionLeaks",
-          "recallDegradationDetectionRate",
-          "latencyP50Ms",
-          "latencyP95Ms"
-        ]
-      }
-    },
-    "contract": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["fullContractPass", "externalAdapterRule"],
-      "properties": {
-        "fullContractPass": { "type": "string", "minLength": 1 },
-        "externalAdapterRule": { "type": "string", "minLength": 1 }
-      }
-    },
-    "scenarios": {
-      "type": "array",
-      "minItems": 10,
-      "items": { "$ref": "#/$defs/scenario" }
-    }
-  },
-  "$defs": {
-    "subject": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["id", "name", "description", "external"],
-      "properties": {
-        "id": { "type": "string", "minLength": 1 },
-        "name": { "type": "string", "minLength": 1 },
-        "description": { "type": "string", "minLength": 1 },
-        "external": { "type": "boolean" }
-      }
-    },
-    "scenario": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "id",
-        "name",
-        "expectedDecision",
-        "action",
-        "seed",
-        "expectedEvidenceClass",
-        "requiredEvidence",
-        "redactionSecretsSeeded"
-      ],
-      "properties": {
-        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
-        "name": { "type": "string", "minLength": 1 },
-        "expectedDecision": { "enum": ["allow", "warn", "block"] },
-        "action": { "$ref": "#/$defs/action" },
-        "seed": { "$ref": "#/$defs/seed" },
-        "expectedEvidenceClass": { "type": "string", "minLength": 1 },
-        "requiredEvidence": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "redactionSecretsSeeded": { "type": "integer", "minimum": 0 }
-      }
-    },
-    "action": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["tool", "action"],
-      "properties": {
-        "tool": { "type": "string", "minLength": 1 },
-        "action": { "type": "string", "minLength": 1 },
-        "command": { "type": "string", "minLength": 1 },
-        "cwd": { "type": "string" },
-        "files": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        }
-      }
-    },
-    "seed": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": [
-        "seededMemories",
-        "seededToolEvents",
-        "seededNoise",
-        "faultInjection",
-        "seededSecretRefs"
-      ],
-      "properties": {
-        "seededMemories": {
-          "type": "array",
-          "items": { "$ref": "#/$defs/memorySeed" }
-        },
-        "seededToolEvents": {
-          "type": "array",
-          "items": { "$ref": "#/$defs/toolEventSeed" }
-        },
-        "seededNoise": {
-          "anyOf": [
-            { "type": "null" },
-            { "$ref": "#/$defs/noiseSeed" }
-          ]
-        },
-        "faultInjection": {
-          "anyOf": [
-            { "type": "null" },
-            { "type": "string", "minLength": 1 }
-          ]
-        },
-        "seededSecretRefs": {
-          "type": "array",
-          "items": { "$ref": "#/$defs/secretRef" }
-        }
-      }
-    },
-    "memorySeed": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["content", "source", "tags", "salience"],
-      "properties": {
-        "content": { "type": "string", "minLength": 1 },
-        "source": { "type": "string", "minLength": 1 },
-        "tags": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "salience": { "type": "number", "minimum": 0, "maximum": 1 }
-      }
-    },
-    "toolEventSeed": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["tool"],
-      "properties": {
-        "tool": { "type": "string", "minLength": 1 },
-        "action": { "type": "string", "minLength": 1 },
-        "files": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "outcome": { "type": "string", "minLength": 1 },
-        "errorSummary": { "type": "string", "minLength": 1 },
-        "errorSummaryPattern": { "type": "string", "minLength": 1 },
-        "output": { "type": "string", "minLength": 1 }
-      }
-    },
-    "noiseSeed": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["count", "source", "tags", "salience"],
-      "properties": {
-        "count": { "type": "integer", "minimum": 0 },
-        "source": { "type": "string", "minLength": 1 },
-        "tags": {
-          "type": "array",
-          "items": { "type": "string", "minLength": 1 }
-        },
-        "salience": { "type": "number", "minimum": 0, "maximum": 1 }
-      }
-    },
-    "secretRef": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["id", "fingerprint", "length"],
-      "properties": {
-        "id": { "type": "string", "minLength": 1 },
-        "fingerprint": { "type": "string", "pattern": "^[a-f0-9]{16}$" },
-        "length": { "type": "integer", "minimum": 1 }
-      }
-    }
-  }
-}

package/benchmarks/output/submission-bundle/schemas/guardbench-publication-verification.schema.json

@@ -1,47 +0,0 @@
-{
-  "$schema": "https://json-schema.org/draft/2020-12/schema",
-  "$id": "https://audrey-memory.org/schemas/guardbench-publication-verification.schema.json",
-  "title": "GuardBench Publication Artifact Verification",
-  "type": "object",
-  "additionalProperties": false,
-  "required": ["schemaVersion", "suite", "generatedAt", "ok", "checks", "failures"],
-  "properties": {
-    "schemaVersion": { "const": "1.0.0" },
-    "suite": { "const": "GuardBench publication artifact verification" },
-    "generatedAt": { "type": "string", "minLength": 1 },
-    "ok": { "type": "boolean" },
-    "checks": {
-      "type": "object",
-      "additionalProperties": false,
-      "required": ["registry", "adapterModule", "selfTest", "artifacts", "bundle", "externalDryRun", "externalEvidence", "leaderboard", "localPaths"],
-      "properties": {
-        "registry": { "$ref": "#/$defs/basicCheck" },
-        "adapterModule": { "$ref": "#/$defs/basicCheck" },
-        "selfTest": { "$ref": "#/$defs/basicCheck" },
-        "artifacts": { "$ref": "#/$defs/basicCheck" },
-        "bundle": { "$ref": "#/$defs/basicCheck" },
-        "externalDryRun": { "$ref": "#/$defs/basicCheck" },
-        "externalEvidence": { "$ref": "#/$defs/basicCheck" },
-        "leaderboard": { "$ref": "#/$defs/basicCheck" },
-        "localPaths": { "$ref": "#/$defs/basicCheck" }
-      }
-    },
-    "failures": {
-      "type": "array",
-      "items": { "type": "string" }
-    }
-  },
-  "$defs": {
-    "basicCheck": {
-      "type": "object",
-      "required": ["ok", "failures"],
-      "properties": {
-        "ok": { "type": "boolean" },
-        "failures": {
-          "type": "array",
-          "items": { "type": "string" }
-        }
-      }
-    }
-  }
-}