audrey 0.23.1 → 1.0.0

package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-raw.schema.json

@@ -0,0 +1,164 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-raw.schema.json",
+  "title": "GuardBench Raw Output",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "suite",
+    "generatedAt",
+    "manifestVersion",
+    "provenance",
+    "cases",
+    "artifactRedactionSweep"
+  ],
+  "properties": {
+    "suite": { "const": "GuardBench comparative" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "manifestVersion": { "type": "string", "minLength": 1 },
+    "provenance": { "$ref": "#/$defs/provenance" },
+    "cases": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/caseResult" }
+    },
+    "artifactRedactionSweep": { "$ref": "#/$defs/artifactRedactionSweep" }
+  },
+  "$defs": {
+    "provenance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "generatedAt",
+        "gitSha",
+        "gitDirty",
+        "node",
+        "v8",
+        "platform",
+        "arch",
+        "osRelease",
+        "cpuModel",
+        "cpuCount",
+        "totalMemoryGb",
+        "embeddingProvider",
+        "embeddingDimensions",
+        "llmProvider"
+      ],
+      "properties": {
+        "generatedAt": { "type": "string", "minLength": 1 },
+        "gitSha": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "string", "minLength": 7 }
+          ]
+        },
+        "gitDirty": { "type": "boolean" },
+        "node": { "type": "string", "minLength": 1 },
+        "v8": { "type": "string", "minLength": 1 },
+        "platform": { "type": "string", "minLength": 1 },
+        "arch": { "type": "string", "minLength": 1 },
+        "osRelease": { "type": "string", "minLength": 1 },
+        "cpuModel": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "string", "minLength": 1 }
+          ]
+        },
+        "cpuCount": { "type": "integer", "minimum": 1 },
+        "totalMemoryGb": { "type": "number", "minimum": 0 },
+        "embeddingProvider": { "type": "string", "minLength": 1 },
+        "embeddingDimensions": { "type": "integer", "minimum": 1 },
+        "llmProvider": { "type": "string", "minLength": 1 }
+      }
+    },
+    "caseResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "name", "expectedDecision", "results"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
+        "name": { "type": "string", "minLength": 1 },
+        "expectedDecision": { "enum": ["allow", "warn", "block"] },
+        "results": {
+          "type": "array",
+          "minItems": 5,
+          "items": { "$ref": "#/$defs/resultRow" }
+        }
+      }
+    },
+    "resultRow": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "system",
+        "id",
+        "name",
+        "expectedDecision",
+        "decision",
+        "decisionCorrect",
+        "riskScore",
+        "passed",
+        "latencyMs",
+        "evidenceCount",
+        "evidenceIds",
+        "recommendedActions",
+        "summary",
+        "recallErrors",
+        "leakedSecrets",
+        "requiredEvidenceMatched"
+      ],
+      "properties": {
+        "system": { "type": "string", "minLength": 1 },
+        "external": { "type": "boolean" },
+        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
+        "name": { "type": "string", "minLength": 1 },
+        "expectedDecision": { "enum": ["allow", "warn", "block"] },
+        "decision": { "enum": ["allow", "warn", "block"] },
+        "decisionCorrect": { "type": "boolean" },
+        "riskScore": { "type": "number", "minimum": 0, "maximum": 1 },
+        "passed": { "type": "boolean" },
+        "latencyMs": { "type": "number", "minimum": 0 },
+        "evidenceCount": { "type": "integer", "minimum": 0 },
+        "evidenceIds": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "recommendedActions": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "summary": { "type": "string", "minLength": 1 },
+        "recallErrors": { "type": "array" },
+        "leakedSecrets": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "requiredEvidenceMatched": { "type": "boolean" }
+      }
+    },
+    "artifactRedactionSweep": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "checkedAt",
+        "filesChecked",
+        "seededSecretsChecked",
+        "leakCount",
+        "leaks",
+        "passed"
+      ],
+      "properties": {
+        "checkedAt": { "type": "string", "minLength": 1 },
+        "filesChecked": {
+          "type": "array",
+          "minItems": 3,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "seededSecretsChecked": { "type": "integer", "minimum": 0 },
+        "leakCount": { "type": "integer", "minimum": 0 },
+        "leaks": { "type": "array" },
+        "passed": { "type": "boolean" }
+      }
+    }
+  }
+}

package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-submission-manifest.schema.json

@@ -0,0 +1,151 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-submission-manifest.schema.json",
+  "title": "GuardBench Submission Manifest",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "suite",
+    "generatedAt",
+    "sourceDir",
+    "subject",
+    "score",
+    "conformance",
+    "validation",
+    "files"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench submission bundle" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "sourceDir": { "type": "string", "minLength": 1 },
+    "subject": { "$ref": "#/$defs/subject" },
+    "score": { "$ref": "#/$defs/score" },
+    "conformance": { "$ref": "#/$defs/conformance" },
+    "validation": { "$ref": "#/$defs/validation" },
+    "files": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/fileRecord" }
+    }
+  },
+  "$defs": {
+    "subject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "requestedAdapter", "external"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "requestedAdapter": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "external": { "type": "boolean" }
+      }
+    },
+    "score": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "scenarios",
+        "fullContractPassed",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "evidenceRecall",
+        "redactionLeaks",
+        "latency"
+      ],
+      "properties": {
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassed": { "$ref": "#/$defs/nullableNumber" },
+        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
+        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
+        "evidenceRecall": { "$ref": "#/$defs/nullableRate" },
+        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
+        "latency": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["p50Ms", "p95Ms", "maxMs"],
+              "properties": {
+                "p50Ms": { "type": "number", "minimum": 0 },
+                "p95Ms": { "type": "number", "minimum": 0 },
+                "maxMs": { "type": "number", "minimum": 0 }
+              }
+            },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "conformance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "failures", "artifactValidationOk", "artifactValidationFailures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "artifactValidationOk": { "type": "boolean" },
+        "artifactValidationFailures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "validation": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "dir", "schemasDir", "files", "optionalFiles", "failures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "dir": { "type": "string", "minLength": 1 },
+        "schemasDir": { "type": "string", "minLength": 1 },
+        "files": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "optionalFiles": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "fileRecord": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["path", "bytes", "sha256"],
+      "properties": {
+        "path": { "type": "string", "minLength": 1 },
+        "bytes": { "type": "integer", "minimum": 0 },
+        "sha256": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "nullableNumber": {
+      "anyOf": [
+        { "type": "number", "minimum": 0 },
+        { "type": "null" }
+      ]
+    },
+    "nullableRate": {
+      "anyOf": [
+        { "type": "number", "minimum": 0, "maximum": 1 },
+        { "type": "null" }
+      ]
+    },
+    "sha256": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    }
+  }
+}

package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-summary.schema.json

@@ -0,0 +1,228 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-summary.schema.json",
+  "title": "GuardBench Summary",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "suite",
+    "generatedAt",
+    "manifest",
+    "provenance",
+    "subjects",
+    "scenarios",
+    "passed",
+    "passRate",
+    "preventionRate",
+    "falseBlockRate",
+    "decisionAccuracy",
+    "evidenceRecall",
+    "redactionLeaks",
+    "recallDegradationDetectionRate",
+    "latency",
+    "systemSummaries",
+    "comparisons",
+    "rows",
+    "cases",
+    "artifactRedactionSweep"
+  ],
+  "properties": {
+    "suite": { "const": "GuardBench comparative" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "manifest": { "type": "object" },
+    "provenance": { "type": "object" },
+    "subjects": {
+      "type": "array",
+      "minItems": 5,
+      "items": { "type": "string", "minLength": 1 }
+    },
+    "scenarios": { "type": "integer", "minimum": 10 },
+    "passed": { "type": "integer", "minimum": 0 },
+    "passRate": { "type": "number", "minimum": 0, "maximum": 1 },
+    "preventionRate": { "type": "number", "minimum": 0, "maximum": 1 },
+    "falseBlockRate": { "type": "number", "minimum": 0, "maximum": 1 },
+    "decisionAccuracy": { "type": "number", "minimum": 0, "maximum": 1 },
+    "usefulWarningPrecision": {
+      "anyOf": [
+        { "type": "null" },
+        { "type": "number", "minimum": 0, "maximum": 1 }
+      ]
+    },
+    "evidenceRecall": { "type": "number", "minimum": 0, "maximum": 1 },
+    "redactionLeaks": { "type": "integer", "minimum": 0 },
+    "recallDegradationDetectionRate": { "type": "number", "minimum": 0, "maximum": 1 },
+    "latency": { "$ref": "#/$defs/latency" },
+    "systemSummaries": {
+      "type": "array",
+      "minItems": 5,
+      "items": { "$ref": "#/$defs/systemSummary" }
+    },
+    "comparisons": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["bestBaseline", "audreyMarginOverBestBaseline"],
+      "properties": {
+        "bestBaseline": { "$ref": "#/$defs/systemSummary" },
+        "audreyMarginOverBestBaseline": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "number", "minimum": 0, "maximum": 1 }
+          ]
+        }
+      }
+    },
+    "rows": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/resultRow" }
+    },
+    "cases": {
+      "type": "array",
+      "minItems": 10,
+      "items": { "$ref": "#/$defs/caseResult" }
+    },
+    "artifactRedactionSweep": { "$ref": "#/$defs/artifactRedactionSweep" }
+  },
+  "$defs": {
+    "latency": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["p50Ms", "p95Ms", "maxMs"],
+      "properties": {
+        "p50Ms": { "type": "number", "minimum": 0 },
+        "p95Ms": { "type": "number", "minimum": 0 },
+        "maxMs": { "type": "number", "minimum": 0 }
+      }
+    },
+    "systemSummary": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "system",
+        "generatedAt",
+        "scenarios",
+        "passed",
+        "passRate",
+        "decisionCorrect",
+        "decisionAccuracy",
+        "preventionRate",
+        "falseBlockRate",
+        "usefulWarningPrecision",
+        "evidenceRecall",
+        "redactionLeaks",
+        "recallDegradationDetectionRate",
+        "latency"
+      ],
+      "properties": {
+        "system": { "type": "string", "minLength": 1 },
+        "generatedAt": { "type": "string", "minLength": 1 },
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "passed": { "type": "integer", "minimum": 0 },
+        "passRate": { "type": "number", "minimum": 0, "maximum": 1 },
+        "decisionCorrect": { "type": "integer", "minimum": 0 },
+        "decisionAccuracy": { "type": "number", "minimum": 0, "maximum": 1 },
+        "preventionRate": { "type": "number", "minimum": 0, "maximum": 1 },
+        "falseBlockRate": { "type": "number", "minimum": 0, "maximum": 1 },
+        "usefulWarningPrecision": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "number", "minimum": 0, "maximum": 1 }
+          ]
+        },
+        "evidenceRecall": { "type": "number", "minimum": 0, "maximum": 1 },
+        "redactionLeaks": { "type": "integer", "minimum": 0 },
+        "recallDegradationDetectionRate": { "type": "number", "minimum": 0, "maximum": 1 },
+        "latency": { "$ref": "#/$defs/latency" }
+      }
+    },
+    "resultRow": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "system",
+        "id",
+        "name",
+        "expectedDecision",
+        "decision",
+        "decisionCorrect",
+        "riskScore",
+        "passed",
+        "latencyMs",
+        "evidenceCount",
+        "evidenceIds",
+        "recommendedActions",
+        "summary",
+        "recallErrors",
+        "leakedSecrets",
+        "requiredEvidenceMatched"
+      ],
+      "properties": {
+        "system": { "type": "string", "minLength": 1 },
+        "external": { "type": "boolean" },
+        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
+        "name": { "type": "string", "minLength": 1 },
+        "expectedDecision": { "enum": ["allow", "warn", "block"] },
+        "decision": { "enum": ["allow", "warn", "block"] },
+        "decisionCorrect": { "type": "boolean" },
+        "riskScore": { "type": "number", "minimum": 0, "maximum": 1 },
+        "passed": { "type": "boolean" },
+        "latencyMs": { "type": "number", "minimum": 0 },
+        "evidenceCount": { "type": "integer", "minimum": 0 },
+        "evidenceIds": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "recommendedActions": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "summary": { "type": "string", "minLength": 1 },
+        "recallErrors": { "type": "array" },
+        "leakedSecrets": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "requiredEvidenceMatched": { "type": "boolean" }
+      }
+    },
+    "caseResult": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["id", "name", "expectedDecision", "results"],
+      "properties": {
+        "id": { "type": "string", "pattern": "^GB-[0-9]{2}$" },
+        "name": { "type": "string", "minLength": 1 },
+        "expectedDecision": { "enum": ["allow", "warn", "block"] },
+        "results": {
+          "type": "array",
+          "minItems": 5,
+          "items": { "$ref": "#/$defs/resultRow" }
+        }
+      }
+    },
+    "artifactRedactionSweep": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "checkedAt",
+        "filesChecked",
+        "seededSecretsChecked",
+        "leakCount",
+        "leaks",
+        "passed"
+      ],
+      "properties": {
+        "checkedAt": { "type": "string", "minLength": 1 },
+        "filesChecked": {
+          "type": "array",
+          "minItems": 3,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "seededSecretsChecked": { "type": "integer", "minimum": 0 },
+        "leakCount": { "type": "integer", "minimum": 0 },
+        "leaks": { "type": "array" },
+        "passed": { "type": "boolean" }
+      }
+    }
+  }
+}