audrey 0.23.1 → 1.0.1

package/benchmarks/output/submission-bundle/schemas/guardbench-adapter-registry.schema.json

@@ -0,0 +1,69 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-adapter-registry.schema.json",
+  "title": "GuardBench Adapter Registry",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "suite", "adapters"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench adapter registry" },
+    "adapters": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/adapter" }
+    }
+  },
+  "$defs": {
+    "adapter": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "name",
+        "path",
+        "status",
+        "credentialMode",
+        "requiredEnv",
+        "description",
+        "commands"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "name": { "type": "string", "minLength": 1 },
+        "path": {
+          "type": "string",
+          "pattern": "^benchmarks/adapters/.+\\.mjs$"
+        },
+        "status": {
+          "enum": ["reference", "external-system", "experimental"]
+        },
+        "credentialMode": {
+          "enum": ["none", "runtime-env"]
+        },
+        "requiredEnv": {
+          "type": "array",
+          "items": {
+            "type": "string",
+            "pattern": "^[A-Z][A-Z0-9_]*$"
+          }
+        },
+        "description": { "type": "string", "minLength": 1 },
+        "commands": {
+          "type": "object",
+          "additionalProperties": false,
+          "required": ["moduleValidate", "selfTest", "selfTestValidate", "externalRun"],
+          "properties": {
+            "moduleValidate": { "type": "string", "minLength": 1 },
+            "selfTest": { "type": "string", "minLength": 1 },
+            "selfTestValidate": { "type": "string", "minLength": 1 },
+            "externalRun": { "type": "string", "minLength": 1 }
+          }
+        }
+      }
+    }
+  }
+}

package/benchmarks/output/submission-bundle/schemas/guardbench-adapter-self-test.schema.json

@@ -0,0 +1,156 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-adapter-self-test.schema.json",
+  "title": "GuardBench Adapter Self-Test",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "suite",
+    "generatedAt",
+    "ok",
+    "adapter",
+    "conformance",
+    "score",
+    "contract",
+    "failures"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench adapter self-test" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "ok": { "type": "boolean" },
+    "adapter": { "$ref": "#/$defs/adapter" },
+    "conformance": { "$ref": "#/$defs/conformance" },
+    "score": { "$ref": "#/$defs/score" },
+    "contract": { "$ref": "#/$defs/contract" },
+    "failures": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "$defs": {
+    "adapter": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "path", "moduleFile", "description"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "path": { "type": "string", "minLength": 1 },
+        "moduleFile": { "type": "string", "minLength": 1 },
+        "description": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "conformance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "ok",
+        "adapter",
+        "requestedAdapter",
+        "scenarios",
+        "expectedScenarios",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "redactionLeaks",
+        "failures"
+      ],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "adapter": { "type": "string", "minLength": 1 },
+        "requestedAdapter": { "type": "string", "minLength": 1 },
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "expectedScenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
+        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
+        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "score": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "scenarios",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "evidenceRecall",
+        "redactionLeaks",
+        "latency"
+      ],
+      "properties": {
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
+        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
+        "evidenceRecall": { "$ref": "#/$defs/nullableRate" },
+        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
+        "latency": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["p50Ms", "p95Ms", "maxMs"],
+              "properties": {
+                "p50Ms": { "type": "number", "minimum": 0 },
+                "p95Ms": { "type": "number", "minimum": 0 },
+                "maxMs": { "type": "number", "minimum": 0 }
+              }
+            },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "contract": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "expectedAnswersWithheld",
+        "lowScoreAllowed",
+        "requiredScenarioRows",
+        "requiredResultFields",
+        "redactionLeakTolerance"
+      ],
+      "properties": {
+        "expectedAnswersWithheld": { "const": true },
+        "lowScoreAllowed": { "const": true },
+        "requiredScenarioRows": { "type": "integer", "minimum": 1 },
+        "requiredResultFields": {
+          "type": "array",
+          "minItems": 6,
+          "items": {
+            "enum": [
+              "decision",
+              "riskScore",
+              "evidenceIds",
+              "recommendedActions",
+              "summary",
+              "recallErrors"
+            ]
+          }
+        },
+        "redactionLeakTolerance": { "const": 0 }
+      }
+    },
+    "nullableNumber": {
+      "anyOf": [
+        { "type": "number", "minimum": 0 },
+        { "type": "null" }
+      ]
+    },
+    "nullableRate": {
+      "anyOf": [
+        { "type": "number", "minimum": 0, "maximum": 1 },
+        { "type": "null" }
+      ]
+    }
+  }
+}

package/benchmarks/output/submission-bundle/schemas/guardbench-conformance-card.schema.json

@@ -0,0 +1,184 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-conformance-card.schema.json",
+  "title": "GuardBench Conformance Card",
+  "type": "object",
+  "additionalProperties": false,
+  "required": [
+    "schemaVersion",
+    "suite",
+    "generatedAt",
+    "sourceDir",
+    "manifestVersion",
+    "suiteId",
+    "subject",
+    "run",
+    "score",
+    "conformance",
+    "integrity",
+    "provenance"
+  ],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench conformance card" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "sourceDir": { "type": "string", "minLength": 1 },
+    "manifestVersion": { "type": "string", "minLength": 1 },
+    "suiteId": { "type": "string", "minLength": 1 },
+    "subject": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["name", "requestedAdapter", "external"],
+      "properties": {
+        "name": { "type": "string", "minLength": 1 },
+        "requestedAdapter": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "external": { "type": "boolean" }
+      }
+    },
+    "run": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["status", "startedAt", "completedAt", "command", "validationCommand"],
+      "properties": {
+        "status": { "type": "string", "minLength": 1 },
+        "startedAt": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "completedAt": {
+          "anyOf": [
+            { "type": "string", "minLength": 1 },
+            { "type": "null" }
+          ]
+        },
+        "command": {
+          "anyOf": [
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": { "type": "string", "minLength": 1 }
+            },
+            { "type": "null" }
+          ]
+        },
+        "validationCommand": {
+          "anyOf": [
+            {
+              "type": "array",
+              "minItems": 1,
+              "items": { "type": "string", "minLength": 1 }
+            },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "score": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "scenarios",
+        "fullContractPassed",
+        "fullContractPassRate",
+        "decisionAccuracy",
+        "evidenceRecall",
+        "redactionLeaks",
+        "latency"
+      ],
+      "properties": {
+        "scenarios": { "type": "integer", "minimum": 0 },
+        "fullContractPassed": { "$ref": "#/$defs/nullableNumber" },
+        "fullContractPassRate": { "$ref": "#/$defs/nullableRate" },
+        "decisionAccuracy": { "$ref": "#/$defs/nullableRate" },
+        "evidenceRecall": { "$ref": "#/$defs/nullableRate" },
+        "redactionLeaks": { "$ref": "#/$defs/nullableNumber" },
+        "latency": {
+          "anyOf": [
+            {
+              "type": "object",
+              "additionalProperties": false,
+              "required": ["p50Ms", "p95Ms", "maxMs"],
+              "properties": {
+                "p50Ms": { "type": "number", "minimum": 0 },
+                "p95Ms": { "type": "number", "minimum": 0 },
+                "maxMs": { "type": "number", "minimum": 0 }
+              }
+            },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "conformance": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["ok", "failures", "artifactValidationOk", "artifactValidationFailures"],
+      "properties": {
+        "ok": { "type": "boolean" },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        },
+        "artifactValidationOk": { "type": "boolean" },
+        "artifactValidationFailures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    },
+    "integrity": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": ["artifactHashes", "externalRunMetadataHash"],
+      "properties": {
+        "artifactHashes": { "$ref": "#/$defs/artifactHashes" },
+        "externalRunMetadataHash": {
+          "anyOf": [
+            { "$ref": "#/$defs/sha256" },
+            { "type": "null" }
+          ]
+        }
+      }
+    },
+    "provenance": { "type": "object" }
+  },
+  "$defs": {
+    "artifactHashes": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "guardbench-manifest.json",
+        "guardbench-summary.json",
+        "guardbench-raw.json"
+      ],
+      "properties": {
+        "guardbench-manifest.json": { "$ref": "#/$defs/sha256" },
+        "guardbench-summary.json": { "$ref": "#/$defs/sha256" },
+        "guardbench-raw.json": { "$ref": "#/$defs/sha256" }
+      }
+    },
+    "nullableNumber": {
+      "anyOf": [
+        { "type": "number", "minimum": 0 },
+        { "type": "null" }
+      ]
+    },
+    "nullableRate": {
+      "anyOf": [
+        { "type": "number", "minimum": 0, "maximum": 1 },
+        { "type": "null" }
+      ]
+    },
+    "sha256": {
+      "type": "string",
+      "pattern": "^[a-f0-9]{64}$"
+    }
+  }
+}

package/benchmarks/output/submission-bundle/schemas/guardbench-external-dry-run.schema.json

@@ -0,0 +1,74 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-external-dry-run.schema.json",
+  "title": "GuardBench External Adapter Dry-Run Matrix",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "suite", "generatedAt", "ok", "registry", "outRoot", "adapters", "failures"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench external adapter dry-run matrix" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "ok": { "type": "boolean" },
+    "registry": { "type": "string", "minLength": 1 },
+    "outRoot": { "type": "string", "minLength": 1 },
+    "adapters": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/adapterDryRun" }
+    },
+    "failures": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "$defs": {
+    "adapterDryRun": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "name",
+        "credentialMode",
+        "requiredEnv",
+        "missingEnv",
+        "status",
+        "command",
+        "validationCommand",
+        "metadataPath"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "name": { "type": "string", "minLength": 1 },
+        "credentialMode": {
+          "enum": ["none", "runtime-env"]
+        },
+        "requiredEnv": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "missingEnv": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "status": {
+          "enum": ["dry-run-missing-env", "dry-run-ready"]
+        },
+        "command": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "validationCommand": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "metadataPath": { "type": "string", "minLength": 1 }
+      }
+    }
+  }
+}

package/benchmarks/output/submission-bundle/schemas/guardbench-external-evidence.schema.json

@@ -0,0 +1,108 @@
+{
+  "$schema": "https://json-schema.org/draft/2020-12/schema",
+  "$id": "https://audrey-memory.org/schemas/guardbench-external-evidence.schema.json",
+  "title": "GuardBench External Evidence Verification",
+  "type": "object",
+  "additionalProperties": false,
+  "required": ["schemaVersion", "suite", "generatedAt", "ok", "allowPending", "registry", "outRoot", "adapters", "failures"],
+  "properties": {
+    "schemaVersion": { "const": "1.0.0" },
+    "suite": { "const": "GuardBench external evidence verification" },
+    "generatedAt": { "type": "string", "minLength": 1 },
+    "ok": { "type": "boolean" },
+    "allowPending": { "type": "boolean" },
+    "registry": { "type": "string", "minLength": 1 },
+    "outRoot": { "type": "string", "minLength": 1 },
+    "adapters": {
+      "type": "array",
+      "minItems": 1,
+      "items": { "$ref": "#/$defs/adapterEvidence" }
+    },
+    "failures": {
+      "type": "array",
+      "items": { "type": "string" }
+    }
+  },
+  "$defs": {
+    "adapterEvidence": {
+      "type": "object",
+      "additionalProperties": false,
+      "required": [
+        "id",
+        "name",
+        "path",
+        "credentialMode",
+        "requiredEnv",
+        "outDir",
+        "metadataPath",
+        "status",
+        "evidenceKind",
+        "metadataStatus",
+        "dryRun",
+        "missingEnv",
+        "artifactValidationOk",
+        "adapterConformanceOk",
+        "secretLeakCount",
+        "failures"
+      ],
+      "properties": {
+        "id": {
+          "type": "string",
+          "pattern": "^[a-z0-9][a-z0-9-]*$"
+        },
+        "name": { "type": "string", "minLength": 1 },
+        "path": { "type": "string", "minLength": 1 },
+        "credentialMode": { "const": "runtime-env" },
+        "requiredEnv": {
+          "type": "array",
+          "minItems": 1,
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "outDir": { "type": "string", "minLength": 1 },
+        "metadataPath": { "type": "string", "minLength": 1 },
+        "status": {
+          "enum": ["failed", "pending", "verified"]
+        },
+        "evidenceKind": {
+          "enum": ["blocked", "dry-run", "live", "missing"]
+        },
+        "metadataStatus": {
+          "anyOf": [
+            { "type": "null" },
+            { "type": "string", "minLength": 1 }
+          ]
+        },
+        "dryRun": {
+          "anyOf": [
+            { "type": "boolean" },
+            { "type": "null" }
+          ]
+        },
+        "missingEnv": {
+          "type": "array",
+          "items": { "type": "string", "minLength": 1 }
+        },
+        "artifactValidationOk": {
+          "anyOf": [
+            { "type": "boolean" },
+            { "type": "null" }
+          ]
+        },
+        "adapterConformanceOk": {
+          "anyOf": [
+            { "type": "boolean" },
+            { "type": "null" }
+          ]
+        },
+        "secretLeakCount": {
+          "type": "integer",
+          "minimum": 0
+        },
+        "failures": {
+          "type": "array",
+          "items": { "type": "string" }
+        }
+      }
+    }
+  }
+}