audrey 1.1.0 → 1.1.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +27 -0
- package/README.md +16 -7
- package/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +4 -4
- package/benchmarks/output/external/guardbench-external-dry-run.json +1 -1
- package/benchmarks/output/external/guardbench-external-evidence.json +1 -1
- package/benchmarks/output/guardbench-conformance-card.json +8 -8
- package/benchmarks/output/guardbench-raw.json +122 -114
- package/benchmarks/output/guardbench-summary.json +205 -189
- package/benchmarks/output/leaderboard/guardbench-leaderboard.json +5 -5
- package/benchmarks/output/leaderboard/guardbench-leaderboard.md +2 -2
- package/benchmarks/output/submission-bundle/guardbench-conformance-card.json +8 -8
- package/benchmarks/output/submission-bundle/guardbench-raw.json +122 -114
- package/benchmarks/output/submission-bundle/guardbench-summary.json +205 -189
- package/benchmarks/output/submission-bundle/submission-manifest.json +10 -10
- package/benchmarks/output/submission-bundle/validation-report.json +1 -1
- package/benchmarks/output/summary.json +44 -44
- package/dist/mcp-server/config.d.ts +1 -1
- package/dist/mcp-server/config.js +1 -1
- package/dist/mcp-server/index.d.ts.map +1 -1
- package/dist/mcp-server/index.js +88 -37
- package/dist/mcp-server/index.js.map +1 -1
- package/dist/src/audrey.d.ts +2 -0
- package/dist/src/audrey.d.ts.map +1 -1
- package/dist/src/audrey.js +2 -0
- package/dist/src/audrey.js.map +1 -1
- package/dist/src/autopilot.d.ts +3 -1
- package/dist/src/autopilot.d.ts.map +1 -1
- package/dist/src/autopilot.js +150 -37
- package/dist/src/autopilot.js.map +1 -1
- package/dist/src/capsule.d.ts +2 -0
- package/dist/src/capsule.d.ts.map +1 -1
- package/dist/src/capsule.js +43 -3
- package/dist/src/capsule.js.map +1 -1
- package/dist/src/controller.d.ts.map +1 -1
- package/dist/src/controller.js +7 -3
- package/dist/src/controller.js.map +1 -1
- package/dist/src/events.d.ts +23 -6
- package/dist/src/events.d.ts.map +1 -1
- package/dist/src/events.js +80 -13
- package/dist/src/events.js.map +1 -1
- package/dist/src/preflight.d.ts.map +1 -1
- package/dist/src/preflight.js +19 -6
- package/dist/src/preflight.js.map +1 -1
- package/dist/src/project.d.ts +17 -0
- package/dist/src/project.d.ts.map +1 -0
- package/dist/src/project.js +56 -0
- package/dist/src/project.js.map +1 -0
- package/dist/src/redact.d.ts.map +1 -1
- package/dist/src/redact.js +20 -0
- package/dist/src/redact.js.map +1 -1
- package/docs/paper/07-evaluation.md +5 -5
- package/docs/paper/audrey-paper-v1.md +5 -5
- package/docs/paper/evidence-ledger.md +1 -1
- package/docs/paper/output/arxiv/arxiv-manifest.json +4 -4
- package/docs/paper/output/arxiv/main.tex +5 -5
- package/docs/paper/output/arxiv-compile-report.json +3 -3
- package/docs/paper/output/submission-bundle/README.md +16 -7
- package/docs/paper/output/submission-bundle/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +4 -4
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-dry-run.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-evidence.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-conformance-card.json +8 -8
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-raw.json +122 -114
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-summary.json +205 -189
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.json +5 -5
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.md +2 -2
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/submission-manifest.json +10 -10
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/validation-report.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/summary.json +42 -42
- package/docs/paper/output/submission-bundle/docs/paper/07-evaluation.md +5 -5
- package/docs/paper/output/submission-bundle/docs/paper/audrey-paper-v1.md +5 -5
- package/docs/paper/output/submission-bundle/docs/paper/evidence-ledger.md +1 -1
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/arxiv-manifest.json +4 -4
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/main.tex +5 -5
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv-compile-report.json +3 -3
- package/docs/paper/output/submission-bundle/package.json +8 -1
- package/docs/paper/output/submission-bundle/paper-submission-manifest.json +31 -31
- package/package.json +8 -1
- package/scripts/audit-release-completion.mjs +1 -1
- package/scripts/finalize-release.mjs +1 -1
- package/scripts/prepare-release-cut.mjs +1 -1
- package/scripts/publish-release-bundle.mjs +1 -1
- package/scripts/publish-release-github-api.mjs +1 -1
- package/scripts/verify-release-readiness.mjs +1 -1
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "GuardBench leaderboard",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T14:48:50.016Z",
|
|
5
5
|
"ranking": [
|
|
6
6
|
"verified bundle",
|
|
7
7
|
"adapter conformance",
|
|
@@ -28,9 +28,9 @@
|
|
|
28
28
|
"evidenceRecall": 1,
|
|
29
29
|
"redactionLeaks": 0,
|
|
30
30
|
"latency": {
|
|
31
|
-
"p50Ms":
|
|
32
|
-
"p95Ms":
|
|
33
|
-
"maxMs":
|
|
31
|
+
"p50Ms": 4.253,
|
|
32
|
+
"p95Ms": 21.196,
|
|
33
|
+
"maxMs": 21.196
|
|
34
34
|
}
|
|
35
35
|
},
|
|
36
36
|
"conformance": {
|
|
@@ -41,7 +41,7 @@
|
|
|
41
41
|
},
|
|
42
42
|
"source": {
|
|
43
43
|
"dir": "benchmarks/output/submission-bundle",
|
|
44
|
-
"manifestGeneratedAt": "2026-07-
|
|
44
|
+
"manifestGeneratedAt": "2026-07-13T14:48:49.680Z",
|
|
45
45
|
"fileCount": 17
|
|
46
46
|
},
|
|
47
47
|
"verification": {
|
package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.md
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
# GuardBench Leaderboard
|
|
2
2
|
|
|
3
|
-
Generated: 2026-07-
|
|
3
|
+
Generated: 2026-07-13T14:48:50.016Z
|
|
4
4
|
|
|
5
5
|
| Rank | Subject | Verified | Conformant | Full Contract | Decision Accuracy | Evidence Recall | Redaction Leaks | p95 Latency | Bundle |
|
|
6
6
|
|---:|---|---:|---:|---:|---:|---:|---:|---:|---|
|
|
7
|
-
| 1 | Audrey Guard | yes | yes | 100.0% | 100.0% | 100.0% | 0 |
|
|
7
|
+
| 1 | Audrey Guard | yes | yes | 100.0% | 100.0% | 100.0% | 0 | 21.196ms | benchmarks/output/submission-bundle |
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "GuardBench submission bundle",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T14:48:49.680Z",
|
|
5
5
|
"sourceDir": "benchmarks/output",
|
|
6
6
|
"subject": {
|
|
7
7
|
"name": "Audrey Guard",
|
|
@@ -16,9 +16,9 @@
|
|
|
16
16
|
"evidenceRecall": 1,
|
|
17
17
|
"redactionLeaks": 0,
|
|
18
18
|
"latency": {
|
|
19
|
-
"p50Ms":
|
|
20
|
-
"p95Ms":
|
|
21
|
-
"maxMs":
|
|
19
|
+
"p50Ms": 4.253,
|
|
20
|
+
"p95Ms": 21.196,
|
|
21
|
+
"maxMs": 21.196
|
|
22
22
|
}
|
|
23
23
|
},
|
|
24
24
|
"conformance": {
|
|
@@ -45,7 +45,7 @@
|
|
|
45
45
|
{
|
|
46
46
|
"path": "guardbench-conformance-card.json",
|
|
47
47
|
"bytes": 1735,
|
|
48
|
-
"sha256": "
|
|
48
|
+
"sha256": "8613dfacbf592e7d409b38f7c92ffa188bc5e8dc18664fc338b4de2ad80a9f6a"
|
|
49
49
|
},
|
|
50
50
|
{
|
|
51
51
|
"path": "guardbench-manifest.json",
|
|
@@ -54,13 +54,13 @@
|
|
|
54
54
|
},
|
|
55
55
|
{
|
|
56
56
|
"path": "guardbench-raw.json",
|
|
57
|
-
"bytes":
|
|
58
|
-
"sha256": "
|
|
57
|
+
"bytes": 43837,
|
|
58
|
+
"sha256": "786b03ff8570444620c23b35621bbb14897f06310de7e8bd3a337a1acf5ce776"
|
|
59
59
|
},
|
|
60
60
|
{
|
|
61
61
|
"path": "guardbench-summary.json",
|
|
62
|
-
"bytes":
|
|
63
|
-
"sha256": "
|
|
62
|
+
"bytes": 70085,
|
|
63
|
+
"sha256": "3ba1c059558aa8b511ae510276078930649ba6aca9669324148726fac10c7ee8"
|
|
64
64
|
},
|
|
65
65
|
{
|
|
66
66
|
"path": "schemas/guardbench-adapter-registry.schema.json",
|
|
@@ -125,7 +125,7 @@
|
|
|
125
125
|
{
|
|
126
126
|
"path": "validation-report.json",
|
|
127
127
|
"bytes": 739,
|
|
128
|
-
"sha256": "
|
|
128
|
+
"sha256": "b8358a7b2833672d06b44809aff428211b9936e7e91a913b45d49248b66de701"
|
|
129
129
|
}
|
|
130
130
|
]
|
|
131
131
|
}
|
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
{
|
|
2
|
-
"generatedAt": "2026-07-
|
|
2
|
+
"generatedAt": "2026-07-13T14:48:45.265Z",
|
|
3
3
|
"command": "node benchmarks/run.js --provider mock --dimensions 64",
|
|
4
4
|
"config": {
|
|
5
5
|
"provider": "mock",
|
|
@@ -23,19 +23,19 @@
|
|
|
23
23
|
"system": "Audrey",
|
|
24
24
|
"scorePercent": 100,
|
|
25
25
|
"passRate": 100,
|
|
26
|
-
"avgDurationMs": 16.
|
|
26
|
+
"avgDurationMs": 16.583333333333332
|
|
27
27
|
},
|
|
28
28
|
{
|
|
29
29
|
"system": "Vector Only",
|
|
30
30
|
"scorePercent": 41.66666666666667,
|
|
31
31
|
"passRate": 25,
|
|
32
|
-
"avgDurationMs": 0.
|
|
32
|
+
"avgDurationMs": 0.16666666666666666
|
|
33
33
|
},
|
|
34
34
|
{
|
|
35
35
|
"system": "Keyword + Recency",
|
|
36
36
|
"scorePercent": 41.66666666666667,
|
|
37
37
|
"passRate": 25,
|
|
38
|
-
"avgDurationMs": 0.
|
|
38
|
+
"avgDurationMs": 0.75
|
|
39
39
|
},
|
|
40
40
|
{
|
|
41
41
|
"system": "Recent Window",
|
|
@@ -170,7 +170,7 @@
|
|
|
170
170
|
"system": "Audrey",
|
|
171
171
|
"scorePercent": 100,
|
|
172
172
|
"passRate": 100,
|
|
173
|
-
"avgDurationMs": 16.
|
|
173
|
+
"avgDurationMs": 16.625
|
|
174
174
|
},
|
|
175
175
|
{
|
|
176
176
|
"system": "Vector Only",
|
|
@@ -182,7 +182,7 @@
|
|
|
182
182
|
"system": "Keyword + Recency",
|
|
183
183
|
"scorePercent": 50,
|
|
184
184
|
"passRate": 37.5,
|
|
185
|
-
"avgDurationMs":
|
|
185
|
+
"avgDurationMs": 1
|
|
186
186
|
},
|
|
187
187
|
{
|
|
188
188
|
"system": "Recent Window",
|
|
@@ -277,7 +277,7 @@
|
|
|
277
277
|
"results": [
|
|
278
278
|
{
|
|
279
279
|
"system": "Audrey",
|
|
280
|
-
"durationMs":
|
|
280
|
+
"durationMs": 26,
|
|
281
281
|
"passed": true,
|
|
282
282
|
"score": 1,
|
|
283
283
|
"summary": "retrieved expected evidence",
|
|
@@ -289,7 +289,7 @@
|
|
|
289
289
|
},
|
|
290
290
|
{
|
|
291
291
|
"system": "Vector Only",
|
|
292
|
-
"durationMs":
|
|
292
|
+
"durationMs": 0,
|
|
293
293
|
"passed": true,
|
|
294
294
|
"score": 1,
|
|
295
295
|
"summary": "retrieved expected evidence",
|
|
@@ -301,7 +301,7 @@
|
|
|
301
301
|
},
|
|
302
302
|
{
|
|
303
303
|
"system": "Keyword + Recency",
|
|
304
|
-
"durationMs":
|
|
304
|
+
"durationMs": 8,
|
|
305
305
|
"passed": true,
|
|
306
306
|
"score": 1,
|
|
307
307
|
"summary": "retrieved expected evidence",
|
|
@@ -347,7 +347,7 @@
|
|
|
347
347
|
},
|
|
348
348
|
{
|
|
349
349
|
"system": "Vector Only",
|
|
350
|
-
"durationMs":
|
|
350
|
+
"durationMs": 1,
|
|
351
351
|
"passed": false,
|
|
352
352
|
"score": 0.5,
|
|
353
353
|
"summary": "retrieved expected evidence, but conflicting evidence still appeared later",
|
|
@@ -407,7 +407,7 @@
|
|
|
407
407
|
},
|
|
408
408
|
{
|
|
409
409
|
"system": "Vector Only",
|
|
410
|
-
"durationMs":
|
|
410
|
+
"durationMs": 0,
|
|
411
411
|
"passed": true,
|
|
412
412
|
"score": 1,
|
|
413
413
|
"summary": "retrieved expected evidence",
|
|
@@ -518,7 +518,7 @@
|
|
|
518
518
|
"results": [
|
|
519
519
|
{
|
|
520
520
|
"system": "Audrey",
|
|
521
|
-
"durationMs":
|
|
521
|
+
"durationMs": 15,
|
|
522
522
|
"passed": true,
|
|
523
523
|
"score": 1,
|
|
524
524
|
"summary": "correct abstention",
|
|
@@ -643,7 +643,7 @@
|
|
|
643
643
|
},
|
|
644
644
|
{
|
|
645
645
|
"system": "Vector Only",
|
|
646
|
-
"durationMs":
|
|
646
|
+
"durationMs": 1,
|
|
647
647
|
"passed": false,
|
|
648
648
|
"score": 0,
|
|
649
649
|
"summary": "missed target evidence",
|
|
@@ -750,19 +750,19 @@
|
|
|
750
750
|
"system": "Audrey",
|
|
751
751
|
"scorePercent": 100,
|
|
752
752
|
"passRate": 100,
|
|
753
|
-
"avgDurationMs": 16
|
|
753
|
+
"avgDurationMs": 16.5
|
|
754
754
|
},
|
|
755
755
|
{
|
|
756
756
|
"system": "Keyword + Recency",
|
|
757
757
|
"scorePercent": 25,
|
|
758
758
|
"passRate": 0,
|
|
759
|
-
"avgDurationMs": 0
|
|
759
|
+
"avgDurationMs": 0.25
|
|
760
760
|
},
|
|
761
761
|
{
|
|
762
762
|
"system": "Vector Only",
|
|
763
763
|
"scorePercent": 12.5,
|
|
764
764
|
"passRate": 0,
|
|
765
|
-
"avgDurationMs": 0
|
|
765
|
+
"avgDurationMs": 0
|
|
766
766
|
},
|
|
767
767
|
{
|
|
768
768
|
"system": "Recent Window",
|
|
@@ -821,7 +821,7 @@
|
|
|
821
821
|
"results": [
|
|
822
822
|
{
|
|
823
823
|
"system": "Audrey",
|
|
824
|
-
"durationMs":
|
|
824
|
+
"durationMs": 15,
|
|
825
825
|
"passed": true,
|
|
826
826
|
"score": 1,
|
|
827
827
|
"summary": "retrieved expected evidence",
|
|
@@ -899,7 +899,7 @@
|
|
|
899
899
|
},
|
|
900
900
|
{
|
|
901
901
|
"system": "Keyword + Recency",
|
|
902
|
-
"durationMs":
|
|
902
|
+
"durationMs": 1,
|
|
903
903
|
"passed": false,
|
|
904
904
|
"score": 0.5,
|
|
905
905
|
"summary": "no leak, but retrieved tangential context",
|
|
@@ -932,7 +932,7 @@
|
|
|
932
932
|
"results": [
|
|
933
933
|
{
|
|
934
934
|
"system": "Audrey",
|
|
935
|
-
"durationMs":
|
|
935
|
+
"durationMs": 18,
|
|
936
936
|
"passed": true,
|
|
937
937
|
"score": 1,
|
|
938
938
|
"summary": "retrieved expected evidence",
|
|
@@ -943,7 +943,7 @@
|
|
|
943
943
|
},
|
|
944
944
|
{
|
|
945
945
|
"system": "Vector Only",
|
|
946
|
-
"durationMs":
|
|
946
|
+
"durationMs": 0,
|
|
947
947
|
"passed": false,
|
|
948
948
|
"score": 0,
|
|
949
949
|
"summary": "missed target evidence",
|
|
@@ -1055,7 +1055,7 @@
|
|
|
1055
1055
|
"system": "Audrey",
|
|
1056
1056
|
"scorePercent": 100,
|
|
1057
1057
|
"passRate": 100,
|
|
1058
|
-
"avgDurationMs":
|
|
1058
|
+
"avgDurationMs": 16
|
|
1059
1059
|
},
|
|
1060
1060
|
{
|
|
1061
1061
|
"system": "Vector Only",
|
|
@@ -1118,7 +1118,7 @@
|
|
|
1118
1118
|
"results": [
|
|
1119
1119
|
{
|
|
1120
1120
|
"system": "Audrey",
|
|
1121
|
-
"durationMs":
|
|
1121
|
+
"durationMs": 21,
|
|
1122
1122
|
"passed": true,
|
|
1123
1123
|
"score": 1,
|
|
1124
1124
|
"summary": "matched all required signals",
|
|
@@ -1176,7 +1176,7 @@
|
|
|
1176
1176
|
"results": [
|
|
1177
1177
|
{
|
|
1178
1178
|
"system": "Audrey",
|
|
1179
|
-
"durationMs":
|
|
1179
|
+
"durationMs": 15,
|
|
1180
1180
|
"passed": true,
|
|
1181
1181
|
"score": 1,
|
|
1182
1182
|
"summary": "matched all required signals",
|
|
@@ -1239,7 +1239,7 @@
|
|
|
1239
1239
|
"score": 1,
|
|
1240
1240
|
"summary": "matched all required signals",
|
|
1241
1241
|
"topResults": [
|
|
1242
|
-
"guard_hardened:replay_rejected error:guard receipt already has an outcome:
|
|
1242
|
+
"guard_hardened:replay_rejected error:guard receipt already has an outcome: 01KXDZ7CDYHYCCDW5B9KP7YG7W",
|
|
1243
1243
|
"decision:caution verdict:caution risk:0.55 Caution: 1 memory signal, 1 medium severity found before acting.",
|
|
1244
1244
|
"warning:recent_failure severity:medium npm test failed 1x recently: Vitest failed with spawn EPERM Before re-running npm test, check what changed since the last failure."
|
|
1245
1245
|
],
|
|
@@ -1292,12 +1292,12 @@
|
|
|
1292
1292
|
"results": [
|
|
1293
1293
|
{
|
|
1294
1294
|
"system": "Audrey",
|
|
1295
|
-
"durationMs":
|
|
1295
|
+
"durationMs": 13,
|
|
1296
1296
|
"passed": true,
|
|
1297
1297
|
"score": 1,
|
|
1298
1298
|
"summary": "matched all required signals",
|
|
1299
1299
|
"topResults": [
|
|
1300
|
-
"guard_hardened:non_guard_receipt_rejected error:not a guard receipt:
|
|
1300
|
+
"guard_hardened:non_guard_receipt_rejected error:not a guard receipt: 01KXDZ7CEBR5TGNA1TZ3XXB00T",
|
|
1301
1301
|
"decision:go verdict:clear risk:0 No relevant memory risks, prior failures, or must-follow procedures were found."
|
|
1302
1302
|
],
|
|
1303
1303
|
"retrievalSummary": "guard_hardened:non_guard_receipt_rejected error:not a guard receipt: 01K | decision:go verdict:clear risk:0 No relevant memory risks, prior failure"
|
|
@@ -1352,7 +1352,7 @@
|
|
|
1352
1352
|
"results": [
|
|
1353
1353
|
{
|
|
1354
1354
|
"system": "Audrey",
|
|
1355
|
-
"durationMs":
|
|
1355
|
+
"durationMs": 26,
|
|
1356
1356
|
"passed": true,
|
|
1357
1357
|
"score": 1,
|
|
1358
1358
|
"summary": "retrieved expected evidence",
|
|
@@ -1364,7 +1364,7 @@
|
|
|
1364
1364
|
},
|
|
1365
1365
|
{
|
|
1366
1366
|
"system": "Vector Only",
|
|
1367
|
-
"durationMs":
|
|
1367
|
+
"durationMs": 0,
|
|
1368
1368
|
"passed": true,
|
|
1369
1369
|
"score": 1,
|
|
1370
1370
|
"summary": "retrieved expected evidence",
|
|
@@ -1376,7 +1376,7 @@
|
|
|
1376
1376
|
},
|
|
1377
1377
|
{
|
|
1378
1378
|
"system": "Keyword + Recency",
|
|
1379
|
-
"durationMs":
|
|
1379
|
+
"durationMs": 8,
|
|
1380
1380
|
"passed": true,
|
|
1381
1381
|
"score": 1,
|
|
1382
1382
|
"summary": "retrieved expected evidence",
|
|
@@ -1422,7 +1422,7 @@
|
|
|
1422
1422
|
},
|
|
1423
1423
|
{
|
|
1424
1424
|
"system": "Vector Only",
|
|
1425
|
-
"durationMs":
|
|
1425
|
+
"durationMs": 1,
|
|
1426
1426
|
"passed": false,
|
|
1427
1427
|
"score": 0.5,
|
|
1428
1428
|
"summary": "retrieved expected evidence, but conflicting evidence still appeared later",
|
|
@@ -1482,7 +1482,7 @@
|
|
|
1482
1482
|
},
|
|
1483
1483
|
{
|
|
1484
1484
|
"system": "Vector Only",
|
|
1485
|
-
"durationMs":
|
|
1485
|
+
"durationMs": 0,
|
|
1486
1486
|
"passed": true,
|
|
1487
1487
|
"score": 1,
|
|
1488
1488
|
"summary": "retrieved expected evidence",
|
|
@@ -1593,7 +1593,7 @@
|
|
|
1593
1593
|
"results": [
|
|
1594
1594
|
{
|
|
1595
1595
|
"system": "Audrey",
|
|
1596
|
-
"durationMs":
|
|
1596
|
+
"durationMs": 15,
|
|
1597
1597
|
"passed": true,
|
|
1598
1598
|
"score": 1,
|
|
1599
1599
|
"summary": "correct abstention",
|
|
@@ -1718,7 +1718,7 @@
|
|
|
1718
1718
|
},
|
|
1719
1719
|
{
|
|
1720
1720
|
"system": "Vector Only",
|
|
1721
|
-
"durationMs":
|
|
1721
|
+
"durationMs": 1,
|
|
1722
1722
|
"passed": false,
|
|
1723
1723
|
"score": 0,
|
|
1724
1724
|
"summary": "missed target evidence",
|
|
@@ -1824,7 +1824,7 @@
|
|
|
1824
1824
|
"results": [
|
|
1825
1825
|
{
|
|
1826
1826
|
"system": "Audrey",
|
|
1827
|
-
"durationMs":
|
|
1827
|
+
"durationMs": 15,
|
|
1828
1828
|
"passed": true,
|
|
1829
1829
|
"score": 1,
|
|
1830
1830
|
"summary": "retrieved expected evidence",
|
|
@@ -1902,7 +1902,7 @@
|
|
|
1902
1902
|
},
|
|
1903
1903
|
{
|
|
1904
1904
|
"system": "Keyword + Recency",
|
|
1905
|
-
"durationMs":
|
|
1905
|
+
"durationMs": 1,
|
|
1906
1906
|
"passed": false,
|
|
1907
1907
|
"score": 0.5,
|
|
1908
1908
|
"summary": "no leak, but retrieved tangential context",
|
|
@@ -1935,7 +1935,7 @@
|
|
|
1935
1935
|
"results": [
|
|
1936
1936
|
{
|
|
1937
1937
|
"system": "Audrey",
|
|
1938
|
-
"durationMs":
|
|
1938
|
+
"durationMs": 18,
|
|
1939
1939
|
"passed": true,
|
|
1940
1940
|
"score": 1,
|
|
1941
1941
|
"summary": "retrieved expected evidence",
|
|
@@ -1946,7 +1946,7 @@
|
|
|
1946
1946
|
},
|
|
1947
1947
|
{
|
|
1948
1948
|
"system": "Vector Only",
|
|
1949
|
-
"durationMs":
|
|
1949
|
+
"durationMs": 0,
|
|
1950
1950
|
"passed": false,
|
|
1951
1951
|
"score": 0,
|
|
1952
1952
|
"summary": "missed target evidence",
|
|
@@ -2058,7 +2058,7 @@
|
|
|
2058
2058
|
"results": [
|
|
2059
2059
|
{
|
|
2060
2060
|
"system": "Audrey",
|
|
2061
|
-
"durationMs":
|
|
2061
|
+
"durationMs": 21,
|
|
2062
2062
|
"passed": true,
|
|
2063
2063
|
"score": 1,
|
|
2064
2064
|
"summary": "matched all required signals",
|
|
@@ -2116,7 +2116,7 @@
|
|
|
2116
2116
|
"results": [
|
|
2117
2117
|
{
|
|
2118
2118
|
"system": "Audrey",
|
|
2119
|
-
"durationMs":
|
|
2119
|
+
"durationMs": 15,
|
|
2120
2120
|
"passed": true,
|
|
2121
2121
|
"score": 1,
|
|
2122
2122
|
"summary": "matched all required signals",
|
|
@@ -2179,7 +2179,7 @@
|
|
|
2179
2179
|
"score": 1,
|
|
2180
2180
|
"summary": "matched all required signals",
|
|
2181
2181
|
"topResults": [
|
|
2182
|
-
"guard_hardened:replay_rejected error:guard receipt already has an outcome:
|
|
2182
|
+
"guard_hardened:replay_rejected error:guard receipt already has an outcome: 01KXDZ7CDYHYCCDW5B9KP7YG7W",
|
|
2183
2183
|
"decision:caution verdict:caution risk:0.55 Caution: 1 memory signal, 1 medium severity found before acting.",
|
|
2184
2184
|
"warning:recent_failure severity:medium npm test failed 1x recently: Vitest failed with spawn EPERM Before re-running npm test, check what changed since the last failure."
|
|
2185
2185
|
],
|
|
@@ -2232,12 +2232,12 @@
|
|
|
2232
2232
|
"results": [
|
|
2233
2233
|
{
|
|
2234
2234
|
"system": "Audrey",
|
|
2235
|
-
"durationMs":
|
|
2235
|
+
"durationMs": 13,
|
|
2236
2236
|
"passed": true,
|
|
2237
2237
|
"score": 1,
|
|
2238
2238
|
"summary": "matched all required signals",
|
|
2239
2239
|
"topResults": [
|
|
2240
|
-
"guard_hardened:non_guard_receipt_rejected error:not a guard receipt:
|
|
2240
|
+
"guard_hardened:non_guard_receipt_rejected error:not a guard receipt: 01KXDZ7CEBR5TGNA1TZ3XXB00T",
|
|
2241
2241
|
"decision:go verdict:clear risk:0 No relevant memory risks, prior failures, or must-follow procedures were found."
|
|
2242
2242
|
],
|
|
2243
2243
|
"retrievalSummary": "guard_hardened:non_guard_receipt_rejected error:not a guard receipt: 01K | decision:go verdict:clear risk:0 No relevant memory risks, prior failure"
|
|
@@ -24,13 +24,13 @@ These numbers measure Audrey's local call path under an in-process mock embeddin
|
|
|
24
24
|
|
|
25
25
|
## Behavioral Regression Result
|
|
26
26
|
|
|
27
|
-
The current `benchmarks/output/summary.json` was generated on 2026-07-
|
|
27
|
+
The current `benchmarks/output/summary.json` was generated on 2026-07-13T14:48:45.265Z with command `node benchmarks/run.js --provider mock --dimensions 64` (Ledger: E24). It reports:
|
|
28
28
|
|
|
29
29
|
| System | Score Percent | Pass Rate | Average Duration Ms |
|
|
30
30
|
|---|---:|---:|---:|
|
|
31
|
-
| Audrey | 100 | 100 | 16.
|
|
32
|
-
| Vector Only | 41.66666666666667 | 25 | 0.
|
|
33
|
-
| Keyword + Recency | 41.66666666666667 | 25 | 0.
|
|
31
|
+
| Audrey | 100 | 100 | 16.583333333333332 |
|
|
32
|
+
| Vector Only | 41.66666666666667 | 25 | 0.16666666666666666 |
|
|
33
|
+
| Keyword + Recency | 41.66666666666667 | 25 | 0.75 |
|
|
34
34
|
| Recent Window | 37.5 | 25 | 0 |
|
|
35
35
|
|
|
36
36
|
This output is a regression-gate result. The baselines are toy local baselines used to catch retrieval and lifecycle regressions in the Audrey codebase. They are not external systems, not tuned competitor implementations, and not GuardBench baselines (Ledger: E23-E24). The current suite covers retrieval and operation families such as information extraction, knowledge updates, multi-session reasoning, conflict resolution, procedural learning, privacy boundary, overwrite, delete-and-abstain, semantic merge, and procedural merge (Ledger: E23-E24).
|
|
@@ -55,7 +55,7 @@ It reports local adapters only, not external-system comparisons (Ledger: E46):
|
|
|
55
55
|
| Evidence recall | 100% |
|
|
56
56
|
| Redaction leaks | 0 |
|
|
57
57
|
| Recall-degradation detection | 100% |
|
|
58
|
-
| Guard latency p50 / p95 |
|
|
58
|
+
| Guard latency p50 / p95 | 4.253 ms / 21.196 ms |
|
|
59
59
|
| Published artifact raw-secret leaks | 0 |
|
|
60
60
|
| Audrey Guard decision accuracy | 100% |
|
|
61
61
|
| No-memory decision accuracy | 10% |
|
|
@@ -897,13 +897,13 @@ These numbers measure Audrey's local call path under an in-process mock embeddin
|
|
|
897
897
|
|
|
898
898
|
### Behavioral Regression Result
|
|
899
899
|
|
|
900
|
-
The current `benchmarks/output/summary.json` was generated on 2026-07-
|
|
900
|
+
The current `benchmarks/output/summary.json` was generated on 2026-07-13T14:48:45.265Z with command `node benchmarks/run.js --provider mock --dimensions 64` (Ledger: E24). It reports:
|
|
901
901
|
|
|
902
902
|
| System | Score Percent | Pass Rate | Average Duration Ms |
|
|
903
903
|
|---|---:|---:|---:|
|
|
904
|
-
| Audrey | 100 | 100 | 16.
|
|
905
|
-
| Vector Only | 41.66666666666667 | 25 | 0.
|
|
906
|
-
| Keyword + Recency | 41.66666666666667 | 25 | 0.
|
|
904
|
+
| Audrey | 100 | 100 | 16.583333333333332 |
|
|
905
|
+
| Vector Only | 41.66666666666667 | 25 | 0.16666666666666666 |
|
|
906
|
+
| Keyword + Recency | 41.66666666666667 | 25 | 0.75 |
|
|
907
907
|
| Recent Window | 37.5 | 25 | 0 |
|
|
908
908
|
|
|
909
909
|
This output is a regression-gate result. The baselines are toy local baselines used to catch retrieval and lifecycle regressions in the Audrey codebase. They are not external systems, not tuned competitor implementations, and not GuardBench baselines (Ledger: E23-E24). The current suite covers retrieval and operation families such as information extraction, knowledge updates, multi-session reasoning, conflict resolution, procedural learning, privacy boundary, overwrite, delete-and-abstain, semantic merge, and procedural merge (Ledger: E23-E24).
|
|
@@ -926,7 +926,7 @@ It reports local adapters only, not external-system comparisons (Ledger: E46):
|
|
|
926
926
|
| Evidence recall | 100% |
|
|
927
927
|
| Redaction leaks | 0 |
|
|
928
928
|
| Recall-degradation detection | 100% |
|
|
929
|
-
| Guard latency p50 / p95 |
|
|
929
|
+
| Guard latency p50 / p95 | 4.253 ms / 21.196 ms |
|
|
930
930
|
| Published artifact raw-secret leaks | 0 |
|
|
931
931
|
| Audrey Guard decision accuracy | 100% |
|
|
932
932
|
| No-memory decision accuracy | 10% |
|
|
@@ -49,7 +49,7 @@ Every implementation claim in the paper should point to one or more ledger IDs i
|
|
|
49
49
|
| E43 - Audrey 1.1 ships one normalized lifecycle adapter and host-specific configuration for current Claude Code and Codex CLIs. `audrey install --host auto` registers MCP and applies hooks for detected hosts from stable executable paths. `hook-config` supports preview, scoped apply, private backups, and idempotent Audrey-handler replacement; `uninstall` supports non-mutating preview and removes Audrey-owned registrations and handlers in the selected scope. Session and prompt hooks inject bounded, agent-scoped context; PreToolUse creates Guard receipts; post-tool events correlate outcomes by `session_id + tool_use_id`; sanitized failures may form durable memories; Stop and PostCompact run due-only maintenance. Codex requires explicit non-managed hook trust through `/hooks`. Hook errors fail open unless `AUDREY_HOOK_FAIL_CLOSED=1`. Coverage remains limited to events and tool paths emitted by the host, and each hook starts a short-lived Audrey process rather than using a persistent daemon. | Automatic lifecycle integration and boundary | README.md; CHANGELOG.md; mcp-server/hooks.ts; mcp-server/index.ts; src/autopilot.ts; tests/autopilot.test.js; tests/hooks-config.test.js; tests/mcp-server.test.js | Yes, build plus 191 focused install, hook, Autopilot, preflight, controller, and HTTP tests passed on 2026-07-10 |
|
|
50
50
|
| E44 - Audrey preflight events now persist `preflight_evidence_ids` and `audrey_guard_action_key`; `memory_validate` accepts optional `preflight_event_id`, action key, and evidence ids, persists them on the validation audit event, and rejects validation lineage when the memory id was not evidence for that preflight. Preflight embedding queries are bounded, while exact action identity hashes the full normalized, redacted action so large inputs with a shared prefix do not collide. | Validation lineage implementation | src/action-key.ts; src/controller.ts; src/preflight.ts; src/audrey.ts; mcp-server/index.ts; tests/controller.test.js | Yes, focused Vitest passed on 2026-07-10, including a large-action prefix-collision regression |
|
|
51
51
|
| E45 - Preflight risk scoring uses a fixed severity map (`info=0.1`, `low=0.25`, `medium=0.55`, `high=0.85`), sorts warnings by severity, and strict mode blocks on high-severity warnings; the scoring path does not consume validation feedback. | Fixed risk scoring boundary | src/preflight.ts:6-60,291-299,332-338; src/feedback.ts:3-18,70-163 | Yes, 2026-05-08 |
|
|
52
|
-
| E46 - `benchmarks/guardbench.js` runs ten local comparative GuardBench scenarios across Audrey Guard, no-memory, recent-window, vector-only, and FTS-only adapters and writes `benchmarks/output/guardbench-summary.json`, `benchmarks/output/guardbench-manifest.json`, and `benchmarks/output/guardbench-raw.json`; the latest local run has Audrey Guard passing 10/10 scenarios with 100% prevention rate, 0% false-block rate, 100% evidence recall, zero decision-output redaction leaks, zero published artifact raw-secret leaks, 100% recall-degradation detection, 100% decision accuracy, and
|
|
52
|
+
| E46 - `benchmarks/guardbench.js` runs ten local comparative GuardBench scenarios across Audrey Guard, no-memory, recent-window, vector-only, and FTS-only adapters and writes `benchmarks/output/guardbench-summary.json`, `benchmarks/output/guardbench-manifest.json`, and `benchmarks/output/guardbench-raw.json`; the latest local run has Audrey Guard passing 10/10 scenarios with 100% prevention rate, 0% false-block rate, 100% evidence recall, zero decision-output redaction leaks, zero published artifact raw-secret leaks, 100% recall-degradation detection, 100% decision accuracy, and 4.253ms/21.196ms p50/p95 guard latency under the mock-provider methodology. Baseline decision accuracy was no-memory 10%, recent-window 60%, vector-only 40%, and FTS-only 10%, with 0% full-contract pass rate for each baseline. | GuardBench local comparative results | benchmarks/guardbench.js; benchmarks/output/guardbench-summary.json; benchmarks/output/guardbench-manifest.json; benchmarks/output/guardbench-raw.json; package.json | Yes, `npm run bench:guard:check` passed on 2026-05-13 |
|
|
53
53
|
| E47 - GuardBench accepts external ESM adapters through `--adapter`, supports `default`, `adapter`, or `createGuardBenchAdapter()` exports, withholds `expectedDecision` and `requiredEvidence` during adapter execution, then scores adapter output against the same full-contract decision/evidence/redaction checks. | GuardBench external adapter contract | benchmarks/guardbench.js; tests/guardbench.test.js; package.json | Yes, `node scripts/run-vitest.mjs run tests/guardbench.test.js` passed on 2026-05-12 |
|
|
54
54
|
| E48 - Audrey ships a Mem0 Platform GuardBench adapter that uses the current Mem0 REST shape: V3 async memory add with event polling, V2 filtered memory search, and user-entity cleanup. It requires runtime `MEM0_API_KEY` and is not run by default. | First external-system GuardBench adapter | benchmarks/adapters/mem0-platform.mjs; tests/guardbench.test.js; README.md | Import/contract and mocked REST-flow tests passed on 2026-05-12; live Mem0 run not yet executed |
|
|
55
55
|
| E49 - GuardBench ships a credential-free example external adapter and a `bench:guard:adapter-smoke` script so the adapter loader can be exercised through the real CLI path without external credentials. | External adapter smoke path | benchmarks/adapters/example-allow.mjs; package.json; README.md; tests/guardbench.test.js | Yes, `npm run bench:guard:adapter-smoke` passed on 2026-05-12 |
|
|
@@ -1,11 +1,11 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "Audrey arXiv source package",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T14:48:55.293Z",
|
|
5
5
|
"sourceMarkdown": "docs/paper/audrey-paper-v1.md",
|
|
6
6
|
"publicationPack": "docs/paper/publication-pack.json",
|
|
7
7
|
"sourceHashes": {
|
|
8
|
-
"sourceMarkdown": "
|
|
8
|
+
"sourceMarkdown": "7db503c10b25314daa46e31fc65c98d18cd86895fe7bf44501e108626c4a3cac",
|
|
9
9
|
"publicationPack": "a1a523d5938faea72be568b843ac3890e61cea6070b0cfa46acf22ad3d2fb974",
|
|
10
10
|
"referencesBib": "c0bfcaf7bfe37d6933c812e46352be8a95397eaa430a0f5bc94037600a53f654"
|
|
11
11
|
},
|
|
@@ -13,8 +13,8 @@
|
|
|
13
13
|
{
|
|
14
14
|
"path": "main.tex",
|
|
15
15
|
"source": "docs/paper/audrey-paper-v1.md",
|
|
16
|
-
"bytes":
|
|
17
|
-
"sha256": "
|
|
16
|
+
"bytes": 124825,
|
|
17
|
+
"sha256": "d9d0cf25cad4e59d6234643eb875bd5d74b4d45acd106e9bc01c7454e7240622"
|
|
18
18
|
},
|
|
19
19
|
{
|
|
20
20
|
"path": "references.bib",
|
|
@@ -731,13 +731,13 @@ These numbers measure Audrey's local call path under an in-process mock embeddin
|
|
|
731
731
|
|
|
732
732
|
\subsection{Behavioral Regression Result}
|
|
733
733
|
|
|
734
|
-
The current \texttt{benchmarks/output/summary.json} was generated on 2026-07-
|
|
734
|
+
The current \texttt{benchmarks/output/summary.json} was generated on 2026-07-13T14:48:45.265Z with command \texttt{node benchmarks/run.js --provider mock --dimensions 64} (Ledger: E24). It reports:
|
|
735
735
|
|
|
736
736
|
\begin{verbatim}
|
|
737
737
|
| System | Score Percent | Pass Rate | Average Duration Ms |
|
|
738
|
-
| Audrey | 100 | 100 | 16.
|
|
739
|
-
| Vector Only | 41.66666666666667 | 25 | 0.
|
|
740
|
-
| Keyword + Recency | 41.66666666666667 | 25 | 0.
|
|
738
|
+
| Audrey | 100 | 100 | 16.583333333333332 |
|
|
739
|
+
| Vector Only | 41.66666666666667 | 25 | 0.16666666666666666 |
|
|
740
|
+
| Keyword + Recency | 41.66666666666667 | 25 | 0.75 |
|
|
741
741
|
| Recent Window | 37.5 | 25 | 0 |
|
|
742
742
|
\end{verbatim}
|
|
743
743
|
|
|
@@ -761,7 +761,7 @@ It reports local adapters only, not external-system comparisons (Ledger: E46):
|
|
|
761
761
|
| Evidence recall | 100% |
|
|
762
762
|
| Redaction leaks | 0 |
|
|
763
763
|
| Recall-degradation detection | 100% |
|
|
764
|
-
| Guard latency p50 / p95 |
|
|
764
|
+
| Guard latency p50 / p95 | 4.253 ms / 21.196 ms |
|
|
765
765
|
| Published artifact raw-secret leaks | 0 |
|
|
766
766
|
| Audrey Guard decision accuracy | 100% |
|
|
767
767
|
| No-memory decision accuracy | 10% |
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "Audrey arXiv compile check",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T14:48:57.090Z",
|
|
5
5
|
"source": {
|
|
6
6
|
"sourceDir": "docs/paper/output/arxiv",
|
|
7
7
|
"manifest": "docs/paper/output/arxiv/arxiv-manifest.json",
|
|
8
|
-
"manifestSha256": "
|
|
8
|
+
"manifestSha256": "de555f986f650735f8b4184eb93504b20d94dbd1e8510d839311fbf7e7d69429",
|
|
9
9
|
"mainTex": "docs/paper/output/arxiv/main.tex",
|
|
10
|
-
"mainTexSha256": "
|
|
10
|
+
"mainTexSha256": "d9d0cf25cad4e59d6234643eb875bd5d74b4d45acd106e9bc01c7454e7240622",
|
|
11
11
|
"referencesBib": "docs/paper/output/arxiv/references.bib",
|
|
12
12
|
"referencesBibSha256": "c0bfcaf7bfe37d6933c812e46352be8a95397eaa430a0f5bc94037600a53f654"
|
|
13
13
|
},
|
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "audrey",
|
|
3
|
-
"version": "1.1.
|
|
3
|
+
"version": "1.1.2",
|
|
4
|
+
"mcpName": "io.github.evilander/audrey",
|
|
4
5
|
"description": "Automatic, evidence-backed memory and pre-action Guard for Codex, Claude Code, and custom AI agents",
|
|
5
6
|
"type": "module",
|
|
6
7
|
"main": "dist/src/index.js",
|
|
@@ -221,6 +222,12 @@
|
|
|
221
222
|
"overrides": {
|
|
222
223
|
"qs": "^6.15.2"
|
|
223
224
|
},
|
|
225
|
+
"allowScripts": {
|
|
226
|
+
"better-sqlite3@12.9.0": true,
|
|
227
|
+
"onnxruntime-node@1.21.0": true,
|
|
228
|
+
"protobufjs@7.6.5": true,
|
|
229
|
+
"sharp@0.34.5": true
|
|
230
|
+
},
|
|
224
231
|
"directories": {
|
|
225
232
|
"example": "examples",
|
|
226
233
|
"test": "tests"
|