audrey 1.1.2 → 1.1.3
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +8 -0
- package/README.md +1 -1
- package/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +4 -4
- package/benchmarks/output/external/guardbench-external-dry-run.json +1 -1
- package/benchmarks/output/external/guardbench-external-evidence.json +1 -1
- package/benchmarks/output/guardbench-conformance-card.json +9 -9
- package/benchmarks/output/guardbench-raw.json +108 -108
- package/benchmarks/output/guardbench-summary.json +174 -174
- package/benchmarks/output/leaderboard/guardbench-leaderboard.json +5 -5
- package/benchmarks/output/leaderboard/guardbench-leaderboard.md +2 -2
- package/benchmarks/output/submission-bundle/guardbench-conformance-card.json +9 -9
- package/benchmarks/output/submission-bundle/guardbench-raw.json +108 -108
- package/benchmarks/output/submission-bundle/guardbench-summary.json +174 -174
- package/benchmarks/output/submission-bundle/submission-manifest.json +10 -10
- package/benchmarks/output/submission-bundle/validation-report.json +1 -1
- package/benchmarks/output/summary.json +54 -54
- package/dist/mcp-server/config.d.ts +1 -1
- package/dist/mcp-server/config.js +1 -1
- package/docs/paper/07-evaluation.md +6 -6
- package/docs/paper/audrey-paper-v1.md +6 -6
- package/docs/paper/evidence-ledger.md +1 -1
- package/docs/paper/output/arxiv/arxiv-manifest.json +4 -4
- package/docs/paper/output/arxiv/main.tex +6 -6
- package/docs/paper/output/arxiv-compile-report.json +3 -3
- package/docs/paper/output/submission-bundle/README.md +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +4 -4
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-dry-run.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-evidence.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-conformance-card.json +9 -9
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-raw.json +108 -108
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-summary.json +174 -174
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.json +5 -5
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.md +2 -2
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/submission-manifest.json +10 -10
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/validation-report.json +1 -1
- package/docs/paper/output/submission-bundle/benchmarks/output/summary.json +60 -60
- package/docs/paper/output/submission-bundle/docs/paper/07-evaluation.md +6 -6
- package/docs/paper/output/submission-bundle/docs/paper/audrey-paper-v1.md +6 -6
- package/docs/paper/output/submission-bundle/docs/paper/evidence-ledger.md +1 -1
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/arxiv-manifest.json +4 -4
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/main.tex +6 -6
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv-compile-report.json +3 -3
- package/docs/paper/output/submission-bundle/package.json +2 -2
- package/docs/paper/output/submission-bundle/paper-submission-manifest.json +28 -28
- package/package.json +2 -2
|
@@ -731,14 +731,14 @@ These numbers measure Audrey's local call path under an in-process mock embeddin
|
|
|
731
731
|
|
|
732
732
|
\subsection{Behavioral Regression Result}
|
|
733
733
|
|
|
734
|
-
The current \texttt{benchmarks/output/summary.json} was generated on 2026-07-
|
|
734
|
+
The current \texttt{benchmarks/output/summary.json} was generated on 2026-07-13T15:07:26.566Z with command \texttt{node benchmarks/run.js --provider mock --dimensions 64} (Ledger: E24). It reports:
|
|
735
735
|
|
|
736
736
|
\begin{verbatim}
|
|
737
737
|
| System | Score Percent | Pass Rate | Average Duration Ms |
|
|
738
|
-
| Audrey | 100 | 100 |
|
|
739
|
-
| Vector Only | 41.66666666666667 | 25 | 0.
|
|
740
|
-
| Keyword + Recency | 41.66666666666667 | 25 | 0.
|
|
741
|
-
| Recent Window | 37.5 | 25 | 0 |
|
|
738
|
+
| Audrey | 100 | 100 | 19.333333333333332 |
|
|
739
|
+
| Vector Only | 41.66666666666667 | 25 | 0.25 |
|
|
740
|
+
| Keyword + Recency | 41.66666666666667 | 25 | 0.5833333333333334 |
|
|
741
|
+
| Recent Window | 37.5 | 25 | 0.08333333333333333 |
|
|
742
742
|
\end{verbatim}
|
|
743
743
|
|
|
744
744
|
This output is a regression-gate result. The baselines are toy local baselines used to catch retrieval and lifecycle regressions in the Audrey codebase. They are not external systems, not tuned competitor implementations, and not GuardBench baselines (Ledger: E23-E24). The current suite covers retrieval and operation families such as information extraction, knowledge updates, multi-session reasoning, conflict resolution, procedural learning, privacy boundary, overwrite, delete-and-abstain, semantic merge, and procedural merge (Ledger: E23-E24).
|
|
@@ -761,7 +761,7 @@ It reports local adapters only, not external-system comparisons (Ledger: E46):
|
|
|
761
761
|
| Evidence recall | 100% |
|
|
762
762
|
| Redaction leaks | 0 |
|
|
763
763
|
| Recall-degradation detection | 100% |
|
|
764
|
-
| Guard latency p50 / p95 | 4.
|
|
764
|
+
| Guard latency p50 / p95 | 4.607 ms / 21.585 ms |
|
|
765
765
|
| Published artifact raw-secret leaks | 0 |
|
|
766
766
|
| Audrey Guard decision accuracy | 100% |
|
|
767
767
|
| No-memory decision accuracy | 10% |
|
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "Audrey arXiv compile check",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T15:07:39.643Z",
|
|
5
5
|
"source": {
|
|
6
6
|
"sourceDir": "docs/paper/output/arxiv",
|
|
7
7
|
"manifest": "docs/paper/output/arxiv/arxiv-manifest.json",
|
|
8
|
-
"manifestSha256": "
|
|
8
|
+
"manifestSha256": "d91eaefbca39c49e79990ef58314bbaa502184dae14596a498c2b6d8ba2f7df2",
|
|
9
9
|
"mainTex": "docs/paper/output/arxiv/main.tex",
|
|
10
|
-
"mainTexSha256": "
|
|
10
|
+
"mainTexSha256": "0e122e6bd050e3fbf8e62ab3b363b12ebb00c7c8a4618b9fa4971d19e798bcc7",
|
|
11
11
|
"referencesBib": "docs/paper/output/arxiv/references.bib",
|
|
12
12
|
"referencesBibSha256": "c0bfcaf7bfe37d6933c812e46352be8a95397eaa430a0f5bc94037600a53f654"
|
|
13
13
|
},
|
|
@@ -390,7 +390,7 @@ npm run pack:check
|
|
|
390
390
|
GuardBench currently contains ten local, deterministic pre-action scenarios covering repeated failures, procedures, scope changes, recovery, redaction, conflicting instructions, and noisy stores. The checked-in v1 methodology uses a mock 64-dimensional embedding provider and exists to catch regressions. A perfect local pass is not a claim about real-provider latency or production false-positive rates.
|
|
391
391
|
|
|
392
392
|
<!-- guardbench-summary:start -->
|
|
393
|
-
Latest local result in this checkout: 10/10 scenarios passed, 100% prevention rate, 0% false-block rate, 0 raw secret leaks, 0 published artifact leaks, and 4.
|
|
393
|
+
Latest local result in this checkout: 10/10 scenarios passed, 100% prevention rate, 0% false-block rate, 0 raw secret leaks, 0 published artifact leaks, and 4.607ms / 21.585ms p50/p95 Guard latency under the mock-provider methodology.
|
|
394
394
|
<!-- guardbench-summary:end -->
|
|
395
395
|
|
|
396
396
|
`benchmarks/perf-snapshot.js` measures encode and hybrid-recall p50/p95/p99 at configurable corpus sizes with machine and provider provenance. Run it on the hardware and embedding provider you plan to operate; hosted-provider latency is dominated by its network round trip.
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "GuardBench adapter self-test",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T15:07:36.556Z",
|
|
5
5
|
"ok": true,
|
|
6
6
|
"adapter": {
|
|
7
7
|
"name": "Example Allow Adapter",
|
|
@@ -27,9 +27,9 @@
|
|
|
27
27
|
"evidenceRecall": 0.1,
|
|
28
28
|
"redactionLeaks": 0,
|
|
29
29
|
"latency": {
|
|
30
|
-
"p50Ms": 0.
|
|
31
|
-
"p95Ms": 0.
|
|
32
|
-
"maxMs": 0.
|
|
30
|
+
"p50Ms": 0.014,
|
|
31
|
+
"p95Ms": 0.045,
|
|
32
|
+
"maxMs": 0.045
|
|
33
33
|
}
|
|
34
34
|
},
|
|
35
35
|
"contract": {
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "GuardBench external adapter dry-run matrix",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T15:07:37.142Z",
|
|
5
5
|
"ok": true,
|
|
6
6
|
"registry": "benchmarks/adapters/registry.json",
|
|
7
7
|
"outRoot": "benchmarks/output/external",
|
package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-conformance-card.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": "1.0.0",
|
|
3
3
|
"suite": "GuardBench conformance card",
|
|
4
|
-
"generatedAt": "2026-07-
|
|
4
|
+
"generatedAt": "2026-07-13T15:07:31.567Z",
|
|
5
5
|
"sourceDir": "benchmarks/output",
|
|
6
6
|
"manifestVersion": "0.2.0",
|
|
7
7
|
"suiteId": "guardbench-local-comparative",
|
|
@@ -25,9 +25,9 @@
|
|
|
25
25
|
"evidenceRecall": 1,
|
|
26
26
|
"redactionLeaks": 0,
|
|
27
27
|
"latency": {
|
|
28
|
-
"p50Ms": 4.
|
|
29
|
-
"p95Ms": 21.
|
|
30
|
-
"maxMs": 21.
|
|
28
|
+
"p50Ms": 4.607,
|
|
29
|
+
"p95Ms": 21.585,
|
|
30
|
+
"maxMs": 21.585
|
|
31
31
|
}
|
|
32
32
|
},
|
|
33
33
|
"conformance": {
|
|
@@ -39,21 +39,21 @@
|
|
|
39
39
|
"integrity": {
|
|
40
40
|
"artifactHashes": {
|
|
41
41
|
"guardbench-manifest.json": "57636ce19fdaa6e50fc3fc961d9e499a9f43632f588c713a9fefe8e8a6fa724c",
|
|
42
|
-
"guardbench-summary.json": "
|
|
43
|
-
"guardbench-raw.json": "
|
|
42
|
+
"guardbench-summary.json": "53b22e040ace0b91784e99684ffd740c95740171c00bf65388590aeb493c7c7d",
|
|
43
|
+
"guardbench-raw.json": "20a91ed7ca54a3b5b47b3a3f9ecde4ae8d53cc2b9bdbb5cb1ce6347e2224497b"
|
|
44
44
|
},
|
|
45
45
|
"externalRunMetadataHash": null
|
|
46
46
|
},
|
|
47
47
|
"provenance": {
|
|
48
|
-
"generatedAt": "2026-07-
|
|
49
|
-
"gitSha": "
|
|
48
|
+
"generatedAt": "2026-07-13T15:07:31.179Z",
|
|
49
|
+
"gitSha": "36842cfb0cad7ca5da864231a7e265229ba56fbd",
|
|
50
50
|
"gitDirty": false,
|
|
51
51
|
"node": "v24.18.0",
|
|
52
52
|
"v8": "13.6.233.17-node.50",
|
|
53
53
|
"platform": "linux",
|
|
54
54
|
"arch": "x64",
|
|
55
55
|
"osRelease": "6.17.0-1018-azure",
|
|
56
|
-
"cpuModel": "AMD EPYC
|
|
56
|
+
"cpuModel": "AMD EPYC 7763 64-Core Processor",
|
|
57
57
|
"cpuCount": 4,
|
|
58
58
|
"totalMemoryGb": 15.61,
|
|
59
59
|
"embeddingProvider": "mock",
|