audrey 1.1.2 → 1.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +101 -0
- package/README.md +39 -12
- package/dist/mcp-server/codex-hooks-probe.d.ts +22 -0
- package/dist/mcp-server/codex-hooks-probe.d.ts.map +1 -0
- package/dist/mcp-server/codex-hooks-probe.js +160 -0
- package/dist/mcp-server/codex-hooks-probe.js.map +1 -0
- package/dist/mcp-server/config.d.ts +26 -2
- package/dist/mcp-server/config.d.ts.map +1 -1
- package/dist/mcp-server/config.js +52 -20
- package/dist/mcp-server/config.js.map +1 -1
- package/dist/mcp-server/hooks.d.ts +28 -0
- package/dist/mcp-server/hooks.d.ts.map +1 -1
- package/dist/mcp-server/hooks.js +91 -2
- package/dist/mcp-server/hooks.js.map +1 -1
- package/dist/mcp-server/index.d.ts +242 -2
- package/dist/mcp-server/index.d.ts.map +1 -1
- package/dist/mcp-server/index.js +806 -240
- package/dist/mcp-server/index.js.map +1 -1
- package/dist/mcp-server/tool-schemas.d.ts +32 -2
- package/dist/mcp-server/tool-schemas.d.ts.map +1 -1
- package/dist/mcp-server/tool-schemas.js +55 -0
- package/dist/mcp-server/tool-schemas.js.map +1 -1
- package/dist/src/audrey.d.ts +14 -2
- package/dist/src/audrey.d.ts.map +1 -1
- package/dist/src/audrey.js +152 -28
- package/dist/src/audrey.js.map +1 -1
- package/dist/src/autopilot.d.ts +22 -0
- package/dist/src/autopilot.d.ts.map +1 -1
- package/dist/src/autopilot.js +271 -50
- package/dist/src/autopilot.js.map +1 -1
- package/dist/src/capsule.d.ts +23 -0
- package/dist/src/capsule.d.ts.map +1 -1
- package/dist/src/capsule.js +56 -11
- package/dist/src/capsule.js.map +1 -1
- package/dist/src/confidence.d.ts +9 -1
- package/dist/src/confidence.d.ts.map +1 -1
- package/dist/src/confidence.js +16 -2
- package/dist/src/confidence.js.map +1 -1
- package/dist/src/consolidate.d.ts +4 -0
- package/dist/src/consolidate.d.ts.map +1 -1
- package/dist/src/consolidate.js +206 -34
- package/dist/src/consolidate.js.map +1 -1
- package/dist/src/controller.d.ts +17 -1
- package/dist/src/controller.d.ts.map +1 -1
- package/dist/src/controller.js +244 -128
- package/dist/src/controller.js.map +1 -1
- package/dist/src/db.d.ts.map +1 -1
- package/dist/src/db.js +233 -56
- package/dist/src/db.js.map +1 -1
- package/dist/src/encode.d.ts +2 -1
- package/dist/src/encode.d.ts.map +1 -1
- package/dist/src/encode.js +41 -4
- package/dist/src/encode.js.map +1 -1
- package/dist/src/events.d.ts +48 -1
- package/dist/src/events.d.ts.map +1 -1
- package/dist/src/events.js +160 -62
- package/dist/src/events.js.map +1 -1
- package/dist/src/export.d.ts.map +1 -1
- package/dist/src/export.js +4 -4
- package/dist/src/export.js.map +1 -1
- package/dist/src/grounding.d.ts +111 -0
- package/dist/src/grounding.d.ts.map +1 -0
- package/dist/src/grounding.js +259 -0
- package/dist/src/grounding.js.map +1 -0
- package/dist/src/hybrid-recall.d.ts +16 -7
- package/dist/src/hybrid-recall.d.ts.map +1 -1
- package/dist/src/hybrid-recall.js +22 -11
- package/dist/src/hybrid-recall.js.map +1 -1
- package/dist/src/import.d.ts +10 -0
- package/dist/src/import.d.ts.map +1 -1
- package/dist/src/import.js +122 -22
- package/dist/src/import.js.map +1 -1
- package/dist/src/llm.d.ts.map +1 -1
- package/dist/src/llm.js +16 -0
- package/dist/src/llm.js.map +1 -1
- package/dist/src/preflight.d.ts.map +1 -1
- package/dist/src/preflight.js +89 -47
- package/dist/src/preflight.js.map +1 -1
- package/dist/src/project.d.ts +4 -1
- package/dist/src/project.d.ts.map +1 -1
- package/dist/src/project.js +65 -5
- package/dist/src/project.js.map +1 -1
- package/dist/src/recall.d.ts.map +1 -1
- package/dist/src/recall.js +138 -13
- package/dist/src/recall.js.map +1 -1
- package/dist/src/redact.d.ts +1 -1
- package/dist/src/redact.d.ts.map +1 -1
- package/dist/src/redact.js +185 -5
- package/dist/src/redact.js.map +1 -1
- package/dist/src/routes.d.ts.map +1 -1
- package/dist/src/routes.js +91 -18
- package/dist/src/routes.js.map +1 -1
- package/dist/src/trust.d.ts +35 -0
- package/dist/src/trust.d.ts.map +1 -0
- package/dist/src/trust.js +77 -0
- package/dist/src/trust.js.map +1 -0
- package/dist/src/types.d.ts +24 -0
- package/dist/src/types.d.ts.map +1 -1
- package/package.json +4 -38
- package/benchmarks/adapter-kit.mjs +0 -20
- package/benchmarks/adapter-self-test.mjs +0 -170
- package/benchmarks/adapters/example-allow.mjs +0 -31
- package/benchmarks/adapters/mem0-platform.mjs +0 -274
- package/benchmarks/adapters/registry.json +0 -51
- package/benchmarks/adapters/zep-cloud.mjs +0 -304
- package/benchmarks/baselines.js +0 -174
- package/benchmarks/build-leaderboard.mjs +0 -183
- package/benchmarks/cases.js +0 -549
- package/benchmarks/create-conformance-card.mjs +0 -148
- package/benchmarks/create-submission-bundle.mjs +0 -190
- package/benchmarks/dry-run-external-adapters.mjs +0 -177
- package/benchmarks/guardbench.js +0 -1265
- package/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +0 -50
- package/benchmarks/output/external/guardbench-external-dry-run.json +0 -69
- package/benchmarks/output/external/guardbench-external-evidence.json +0 -56
- package/benchmarks/output/guardbench-conformance-card.json +0 -63
- package/benchmarks/output/guardbench-manifest.json +0 -414
- package/benchmarks/output/guardbench-raw.json +0 -1279
- package/benchmarks/output/guardbench-summary.json +0 -2123
- package/benchmarks/output/leaderboard/guardbench-leaderboard.json +0 -93
- package/benchmarks/output/leaderboard/guardbench-leaderboard.md +0 -7
- package/benchmarks/output/submission-bundle/guardbench-conformance-card.json +0 -63
- package/benchmarks/output/submission-bundle/guardbench-manifest.json +0 -414
- package/benchmarks/output/submission-bundle/guardbench-raw.json +0 -1279
- package/benchmarks/output/submission-bundle/guardbench-summary.json +0 -2123
- package/benchmarks/output/submission-bundle/schemas/guardbench-adapter-registry.schema.json +0 -69
- package/benchmarks/output/submission-bundle/schemas/guardbench-adapter-self-test.schema.json +0 -156
- package/benchmarks/output/submission-bundle/schemas/guardbench-conformance-card.schema.json +0 -184
- package/benchmarks/output/submission-bundle/schemas/guardbench-external-dry-run.schema.json +0 -74
- package/benchmarks/output/submission-bundle/schemas/guardbench-external-evidence.schema.json +0 -108
- package/benchmarks/output/submission-bundle/schemas/guardbench-external-run.schema.json +0 -160
- package/benchmarks/output/submission-bundle/schemas/guardbench-leaderboard.schema.json +0 -179
- package/benchmarks/output/submission-bundle/schemas/guardbench-manifest.schema.json +0 -213
- package/benchmarks/output/submission-bundle/schemas/guardbench-publication-verification.schema.json +0 -47
- package/benchmarks/output/submission-bundle/schemas/guardbench-raw.schema.json +0 -184
- package/benchmarks/output/submission-bundle/schemas/guardbench-submission-manifest.schema.json +0 -151
- package/benchmarks/output/submission-bundle/schemas/guardbench-summary.schema.json +0 -249
- package/benchmarks/output/submission-bundle/submission-manifest.json +0 -131
- package/benchmarks/output/submission-bundle/validation-report.json +0 -31
- package/benchmarks/output/summary.json +0 -2354
- package/benchmarks/perf-snapshot.js +0 -307
- package/benchmarks/perf.bench.js +0 -169
- package/benchmarks/public-paths.mjs +0 -97
- package/benchmarks/reference-results.js +0 -75
- package/benchmarks/report.js +0 -280
- package/benchmarks/run-external-guardbench.mjs +0 -303
- package/benchmarks/run.js +0 -735
- package/benchmarks/schemas/guardbench-adapter-registry.schema.json +0 -69
- package/benchmarks/schemas/guardbench-adapter-self-test.schema.json +0 -156
- package/benchmarks/schemas/guardbench-conformance-card.schema.json +0 -184
- package/benchmarks/schemas/guardbench-external-dry-run.schema.json +0 -74
- package/benchmarks/schemas/guardbench-external-evidence.schema.json +0 -108
- package/benchmarks/schemas/guardbench-external-run.schema.json +0 -160
- package/benchmarks/schemas/guardbench-leaderboard.schema.json +0 -179
- package/benchmarks/schemas/guardbench-manifest.schema.json +0 -213
- package/benchmarks/schemas/guardbench-publication-verification.schema.json +0 -47
- package/benchmarks/schemas/guardbench-raw.schema.json +0 -184
- package/benchmarks/schemas/guardbench-submission-manifest.schema.json +0 -151
- package/benchmarks/schemas/guardbench-summary.schema.json +0 -249
- package/benchmarks/snapshots/perf-0.22.2.json +0 -123
- package/benchmarks/snapshots/perf-0.23.0.json +0 -123
- package/benchmarks/validate-adapter-module.mjs +0 -107
- package/benchmarks/validate-adapter-registry.mjs +0 -145
- package/benchmarks/validate-adapter-self-test.mjs +0 -96
- package/benchmarks/validate-guardbench-artifacts.mjs +0 -400
- package/benchmarks/verify-external-evidence.mjs +0 -351
- package/benchmarks/verify-publication-artifacts.mjs +0 -309
- package/benchmarks/verify-submission-bundle.mjs +0 -172
- package/docs/AUDREY_PAPER_OUTLINE.md +0 -178
- package/docs/MEMORY_BENCHMARKING.md +0 -59
- package/docs/PRODUCTION_BACKLOG.md +0 -321
- package/docs/assets/audrey-feature-grid.jpg +0 -0
- package/docs/assets/audrey-logo.svg +0 -45
- package/docs/assets/audrey-wordmark.png +0 -0
- package/docs/paper/00-master.md +0 -48
- package/docs/paper/01-introduction.md +0 -27
- package/docs/paper/02-related-work.md +0 -47
- package/docs/paper/03-problem-definition.md +0 -108
- package/docs/paper/04-design.md +0 -164
- package/docs/paper/05-guardbench-spec.md +0 -412
- package/docs/paper/06-implementation.md +0 -115
- package/docs/paper/07-evaluation.md +0 -168
- package/docs/paper/08-discussion-limitations.md +0 -61
- package/docs/paper/09-conclusion.md +0 -11
- package/docs/paper/SUBMISSION_README.md +0 -162
- package/docs/paper/appendix-a-demo-transcript.md +0 -114
- package/docs/paper/arxiv-compile-report.schema.json +0 -116
- package/docs/paper/arxiv-source.schema.json +0 -61
- package/docs/paper/audrey-paper-v1.md +0 -1108
- package/docs/paper/browser-launch-plan.json +0 -209
- package/docs/paper/browser-launch-plan.schema.json +0 -100
- package/docs/paper/browser-launch-results.json +0 -86
- package/docs/paper/browser-launch-results.schema.json +0 -66
- package/docs/paper/claim-register.json +0 -138
- package/docs/paper/claim-register.schema.json +0 -81
- package/docs/paper/evidence-ledger.md +0 -103
- package/docs/paper/output/arxiv/README-arxiv.txt +0 -8
- package/docs/paper/output/arxiv/arxiv-manifest.json +0 -41
- package/docs/paper/output/arxiv/main.tex +0 -951
- package/docs/paper/output/arxiv/references.bib +0 -222
- package/docs/paper/output/arxiv-compile-report.json +0 -24
- package/docs/paper/output/submission-bundle/LICENSE +0 -21
- package/docs/paper/output/submission-bundle/README.md +0 -440
- package/docs/paper/output/submission-bundle/benchmarks/output/adapter-self-test/guardbench-adapter-self-test.json +0 -50
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-dry-run.json +0 -69
- package/docs/paper/output/submission-bundle/benchmarks/output/external/guardbench-external-evidence.json +0 -56
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-conformance-card.json +0 -63
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-manifest.json +0 -414
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-raw.json +0 -1279
- package/docs/paper/output/submission-bundle/benchmarks/output/guardbench-summary.json +0 -2123
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.json +0 -93
- package/docs/paper/output/submission-bundle/benchmarks/output/leaderboard/guardbench-leaderboard.md +0 -7
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/submission-manifest.json +0 -131
- package/docs/paper/output/submission-bundle/benchmarks/output/submission-bundle/validation-report.json +0 -31
- package/docs/paper/output/submission-bundle/benchmarks/output/summary.json +0 -2354
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-adapter-registry.schema.json +0 -69
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-adapter-self-test.schema.json +0 -156
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-conformance-card.schema.json +0 -184
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-external-dry-run.schema.json +0 -74
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-external-evidence.schema.json +0 -108
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-external-run.schema.json +0 -160
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-leaderboard.schema.json +0 -179
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-manifest.schema.json +0 -213
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-publication-verification.schema.json +0 -47
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-raw.schema.json +0 -184
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-submission-manifest.schema.json +0 -151
- package/docs/paper/output/submission-bundle/benchmarks/schemas/guardbench-summary.schema.json +0 -249
- package/docs/paper/output/submission-bundle/docs/AUDREY_PAPER_OUTLINE.md +0 -178
- package/docs/paper/output/submission-bundle/docs/paper/00-master.md +0 -48
- package/docs/paper/output/submission-bundle/docs/paper/01-introduction.md +0 -27
- package/docs/paper/output/submission-bundle/docs/paper/02-related-work.md +0 -47
- package/docs/paper/output/submission-bundle/docs/paper/03-problem-definition.md +0 -108
- package/docs/paper/output/submission-bundle/docs/paper/04-design.md +0 -164
- package/docs/paper/output/submission-bundle/docs/paper/05-guardbench-spec.md +0 -412
- package/docs/paper/output/submission-bundle/docs/paper/06-implementation.md +0 -115
- package/docs/paper/output/submission-bundle/docs/paper/07-evaluation.md +0 -168
- package/docs/paper/output/submission-bundle/docs/paper/08-discussion-limitations.md +0 -61
- package/docs/paper/output/submission-bundle/docs/paper/09-conclusion.md +0 -11
- package/docs/paper/output/submission-bundle/docs/paper/SUBMISSION_README.md +0 -162
- package/docs/paper/output/submission-bundle/docs/paper/appendix-a-demo-transcript.md +0 -114
- package/docs/paper/output/submission-bundle/docs/paper/arxiv-compile-report.schema.json +0 -116
- package/docs/paper/output/submission-bundle/docs/paper/arxiv-source.schema.json +0 -61
- package/docs/paper/output/submission-bundle/docs/paper/audrey-paper-v1.md +0 -1108
- package/docs/paper/output/submission-bundle/docs/paper/browser-launch-plan.json +0 -209
- package/docs/paper/output/submission-bundle/docs/paper/browser-launch-plan.schema.json +0 -100
- package/docs/paper/output/submission-bundle/docs/paper/browser-launch-results.json +0 -86
- package/docs/paper/output/submission-bundle/docs/paper/browser-launch-results.schema.json +0 -66
- package/docs/paper/output/submission-bundle/docs/paper/claim-register.json +0 -138
- package/docs/paper/output/submission-bundle/docs/paper/claim-register.schema.json +0 -81
- package/docs/paper/output/submission-bundle/docs/paper/evidence-ledger.md +0 -103
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/README-arxiv.txt +0 -8
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/arxiv-manifest.json +0 -41
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/main.tex +0 -951
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv/references.bib +0 -222
- package/docs/paper/output/submission-bundle/docs/paper/output/arxiv-compile-report.json +0 -24
- package/docs/paper/output/submission-bundle/docs/paper/paper-submission-bundle.schema.json +0 -70
- package/docs/paper/output/submission-bundle/docs/paper/publication-pack.json +0 -81
- package/docs/paper/output/submission-bundle/docs/paper/publication-pack.schema.json +0 -60
- package/docs/paper/output/submission-bundle/docs/paper/references.bib +0 -222
- package/docs/paper/output/submission-bundle/package.json +0 -235
- package/docs/paper/output/submission-bundle/paper-submission-manifest.json +0 -379
- package/docs/paper/paper-submission-bundle.schema.json +0 -70
- package/docs/paper/publication-pack.json +0 -81
- package/docs/paper/publication-pack.schema.json +0 -60
- package/docs/paper/references.bib +0 -222
- package/scripts/audit-release-completion.mjs +0 -440
- package/scripts/create-arxiv-source.mjs +0 -368
- package/scripts/create-paper-submission-bundle.mjs +0 -214
- package/scripts/finalize-release.mjs +0 -601
- package/scripts/prepare-release-cut.mjs +0 -277
- package/scripts/publish-release-bundle.mjs +0 -252
- package/scripts/publish-release-github-api.mjs +0 -494
- package/scripts/run-vitest.mjs +0 -34
- package/scripts/smoke-cli.js +0 -92
- package/scripts/sync-paper-artifacts.mjs +0 -112
- package/scripts/verify-arxiv-compile.mjs +0 -476
- package/scripts/verify-arxiv-source.mjs +0 -224
- package/scripts/verify-browser-launch-plan.mjs +0 -254
- package/scripts/verify-browser-launch-results.mjs +0 -303
- package/scripts/verify-paper-artifacts.mjs +0 -798
- package/scripts/verify-paper-claims.mjs +0 -254
- package/scripts/verify-paper-submission-bundle.mjs +0 -218
- package/scripts/verify-publication-pack.mjs +0 -210
- package/scripts/verify-python-package.py +0 -201
- package/scripts/verify-release-readiness.mjs +0 -920
|
@@ -1,254 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { dirname, resolve } from 'node:path';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
4
|
-
import { validateSchema } from '../benchmarks/validate-guardbench-artifacts.mjs';
|
|
5
|
-
import { verifyGuardBenchPublicationArtifacts } from '../benchmarks/verify-publication-artifacts.mjs';
|
|
6
|
-
|
|
7
|
-
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
8
|
-
const DEFAULT_REGISTER = 'docs/paper/claim-register.json';
|
|
9
|
-
const DEFAULT_SCHEMA = 'docs/paper/claim-register.schema.json';
|
|
10
|
-
const SEEDED_SECRET = 'sk-guardbench-secret-0000000000000000000000000000';
|
|
11
|
-
|
|
12
|
-
function fromRoot(path) {
|
|
13
|
-
return resolve(ROOT, path);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
function readText(path) {
|
|
17
|
-
const absolute = fromRoot(path);
|
|
18
|
-
if (!existsSync(absolute)) throw new Error(`Missing required file: ${path}`);
|
|
19
|
-
return readFileSync(absolute, 'utf-8');
|
|
20
|
-
}
|
|
21
|
-
|
|
22
|
-
function readJson(path) {
|
|
23
|
-
return JSON.parse(readText(path));
|
|
24
|
-
}
|
|
25
|
-
|
|
26
|
-
function parseArgs(argv = process.argv.slice(2)) {
|
|
27
|
-
const args = {
|
|
28
|
-
register: DEFAULT_REGISTER,
|
|
29
|
-
schema: DEFAULT_SCHEMA,
|
|
30
|
-
json: false,
|
|
31
|
-
};
|
|
32
|
-
|
|
33
|
-
for (let i = 0; i < argv.length; i++) {
|
|
34
|
-
const token = argv[i];
|
|
35
|
-
if (token === '--register' && argv[i + 1]) args.register = argv[++i];
|
|
36
|
-
else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
|
|
37
|
-
else if (token === '--json') args.json = true;
|
|
38
|
-
else if (token === '--help' || token === '-h') args.help = true;
|
|
39
|
-
else throw new Error(`Unknown argument: ${token}`);
|
|
40
|
-
}
|
|
41
|
-
|
|
42
|
-
return args;
|
|
43
|
-
}
|
|
44
|
-
|
|
45
|
-
function usage() {
|
|
46
|
-
return `Usage: node scripts/verify-paper-claims.mjs [options]
|
|
47
|
-
|
|
48
|
-
Options:
|
|
49
|
-
--register <path> Claim register JSON. Default: ${DEFAULT_REGISTER}.
|
|
50
|
-
--schema <path> Claim register schema. Default: ${DEFAULT_SCHEMA}.
|
|
51
|
-
--json Print the machine-readable claim verification report.
|
|
52
|
-
`;
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
function assertTextNeedles(needles, shouldExist, failures) {
|
|
56
|
-
for (const needle of needles) {
|
|
57
|
-
let text;
|
|
58
|
-
try {
|
|
59
|
-
text = readText(needle.path);
|
|
60
|
-
} catch (error) {
|
|
61
|
-
failures.push(error.message);
|
|
62
|
-
continue;
|
|
63
|
-
}
|
|
64
|
-
const normalizedText = text.replace(/\s+/g, ' ');
|
|
65
|
-
const normalizedNeedle = needle.text.replace(/\s+/g, ' ');
|
|
66
|
-
const found = text.includes(needle.text) || normalizedText.includes(normalizedNeedle);
|
|
67
|
-
if (shouldExist && !found)
|
|
68
|
-
failures.push(`${needle.path} is missing claim text: ${needle.text}`);
|
|
69
|
-
if (!shouldExist && found)
|
|
70
|
-
failures.push(`${needle.path} contains forbidden claim text: ${needle.text}`);
|
|
71
|
-
}
|
|
72
|
-
}
|
|
73
|
-
|
|
74
|
-
function guardbenchLocalPassed() {
|
|
75
|
-
const summary = readJson('benchmarks/output/guardbench-summary.json');
|
|
76
|
-
const failures = [];
|
|
77
|
-
if (summary.passed !== 10) failures.push(`GuardBench passed expected 10, got ${summary.passed}`);
|
|
78
|
-
if (summary.scenarios !== 10)
|
|
79
|
-
failures.push(`GuardBench scenarios expected 10, got ${summary.scenarios}`);
|
|
80
|
-
if (summary.redactionLeaks !== 0)
|
|
81
|
-
failures.push(`GuardBench decision redaction leaks expected 0, got ${summary.redactionLeaks}`);
|
|
82
|
-
if (summary.artifactRedactionSweep?.passed !== true)
|
|
83
|
-
failures.push('GuardBench artifact redaction sweep did not pass');
|
|
84
|
-
if (summary.artifactRedactionSweep?.leakCount !== 0) {
|
|
85
|
-
failures.push(
|
|
86
|
-
`GuardBench artifact leak count expected 0, got ${summary.artifactRedactionSweep?.leakCount}`,
|
|
87
|
-
);
|
|
88
|
-
}
|
|
89
|
-
return failures;
|
|
90
|
-
}
|
|
91
|
-
|
|
92
|
-
function noPublishedSecretLeaks() {
|
|
93
|
-
const paths = [
|
|
94
|
-
'benchmarks/output/guardbench-manifest.json',
|
|
95
|
-
'benchmarks/output/guardbench-summary.json',
|
|
96
|
-
'benchmarks/output/guardbench-raw.json',
|
|
97
|
-
];
|
|
98
|
-
return paths.flatMap(path =>
|
|
99
|
-
readText(path).includes(SEEDED_SECRET) ? [`${path} contains the seeded raw secret`] : [],
|
|
100
|
-
);
|
|
101
|
-
}
|
|
102
|
-
|
|
103
|
-
function adapterRegistryHasMem0Zep() {
|
|
104
|
-
const registry = readJson('benchmarks/adapters/registry.json');
|
|
105
|
-
const ids = new Set((registry.adapters ?? []).map(adapter => adapter.id));
|
|
106
|
-
const failures = [];
|
|
107
|
-
if (!ids.has('mem0-platform')) failures.push('Adapter registry missing mem0-platform');
|
|
108
|
-
if (!ids.has('zep-cloud')) failures.push('Adapter registry missing zep-cloud');
|
|
109
|
-
return failures;
|
|
110
|
-
}
|
|
111
|
-
|
|
112
|
-
function externalEvidencePending() {
|
|
113
|
-
const evidence = readJson('benchmarks/output/external/guardbench-external-evidence.json');
|
|
114
|
-
const rows = (evidence.adapters ?? []).filter(adapter =>
|
|
115
|
-
['mem0-platform', 'zep-cloud'].includes(adapter.id),
|
|
116
|
-
);
|
|
117
|
-
const failures = [];
|
|
118
|
-
if (rows.length !== 2)
|
|
119
|
-
failures.push(`External evidence expected Mem0 and Zep rows, got ${rows.length}`);
|
|
120
|
-
if (rows.every(row => row.status === 'verified')) {
|
|
121
|
-
failures.push(
|
|
122
|
-
'External evidence is fully verified but claim register still marks external scores pending',
|
|
123
|
-
);
|
|
124
|
-
}
|
|
125
|
-
for (const row of rows) {
|
|
126
|
-
if (row.status !== 'pending')
|
|
127
|
-
failures.push(
|
|
128
|
-
`External evidence row ${row.id} should remain pending until strict live evidence passes`,
|
|
129
|
-
);
|
|
130
|
-
if (row.evidenceKind !== 'dry-run')
|
|
131
|
-
failures.push(
|
|
132
|
-
`External evidence row ${row.id} should be dry-run evidence before live credentials`,
|
|
133
|
-
);
|
|
134
|
-
}
|
|
135
|
-
return failures;
|
|
136
|
-
}
|
|
137
|
-
|
|
138
|
-
function externalEvidenceNoSecrets() {
|
|
139
|
-
const text = readText('benchmarks/output/external/guardbench-external-evidence.json');
|
|
140
|
-
const evidence = JSON.parse(text);
|
|
141
|
-
const failures = [];
|
|
142
|
-
if (text.includes('runtime-key'))
|
|
143
|
-
failures.push('External evidence report contains test runtime-key');
|
|
144
|
-
for (const row of evidence.adapters ?? []) {
|
|
145
|
-
if (row.secretLeakCount !== 0)
|
|
146
|
-
failures.push(
|
|
147
|
-
`External evidence row ${row.id} reports ${row.secretLeakCount} credential leak(s)`,
|
|
148
|
-
);
|
|
149
|
-
}
|
|
150
|
-
return failures;
|
|
151
|
-
}
|
|
152
|
-
|
|
153
|
-
function paperStageBoundaryExcludesExternalScores() {
|
|
154
|
-
const paper = readText('docs/paper/audrey-paper-v1.md');
|
|
155
|
-
const failures = [];
|
|
156
|
-
if (!paper.includes('this paper does not report external-system GuardBench scores')) {
|
|
157
|
-
failures.push('Paper missing explicit external-score exclusion');
|
|
158
|
-
}
|
|
159
|
-
if (
|
|
160
|
-
!paper.includes(
|
|
161
|
-
'External scores added only when live adapter runs and raw outputs are published',
|
|
162
|
-
)
|
|
163
|
-
) {
|
|
164
|
-
failures.push('Paper missing Stage-B external-score condition');
|
|
165
|
-
}
|
|
166
|
-
return failures;
|
|
167
|
-
}
|
|
168
|
-
|
|
169
|
-
async function publicationVerifierOk() {
|
|
170
|
-
const report = await verifyGuardBenchPublicationArtifacts();
|
|
171
|
-
return report.ok ? [] : report.failures.map(failure => `publication verifier: ${failure}`);
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
async function runArtifactCheck(name) {
|
|
175
|
-
if (name === 'adapter-registry-has-mem0-zep') return adapterRegistryHasMem0Zep();
|
|
176
|
-
if (name === 'external-evidence-no-secrets') return externalEvidenceNoSecrets();
|
|
177
|
-
if (name === 'external-evidence-pending') return externalEvidencePending();
|
|
178
|
-
if (name === 'guardbench-local-passed') return guardbenchLocalPassed();
|
|
179
|
-
if (name === 'no-published-secret-leaks') return noPublishedSecretLeaks();
|
|
180
|
-
if (name === 'paper-stage-boundary-excludes-external-scores')
|
|
181
|
-
return paperStageBoundaryExcludesExternalScores();
|
|
182
|
-
if (name === 'publication-verifier-ok') return publicationVerifierOk();
|
|
183
|
-
return [`Unknown claim artifact check: ${name}`];
|
|
184
|
-
}
|
|
185
|
-
|
|
186
|
-
export async function verifyPaperClaims(options = {}) {
|
|
187
|
-
const register = readJson(options.register ?? DEFAULT_REGISTER);
|
|
188
|
-
const schema = readJson(options.schema ?? DEFAULT_SCHEMA);
|
|
189
|
-
const schemaFailures = validateSchema(register, schema, 'audrey-paper-claim-register');
|
|
190
|
-
const claimReports = [];
|
|
191
|
-
|
|
192
|
-
for (const claim of register.claims ?? []) {
|
|
193
|
-
const failures = [];
|
|
194
|
-
assertTextNeedles(claim.requiredText ?? [], true, failures);
|
|
195
|
-
assertTextNeedles(claim.forbiddenText ?? [], false, failures);
|
|
196
|
-
for (const evidence of claim.evidence ?? []) {
|
|
197
|
-
const [path] = evidence.split('#');
|
|
198
|
-
if (!existsSync(fromRoot(path)))
|
|
199
|
-
failures.push(`Missing evidence file for ${claim.id}: ${path}`);
|
|
200
|
-
}
|
|
201
|
-
for (const check of claim.artifactChecks ?? []) {
|
|
202
|
-
failures.push(...(await runArtifactCheck(check)));
|
|
203
|
-
}
|
|
204
|
-
claimReports.push({
|
|
205
|
-
id: claim.id,
|
|
206
|
-
status: claim.status,
|
|
207
|
-
ok: failures.length === 0,
|
|
208
|
-
artifactChecks: claim.artifactChecks ?? [],
|
|
209
|
-
failures,
|
|
210
|
-
});
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
const failures = [
|
|
214
|
-
...schemaFailures.map(failure => `claim register schema: ${failure}`),
|
|
215
|
-
...claimReports.flatMap(report => report.failures.map(failure => `${report.id}: ${failure}`)),
|
|
216
|
-
];
|
|
217
|
-
|
|
218
|
-
return {
|
|
219
|
-
schemaVersion: '1.0.0',
|
|
220
|
-
suite: 'Audrey paper claim verification',
|
|
221
|
-
generatedAt: new Date().toISOString(),
|
|
222
|
-
ok: failures.length === 0,
|
|
223
|
-
register: fromRoot(options.register ?? DEFAULT_REGISTER),
|
|
224
|
-
claims: claimReports,
|
|
225
|
-
failures,
|
|
226
|
-
};
|
|
227
|
-
}
|
|
228
|
-
|
|
229
|
-
async function main() {
|
|
230
|
-
const args = parseArgs();
|
|
231
|
-
if (args.help) {
|
|
232
|
-
console.log(usage());
|
|
233
|
-
return;
|
|
234
|
-
}
|
|
235
|
-
|
|
236
|
-
const report = await verifyPaperClaims(args);
|
|
237
|
-
if (args.json) {
|
|
238
|
-
console.log(JSON.stringify(report, null, 2));
|
|
239
|
-
} else if (report.ok) {
|
|
240
|
-
console.log(`Paper claim verification passed: ${report.claims.length} claim(s)`);
|
|
241
|
-
} else {
|
|
242
|
-
console.error('Paper claim verification failed:');
|
|
243
|
-
for (const failure of report.failures) console.error(`- ${failure}`);
|
|
244
|
-
}
|
|
245
|
-
|
|
246
|
-
if (!report.ok) process.exit(1);
|
|
247
|
-
}
|
|
248
|
-
|
|
249
|
-
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
250
|
-
main().catch(error => {
|
|
251
|
-
console.error(error.stack ?? error.message);
|
|
252
|
-
process.exit(1);
|
|
253
|
-
});
|
|
254
|
-
}
|
|
@@ -1,218 +0,0 @@
|
|
|
1
|
-
import { createHash } from 'node:crypto';
|
|
2
|
-
import { existsSync, readFileSync, readdirSync } from 'node:fs';
|
|
3
|
-
import { join, relative, resolve } from 'node:path';
|
|
4
|
-
import { fileURLToPath } from 'node:url';
|
|
5
|
-
import { validateSchema } from '../benchmarks/validate-guardbench-artifacts.mjs';
|
|
6
|
-
import { publicPath, scanFilesForLocalPaths } from '../benchmarks/public-paths.mjs';
|
|
7
|
-
|
|
8
|
-
const ROOT = resolve(fileURLToPath(new URL('..', import.meta.url)));
|
|
9
|
-
const DEFAULT_DIR = 'docs/paper/output/submission-bundle';
|
|
10
|
-
const DEFAULT_SCHEMA = 'docs/paper/paper-submission-bundle.schema.json';
|
|
11
|
-
const REQUIRED_FILES = [
|
|
12
|
-
'README.md',
|
|
13
|
-
'LICENSE',
|
|
14
|
-
'package.json',
|
|
15
|
-
'docs/paper/arxiv-compile-report.schema.json',
|
|
16
|
-
'docs/paper/output/arxiv/main.tex',
|
|
17
|
-
'docs/paper/output/arxiv/references.bib',
|
|
18
|
-
'docs/paper/output/arxiv/arxiv-manifest.json',
|
|
19
|
-
'docs/paper/output/arxiv-compile-report.json',
|
|
20
|
-
'docs/paper/audrey-paper-v1.md',
|
|
21
|
-
'docs/paper/browser-launch-plan.json',
|
|
22
|
-
'docs/paper/browser-launch-plan.schema.json',
|
|
23
|
-
'docs/paper/browser-launch-results.json',
|
|
24
|
-
'docs/paper/browser-launch-results.schema.json',
|
|
25
|
-
'docs/paper/claim-register.json',
|
|
26
|
-
'docs/paper/publication-pack.json',
|
|
27
|
-
'docs/paper/evidence-ledger.md',
|
|
28
|
-
'docs/paper/references.bib',
|
|
29
|
-
'docs/paper/SUBMISSION_README.md',
|
|
30
|
-
'benchmarks/output/guardbench-summary.json',
|
|
31
|
-
'benchmarks/output/guardbench-raw.json',
|
|
32
|
-
'benchmarks/output/external/guardbench-external-evidence.json',
|
|
33
|
-
'benchmarks/output/submission-bundle/submission-manifest.json',
|
|
34
|
-
];
|
|
35
|
-
const PASSED_COMPILE_FILES = [
|
|
36
|
-
'docs/paper/output/arxiv-compile/main.pdf',
|
|
37
|
-
'docs/paper/output/arxiv-compile/arxiv-compile.log',
|
|
38
|
-
];
|
|
39
|
-
|
|
40
|
-
function fromRoot(path) {
|
|
41
|
-
return resolve(ROOT, path);
|
|
42
|
-
}
|
|
43
|
-
|
|
44
|
-
function readJson(path) {
|
|
45
|
-
return JSON.parse(readFileSync(path, 'utf-8'));
|
|
46
|
-
}
|
|
47
|
-
|
|
48
|
-
function sha256File(path) {
|
|
49
|
-
return createHash('sha256').update(readFileSync(path)).digest('hex');
|
|
50
|
-
}
|
|
51
|
-
|
|
52
|
-
function walkFiles(dir, root = dir) {
|
|
53
|
-
return readdirSync(dir, { withFileTypes: true }).flatMap(entry => {
|
|
54
|
-
const path = join(dir, entry.name);
|
|
55
|
-
if (entry.isDirectory()) return walkFiles(path, root);
|
|
56
|
-
return relative(root, path).replaceAll('\\', '/');
|
|
57
|
-
});
|
|
58
|
-
}
|
|
59
|
-
|
|
60
|
-
function parseArgs(argv = process.argv.slice(2)) {
|
|
61
|
-
const args = {
|
|
62
|
-
dir: DEFAULT_DIR,
|
|
63
|
-
schema: DEFAULT_SCHEMA,
|
|
64
|
-
checkSourceFreshness: true,
|
|
65
|
-
json: false,
|
|
66
|
-
};
|
|
67
|
-
for (let i = 0; i < argv.length; i++) {
|
|
68
|
-
const token = argv[i];
|
|
69
|
-
if ((token === '--dir' || token === '--bundle-dir') && argv[i + 1]) args.dir = argv[++i];
|
|
70
|
-
else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
|
|
71
|
-
else if (token === '--no-source-check') args.checkSourceFreshness = false;
|
|
72
|
-
else if (token === '--json') args.json = true;
|
|
73
|
-
else if (token === '--help' || token === '-h') args.help = true;
|
|
74
|
-
else throw new Error(`Unknown argument: ${token}`);
|
|
75
|
-
}
|
|
76
|
-
return args;
|
|
77
|
-
}
|
|
78
|
-
|
|
79
|
-
function usage() {
|
|
80
|
-
return `Usage: node scripts/verify-paper-submission-bundle.mjs [options]
|
|
81
|
-
|
|
82
|
-
Options:
|
|
83
|
-
--dir <path> Paper submission bundle directory. Default: ${DEFAULT_DIR}.
|
|
84
|
-
--schema <path> Bundle manifest schema. Default: ${DEFAULT_SCHEMA}.
|
|
85
|
-
--no-source-check
|
|
86
|
-
Skip checking bundled hashes against current source files.
|
|
87
|
-
--json Print the machine-readable verification report.
|
|
88
|
-
`;
|
|
89
|
-
}
|
|
90
|
-
|
|
91
|
-
export function verifyPaperSubmissionBundle(options = {}) {
|
|
92
|
-
const dir = fromRoot(options.dir ?? DEFAULT_DIR);
|
|
93
|
-
const schemaPath = fromRoot(options.schema ?? DEFAULT_SCHEMA);
|
|
94
|
-
const checkSourceFreshness = options.checkSourceFreshness !== false;
|
|
95
|
-
const manifestPath = join(dir, 'paper-submission-manifest.json');
|
|
96
|
-
const failures = [];
|
|
97
|
-
let manifest;
|
|
98
|
-
|
|
99
|
-
try {
|
|
100
|
-
manifest = readJson(manifestPath);
|
|
101
|
-
} catch (error) {
|
|
102
|
-
return {
|
|
103
|
-
ok: false,
|
|
104
|
-
dir,
|
|
105
|
-
files: [],
|
|
106
|
-
failures: [`paper-submission-manifest.json: ${error.message}`],
|
|
107
|
-
};
|
|
108
|
-
}
|
|
109
|
-
|
|
110
|
-
try {
|
|
111
|
-
const schema = readJson(schemaPath);
|
|
112
|
-
failures.push(...validateSchema(manifest, schema, 'audrey-paper-submission-bundle'));
|
|
113
|
-
} catch (error) {
|
|
114
|
-
failures.push(`schema: ${error.message}`);
|
|
115
|
-
}
|
|
116
|
-
|
|
117
|
-
const listed = new Map((manifest.files ?? []).map(file => [file.path, file]));
|
|
118
|
-
for (const file of REQUIRED_FILES) {
|
|
119
|
-
if (!listed.has(file))
|
|
120
|
-
failures.push(`paper-submission-manifest.json: missing required file record ${file}`);
|
|
121
|
-
}
|
|
122
|
-
const compileReport = listed.has('docs/paper/output/arxiv-compile-report.json')
|
|
123
|
-
? readJson(join(dir, 'docs/paper/output/arxiv-compile-report.json'))
|
|
124
|
-
: null;
|
|
125
|
-
if (compileReport?.status === 'passed') {
|
|
126
|
-
for (const file of PASSED_COMPILE_FILES) {
|
|
127
|
-
if (!listed.has(file))
|
|
128
|
-
failures.push(`paper-submission-manifest.json: missing compile-proof file record ${file}`);
|
|
129
|
-
}
|
|
130
|
-
}
|
|
131
|
-
if (listed.has('paper-submission-manifest.json')) {
|
|
132
|
-
failures.push('paper-submission-manifest.json: must not include a self-hash file record');
|
|
133
|
-
}
|
|
134
|
-
|
|
135
|
-
for (const [file, record] of listed) {
|
|
136
|
-
const path = join(dir, file);
|
|
137
|
-
if (!existsSync(path)) {
|
|
138
|
-
failures.push(`${file}: listed in manifest but missing from bundle`);
|
|
139
|
-
continue;
|
|
140
|
-
}
|
|
141
|
-
const bytes = readFileSync(path).byteLength;
|
|
142
|
-
if (record.bytes !== bytes) failures.push(`${file}: byte length mismatch`);
|
|
143
|
-
const sha256 = sha256File(path);
|
|
144
|
-
if (record.sha256 !== sha256) failures.push(`${file}: sha256 mismatch`);
|
|
145
|
-
if (checkSourceFreshness && record.source) {
|
|
146
|
-
const sourcePath = fromRoot(record.source);
|
|
147
|
-
if (existsSync(sourcePath)) {
|
|
148
|
-
const sourceBytes = readFileSync(sourcePath).byteLength;
|
|
149
|
-
const sourceSha256 = sha256File(sourcePath);
|
|
150
|
-
if (record.bytes !== sourceBytes || record.sha256 !== sourceSha256) {
|
|
151
|
-
failures.push(`${file}: source file has changed since bundle creation`);
|
|
152
|
-
}
|
|
153
|
-
}
|
|
154
|
-
}
|
|
155
|
-
}
|
|
156
|
-
|
|
157
|
-
const actualFiles = walkFiles(dir)
|
|
158
|
-
.filter(file => file !== 'paper-submission-manifest.json')
|
|
159
|
-
.sort();
|
|
160
|
-
const listedFiles = [...listed.keys()].sort();
|
|
161
|
-
const actualSet = new Set(actualFiles);
|
|
162
|
-
const listedSet = new Set(listedFiles);
|
|
163
|
-
for (const file of actualFiles) {
|
|
164
|
-
if (!listedSet.has(file)) failures.push(`${file}: present in bundle but missing from manifest`);
|
|
165
|
-
}
|
|
166
|
-
for (const file of listedFiles) {
|
|
167
|
-
if (!actualSet.has(file))
|
|
168
|
-
failures.push(`${file}: listed in manifest but not present in bundle`);
|
|
169
|
-
}
|
|
170
|
-
for (const file of scanFilesForLocalPaths(dir, actualFiles)) {
|
|
171
|
-
failures.push(`${file}: contains a local absolute path`);
|
|
172
|
-
}
|
|
173
|
-
if (manifest.claimVerification?.ok !== true)
|
|
174
|
-
failures.push('paper-submission-manifest.json: claimVerification is not ok');
|
|
175
|
-
if (manifest.publicationPackVerification?.ok !== true)
|
|
176
|
-
failures.push('paper-submission-manifest.json: publicationPackVerification is not ok');
|
|
177
|
-
if (manifest.guardBenchSnapshot?.passed !== 10)
|
|
178
|
-
failures.push('paper-submission-manifest.json: GuardBench passed count is not 10');
|
|
179
|
-
if (manifest.guardBenchSnapshot?.scenarios !== 10)
|
|
180
|
-
failures.push('paper-submission-manifest.json: GuardBench scenario count is not 10');
|
|
181
|
-
if (manifest.guardBenchSnapshot?.redactionLeaks !== 0)
|
|
182
|
-
failures.push('paper-submission-manifest.json: GuardBench decision redaction leaks are not 0');
|
|
183
|
-
if (manifest.guardBenchSnapshot?.artifactLeaks !== 0)
|
|
184
|
-
failures.push('paper-submission-manifest.json: GuardBench artifact leaks are not 0');
|
|
185
|
-
|
|
186
|
-
return {
|
|
187
|
-
ok: failures.length === 0,
|
|
188
|
-
dir: publicPath(dir),
|
|
189
|
-
manifestPath: publicPath(manifestPath),
|
|
190
|
-
files: listedFiles,
|
|
191
|
-
failures,
|
|
192
|
-
};
|
|
193
|
-
}
|
|
194
|
-
|
|
195
|
-
async function main() {
|
|
196
|
-
const args = parseArgs();
|
|
197
|
-
if (args.help) {
|
|
198
|
-
console.log(usage());
|
|
199
|
-
return;
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
const report = verifyPaperSubmissionBundle(args);
|
|
203
|
-
if (args.json) console.log(JSON.stringify(report, null, 2));
|
|
204
|
-
else if (report.ok) console.log(`Paper submission bundle verification passed: ${report.dir}`);
|
|
205
|
-
else {
|
|
206
|
-
console.error('Paper submission bundle verification failed:');
|
|
207
|
-
for (const failure of report.failures) console.error(`- ${failure}`);
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
if (!report.ok) process.exit(1);
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
214
|
-
main().catch(error => {
|
|
215
|
-
console.error(error.stack ?? error.message);
|
|
216
|
-
process.exit(1);
|
|
217
|
-
});
|
|
218
|
-
}
|
|
@@ -1,210 +0,0 @@
|
|
|
1
|
-
import { existsSync, readFileSync } from 'node:fs';
|
|
2
|
-
import { dirname, resolve } from 'node:path';
|
|
3
|
-
import { fileURLToPath } from 'node:url';
|
|
4
|
-
import { publicPath } from '../benchmarks/public-paths.mjs';
|
|
5
|
-
import { validateSchema } from '../benchmarks/validate-guardbench-artifacts.mjs';
|
|
6
|
-
import { verifyPaperClaims } from './verify-paper-claims.mjs';
|
|
7
|
-
|
|
8
|
-
const ROOT = resolve(dirname(fileURLToPath(import.meta.url)), '..');
|
|
9
|
-
const DEFAULT_PACK = 'docs/paper/publication-pack.json';
|
|
10
|
-
const DEFAULT_SCHEMA = 'docs/paper/publication-pack.schema.json';
|
|
11
|
-
const SEEDED_SECRET = 'sk-guardbench-secret-0000000000000000000000000000';
|
|
12
|
-
const REQUIRED_ENTRIES = [
|
|
13
|
-
'arxiv-title',
|
|
14
|
-
'arxiv-abstract',
|
|
15
|
-
'hacker-news-title',
|
|
16
|
-
'hacker-news-comment',
|
|
17
|
-
'reddit-title',
|
|
18
|
-
'reddit-body',
|
|
19
|
-
'x-post-1',
|
|
20
|
-
'x-post-2',
|
|
21
|
-
'linkedin-post',
|
|
22
|
-
];
|
|
23
|
-
const X_URL_RESERVED_CHARS = 24;
|
|
24
|
-
|
|
25
|
-
function fromRoot(path) {
|
|
26
|
-
return resolve(ROOT, path);
|
|
27
|
-
}
|
|
28
|
-
|
|
29
|
-
function readText(path) {
|
|
30
|
-
const absolute = fromRoot(path);
|
|
31
|
-
if (!existsSync(absolute)) throw new Error(`Missing required file: ${path}`);
|
|
32
|
-
return readFileSync(absolute, 'utf-8');
|
|
33
|
-
}
|
|
34
|
-
|
|
35
|
-
function readJson(path) {
|
|
36
|
-
return JSON.parse(readText(path));
|
|
37
|
-
}
|
|
38
|
-
|
|
39
|
-
function parseArgs(argv = process.argv.slice(2)) {
|
|
40
|
-
const args = {
|
|
41
|
-
pack: DEFAULT_PACK,
|
|
42
|
-
schema: DEFAULT_SCHEMA,
|
|
43
|
-
json: false,
|
|
44
|
-
};
|
|
45
|
-
|
|
46
|
-
for (let i = 0; i < argv.length; i++) {
|
|
47
|
-
const token = argv[i];
|
|
48
|
-
if (token === '--pack' && argv[i + 1]) args.pack = argv[++i];
|
|
49
|
-
else if (token === '--schema' && argv[i + 1]) args.schema = argv[++i];
|
|
50
|
-
else if (token === '--json') args.json = true;
|
|
51
|
-
else if (token === '--help' || token === '-h') args.help = true;
|
|
52
|
-
else throw new Error(`Unknown argument: ${token}`);
|
|
53
|
-
}
|
|
54
|
-
|
|
55
|
-
return args;
|
|
56
|
-
}
|
|
57
|
-
|
|
58
|
-
function usage() {
|
|
59
|
-
return `Usage: node scripts/verify-publication-pack.mjs [options]
|
|
60
|
-
|
|
61
|
-
Options:
|
|
62
|
-
--pack <path> Publication pack JSON. Default: ${DEFAULT_PACK}.
|
|
63
|
-
--schema <path> Publication pack schema. Default: ${DEFAULT_SCHEMA}.
|
|
64
|
-
--json Print the machine-readable verification report.
|
|
65
|
-
`;
|
|
66
|
-
}
|
|
67
|
-
|
|
68
|
-
function referencesPendingClaim(entry, claimMap) {
|
|
69
|
-
return entry.claimIds.some(id => claimMap.get(id)?.status === 'pending');
|
|
70
|
-
}
|
|
71
|
-
|
|
72
|
-
function hasPendingBoundaryLanguage(text) {
|
|
73
|
-
return /\b(pending|deferred|does not report|not reporting|not claimed|Stage-B|live credentialed)\b/i.test(
|
|
74
|
-
text,
|
|
75
|
-
);
|
|
76
|
-
}
|
|
77
|
-
|
|
78
|
-
function validateEntry(entry, claimMap, forbiddenNeedles) {
|
|
79
|
-
const failures = [];
|
|
80
|
-
const reservedUrlChars = Number.isInteger(entry.reservedUrlChars) ? entry.reservedUrlChars : 0;
|
|
81
|
-
if (entry.text.length > entry.maxChars) {
|
|
82
|
-
failures.push(
|
|
83
|
-
`${entry.id}: text length ${entry.text.length} exceeds maxChars ${entry.maxChars}`,
|
|
84
|
-
);
|
|
85
|
-
}
|
|
86
|
-
if (entry.text.includes(SEEDED_SECRET)) failures.push(`${entry.id}: contains seeded raw secret`);
|
|
87
|
-
if (entry.text.includes('runtime-key'))
|
|
88
|
-
failures.push(`${entry.id}: contains runtime-key test credential`);
|
|
89
|
-
for (const claimId of entry.claimIds) {
|
|
90
|
-
if (!claimMap.has(claimId)) failures.push(`${entry.id}: unknown claim id ${claimId}`);
|
|
91
|
-
}
|
|
92
|
-
for (const needle of forbiddenNeedles) {
|
|
93
|
-
if (entry.text.includes(needle))
|
|
94
|
-
failures.push(`${entry.id}: contains forbidden claim text: ${needle}`);
|
|
95
|
-
}
|
|
96
|
-
if (referencesPendingClaim(entry, claimMap) && !hasPendingBoundaryLanguage(entry.text)) {
|
|
97
|
-
failures.push(
|
|
98
|
-
`${entry.id}: references a pending claim without explicit pending/deferred boundary language`,
|
|
99
|
-
);
|
|
100
|
-
}
|
|
101
|
-
if (/10\/10/.test(entry.text) && !/\b(local|Stage-A)\b/i.test(entry.text)) {
|
|
102
|
-
failures.push(`${entry.id}: 10/10 claim must be scoped as local or Stage-A`);
|
|
103
|
-
}
|
|
104
|
-
if (/\b(Mem0|Zep)\b/.test(entry.text) && !hasPendingBoundaryLanguage(entry.text)) {
|
|
105
|
-
failures.push(`${entry.id}: Mem0/Zep mention must include pending/deferred boundary language`);
|
|
106
|
-
}
|
|
107
|
-
if (entry.platform === 'hacker-news' && entry.kind === 'title' && entry.text.length > 80) {
|
|
108
|
-
failures.push(`${entry.id}: Hacker News title should stay at or below 80 characters`);
|
|
109
|
-
}
|
|
110
|
-
if (entry.platform === 'x' && entry.text.length > 280) {
|
|
111
|
-
failures.push(`${entry.id}: X post should stay at or below 280 characters`);
|
|
112
|
-
}
|
|
113
|
-
if (entry.platform === 'x' && entry.requiresArtifactUrl === true) {
|
|
114
|
-
if (!Number.isInteger(entry.reservedUrlChars)) {
|
|
115
|
-
failures.push(`${entry.id}: X post requiring an artifact URL must set reservedUrlChars`);
|
|
116
|
-
} else if (entry.reservedUrlChars < X_URL_RESERVED_CHARS) {
|
|
117
|
-
failures.push(
|
|
118
|
-
`${entry.id}: X artifact URL reserve must be at least ${X_URL_RESERVED_CHARS} characters`,
|
|
119
|
-
);
|
|
120
|
-
}
|
|
121
|
-
if (entry.text.length + reservedUrlChars > entry.maxChars) {
|
|
122
|
-
failures.push(
|
|
123
|
-
`${entry.id}: text length ${entry.text.length} plus URL reserve ${reservedUrlChars} exceeds maxChars ${entry.maxChars}`,
|
|
124
|
-
);
|
|
125
|
-
}
|
|
126
|
-
}
|
|
127
|
-
return failures;
|
|
128
|
-
}
|
|
129
|
-
|
|
130
|
-
export async function verifyPublicationPack(options = {}) {
|
|
131
|
-
const pack = readJson(options.pack ?? DEFAULT_PACK);
|
|
132
|
-
const schema = readJson(options.schema ?? DEFAULT_SCHEMA);
|
|
133
|
-
const claimReport = await verifyPaperClaims();
|
|
134
|
-
const claimRegister = readJson(pack.claimRegister);
|
|
135
|
-
const claimMap = new Map((claimRegister.claims ?? []).map(claim => [claim.id, claim]));
|
|
136
|
-
const forbiddenNeedles = (claimRegister.claims ?? []).flatMap(claim =>
|
|
137
|
-
(claim.forbiddenText ?? []).map(needle => needle.text),
|
|
138
|
-
);
|
|
139
|
-
|
|
140
|
-
const schemaFailures = validateSchema(pack, schema, 'audrey-publication-pack');
|
|
141
|
-
const ids = new Set();
|
|
142
|
-
const entryReports = [];
|
|
143
|
-
const failures = [...schemaFailures.map(failure => `publication pack schema: ${failure}`)];
|
|
144
|
-
|
|
145
|
-
if (!claimReport.ok)
|
|
146
|
-
failures.push(...claimReport.failures.map(failure => `claim verifier: ${failure}`));
|
|
147
|
-
|
|
148
|
-
for (const entry of pack.entries ?? []) {
|
|
149
|
-
const entryFailures = [];
|
|
150
|
-
const reservedUrlChars = Number.isInteger(entry.reservedUrlChars) ? entry.reservedUrlChars : 0;
|
|
151
|
-
if (ids.has(entry.id)) entryFailures.push(`${entry.id}: duplicate entry id`);
|
|
152
|
-
ids.add(entry.id);
|
|
153
|
-
entryFailures.push(...validateEntry(entry, claimMap, forbiddenNeedles));
|
|
154
|
-
entryReports.push({
|
|
155
|
-
id: entry.id,
|
|
156
|
-
platform: entry.platform,
|
|
157
|
-
kind: entry.kind,
|
|
158
|
-
chars: entry.text.length,
|
|
159
|
-
maxChars: entry.maxChars,
|
|
160
|
-
requiresArtifactUrl: entry.requiresArtifactUrl === true,
|
|
161
|
-
reservedUrlChars,
|
|
162
|
-
effectiveChars: entry.text.length + reservedUrlChars,
|
|
163
|
-
claimIds: entry.claimIds,
|
|
164
|
-
ok: entryFailures.length === 0,
|
|
165
|
-
failures: entryFailures,
|
|
166
|
-
});
|
|
167
|
-
failures.push(...entryFailures);
|
|
168
|
-
}
|
|
169
|
-
|
|
170
|
-
for (const id of REQUIRED_ENTRIES) {
|
|
171
|
-
if (!ids.has(id)) failures.push(`Missing required publication-pack entry: ${id}`);
|
|
172
|
-
}
|
|
173
|
-
|
|
174
|
-
return {
|
|
175
|
-
schemaVersion: '1.0.0',
|
|
176
|
-
suite: 'Audrey publication pack verification',
|
|
177
|
-
generatedAt: new Date().toISOString(),
|
|
178
|
-
ok: failures.length === 0,
|
|
179
|
-
pack: publicPath(fromRoot(options.pack ?? DEFAULT_PACK)),
|
|
180
|
-
entries: entryReports,
|
|
181
|
-
failures,
|
|
182
|
-
};
|
|
183
|
-
}
|
|
184
|
-
|
|
185
|
-
async function main() {
|
|
186
|
-
const args = parseArgs();
|
|
187
|
-
if (args.help) {
|
|
188
|
-
console.log(usage());
|
|
189
|
-
return;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
const report = await verifyPublicationPack(args);
|
|
193
|
-
if (args.json) {
|
|
194
|
-
console.log(JSON.stringify(report, null, 2));
|
|
195
|
-
} else if (report.ok) {
|
|
196
|
-
console.log(`Publication pack verification passed: ${report.entries.length} entry(s)`);
|
|
197
|
-
} else {
|
|
198
|
-
console.error('Publication pack verification failed:');
|
|
199
|
-
for (const failure of report.failures) console.error(`- ${failure}`);
|
|
200
|
-
}
|
|
201
|
-
|
|
202
|
-
if (!report.ok) process.exit(1);
|
|
203
|
-
}
|
|
204
|
-
|
|
205
|
-
if (process.argv[1] && resolve(process.argv[1]) === fileURLToPath(import.meta.url)) {
|
|
206
|
-
main().catch(error => {
|
|
207
|
-
console.error(error.stack ?? error.message);
|
|
208
|
-
process.exit(1);
|
|
209
|
-
});
|
|
210
|
-
}
|