@zanii/blackbox 0.2.0 → 0.4.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +26 -1
- package/dist/a2a/index.d.ts +77 -0
- package/dist/a2a/index.js +305 -0
- package/dist/agents/index.d.ts +8 -0
- package/dist/analysis/accuracy.d.ts +24 -0
- package/dist/analysis/accuracy.js +45 -0
- package/dist/analysis/credential.d.ts +101 -0
- package/dist/analysis/credential.js +142 -0
- package/dist/analysis/faults.js +115 -0
- package/dist/analysis/grounding.d.ts +122 -0
- package/dist/analysis/grounding.js +445 -0
- package/dist/analysis/hallucination.d.ts +32 -0
- package/dist/analysis/hallucination.js +357 -0
- package/dist/analysis/index.d.ts +23 -0
- package/dist/analysis/index.js +93 -0
- package/dist/analysis/memory.d.ts +8 -0
- package/dist/analysis/memory.js +35 -8
- package/dist/analysis/reference.d.ts +49 -0
- package/dist/analysis/reference.js +164 -0
- package/dist/analysis/taxonomy.d.ts +18 -0
- package/dist/analysis/taxonomy.js +66 -0
- package/dist/approvals/index.d.ts +23 -0
- package/dist/approvals/index.js +48 -0
- package/dist/archive/index.d.ts +39 -0
- package/dist/archive/index.js +96 -0
- package/dist/archive/parquet.d.ts +2 -0
- package/dist/archive/parquet.js +185 -0
- package/dist/badge/index.d.ts +16 -0
- package/dist/badge/index.js +48 -0
- package/dist/bom/index.d.ts +14 -0
- package/dist/bom/index.js +152 -0
- package/dist/cli.js +114 -10
- package/dist/compliance/art12.d.ts +35 -0
- package/dist/compliance/art12.js +190 -0
- package/dist/compliance/index.d.ts +36 -2
- package/dist/compliance/index.js +78 -11
- package/dist/compliance/zanii.d.ts +29 -0
- package/dist/compliance/zanii.js +84 -0
- package/dist/constitution/index.d.ts +57 -0
- package/dist/constitution/index.js +131 -0
- package/dist/cv/index.d.ts +39 -0
- package/dist/cv/index.js +108 -0
- package/dist/disclosure/index.d.ts +31 -0
- package/dist/disclosure/index.js +113 -0
- package/dist/encryption/index.d.ts +9 -0
- package/dist/encryption/index.js +31 -0
- package/dist/evidence/index.d.ts +60 -0
- package/dist/evidence/index.js +151 -0
- package/dist/federation/index.d.ts +35 -0
- package/dist/federation/index.js +102 -0
- package/dist/finance/index.d.ts +126 -0
- package/dist/finance/index.js +320 -0
- package/dist/fleet/index.js +9 -0
- package/dist/gov/index.d.ts +108 -0
- package/dist/gov/index.js +225 -0
- package/dist/health/index.d.ts +120 -0
- package/dist/health/index.js +233 -0
- package/dist/index.d.ts +34 -5
- package/dist/index.js +34 -5
- package/dist/memory/index.d.ts +36 -0
- package/dist/memory/index.js +85 -0
- package/dist/occurrence/index.d.ts +11 -0
- package/dist/occurrence/index.js +18 -0
- package/dist/ocsf/index.d.ts +1 -1
- package/dist/ocsf/index.js +36 -3
- package/dist/otlp/index.d.ts +8 -1
- package/dist/otlp/index.js +258 -1
- package/dist/packs/index.js +44 -4
- package/dist/policy/delta.js +7 -1
- package/dist/policy/index.d.ts +40 -6
- package/dist/policy/index.js +186 -8
- package/dist/policy/zanii.d.ts +31 -0
- package/dist/policy/zanii.js +87 -0
- package/dist/pq/index.d.ts +23 -0
- package/dist/pq/index.js +104 -0
- package/dist/search/index.d.ts +23 -0
- package/dist/search/index.js +69 -0
- package/dist/session/index.d.ts +107 -1
- package/dist/session/index.js +189 -11
- package/dist/sla/index.d.ts +61 -0
- package/dist/sla/index.js +197 -0
- package/dist/succession/index.d.ts +50 -0
- package/dist/succession/index.js +123 -0
- package/dist/timestamp/index.d.ts +24 -0
- package/dist/timestamp/index.js +274 -0
- package/dist/tokens/index.d.ts +6 -0
- package/dist/tokens/index.js +46 -0
- package/dist/transparency/index.d.ts +188 -0
- package/dist/transparency/index.js +712 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/walls/index.d.ts +31 -0
- package/dist/walls/index.js +119 -0
- package/package.json +1 -1
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
// EU AI Act Article 12 (record-keeping) for one session's record (spec/art12.md): what the record
|
|
2
|
+
// shows against Art. 12(1), 12(2)(a)-(c), 12(3)(a)-(d), and the six-month log retention of Art. 19(1)
|
|
3
|
+
// and 26(6). Evidence mapping from the public text of Regulation (EU) 2024/1689, not legal advice.
|
|
4
|
+
// Mirrors sdks/python/src/zanii_blackbox/art12.py.
|
|
5
|
+
/** Art. 19(1) and 26(6): "at least six months". Counted as 183 days, the longest six months. */
|
|
6
|
+
export const ART12_MIN_RETENTION_DAYS = 183;
|
|
7
|
+
// "ciba" closes a call its people already voted on (spec/approvals.md §7): the votes are counted
|
|
8
|
+
const NOT_PEOPLE = new Set(["timeout", "unattended", "classifier", "ciba"]);
|
|
9
|
+
const NOTES = "Evidence mapping from the public text of Regulation (EU) 2024/1689, not legal advice. Article 12 applies to high-risk AI systems; whether a system is one, and whether it complies, is for its provider, deployer and counsel to judge. One session shows one period of use, not the system's lifetime.";
|
|
10
|
+
const sorted = (s) => [...new Set(s)].sort();
|
|
11
|
+
/** The person an approval decision names: an SSO login (authenticated) or a key holder's word. */
|
|
12
|
+
function personOf(p) {
|
|
13
|
+
if (typeof p !== "object" || p === null)
|
|
14
|
+
return null;
|
|
15
|
+
const o = p;
|
|
16
|
+
const s = (v) => (typeof v === "string" && v !== "" ? v : undefined);
|
|
17
|
+
if (o.authenticated === true && s(o.sub))
|
|
18
|
+
return {
|
|
19
|
+
key: `sub:${s(o.iss) ?? ""}#${o.sub}`,
|
|
20
|
+
label: s(o.email) ?? s(o.name) ?? o.sub,
|
|
21
|
+
authenticated: true,
|
|
22
|
+
};
|
|
23
|
+
if (o.asserted === true && s(o.name))
|
|
24
|
+
return { key: `name:${o.name}`, label: o.name, authenticated: false };
|
|
25
|
+
return null;
|
|
26
|
+
}
|
|
27
|
+
const count = (into, key) => {
|
|
28
|
+
into[key] = (into[key] ?? 0) + 1;
|
|
29
|
+
};
|
|
30
|
+
/**
|
|
31
|
+
* Checks one session's record against Article 12. `verified`: whether its chain verifies (the caller
|
|
32
|
+
* runs the check). `retentionDays`: the deployment's retention, null when records are kept forever.
|
|
33
|
+
* `annexIII1a`: the system is a remote biometric identification system, so 12(3) applies.
|
|
34
|
+
*/
|
|
35
|
+
export function art12Check(record, options) {
|
|
36
|
+
const events = record.lines.map((l) => JSON.parse(l));
|
|
37
|
+
const annex = options.annexIII1a === true;
|
|
38
|
+
let gaps = 0;
|
|
39
|
+
let lostContact = 0;
|
|
40
|
+
let errors = 0;
|
|
41
|
+
let inputs = 0;
|
|
42
|
+
let outcome = null;
|
|
43
|
+
const findings = {};
|
|
44
|
+
const codes = [];
|
|
45
|
+
const controls = {};
|
|
46
|
+
const models = [];
|
|
47
|
+
const providers = [];
|
|
48
|
+
const sources = [];
|
|
49
|
+
// spec/approvals.md §2.1: each human decision, and the person it names (if any)
|
|
50
|
+
const human = [];
|
|
51
|
+
for (const e of events) {
|
|
52
|
+
const m = e.meta;
|
|
53
|
+
if (e.kind === "capture.gap")
|
|
54
|
+
gaps += typeof m.missed === "number" ? m.missed : 1;
|
|
55
|
+
else if (e.kind === "session.lost_contact")
|
|
56
|
+
lostContact++;
|
|
57
|
+
else if (e.kind === "finding") {
|
|
58
|
+
count(findings, typeof m.severity === "string" ? m.severity : "unknown");
|
|
59
|
+
if (typeof m.code === "string")
|
|
60
|
+
codes.push(m.code);
|
|
61
|
+
}
|
|
62
|
+
else if (e.kind === "control" && typeof m.action === "string") {
|
|
63
|
+
count(controls, m.action);
|
|
64
|
+
if (m.action === "approval" && typeof m.by === "string") {
|
|
65
|
+
if (!NOT_PEOPLE.has(m.by))
|
|
66
|
+
human.push(personOf(m.person));
|
|
67
|
+
}
|
|
68
|
+
if (m.action === "approval_vote" && m.counted !== false && m.person !== undefined)
|
|
69
|
+
human.push(personOf(m.person));
|
|
70
|
+
}
|
|
71
|
+
else if (e.kind === "llm.request") {
|
|
72
|
+
inputs++;
|
|
73
|
+
if (typeof m.provider === "string")
|
|
74
|
+
providers.push(m.provider);
|
|
75
|
+
}
|
|
76
|
+
else if (e.kind === "llm.response") {
|
|
77
|
+
if (typeof m.model === "string")
|
|
78
|
+
models.push(m.model);
|
|
79
|
+
if (typeof m.status === "number" && m.status >= 400)
|
|
80
|
+
errors++;
|
|
81
|
+
}
|
|
82
|
+
else if (e.kind === "tool.call") {
|
|
83
|
+
inputs++;
|
|
84
|
+
if (typeof m.server === "string")
|
|
85
|
+
sources.push(typeof m.tool === "string" ? `${m.server}/${m.tool}` : m.server);
|
|
86
|
+
}
|
|
87
|
+
else if (e.kind === "outcome" && typeof m.outcome === "string")
|
|
88
|
+
outcome = m.outcome;
|
|
89
|
+
}
|
|
90
|
+
const open = events.find((e) => e.kind === "session.open");
|
|
91
|
+
const close = events.find((e) => e.kind === "session.close");
|
|
92
|
+
const named = human.filter((p) => p !== null);
|
|
93
|
+
const persons = new Map(named.map((p) => [p.key, p.label]));
|
|
94
|
+
const authenticated = named.filter((p) => p.authenticated).length;
|
|
95
|
+
const retention = options.retentionDays;
|
|
96
|
+
const checks = [
|
|
97
|
+
{
|
|
98
|
+
id: "automatic-recording",
|
|
99
|
+
article: "12(1)",
|
|
100
|
+
requirement: "Events are recorded automatically over the system's lifetime: here, a record that verifies and has no capture gaps.",
|
|
101
|
+
applies: true,
|
|
102
|
+
ok: record.verified && open !== undefined && gaps === 0,
|
|
103
|
+
evidence: {
|
|
104
|
+
events: events.length,
|
|
105
|
+
verified: record.verified,
|
|
106
|
+
capture_gaps: gaps,
|
|
107
|
+
lost_contact: lostContact,
|
|
108
|
+
},
|
|
109
|
+
},
|
|
110
|
+
{
|
|
111
|
+
id: "risk-situations",
|
|
112
|
+
article: "12(2)(a)",
|
|
113
|
+
requirement: "Events relevant to identifying situations that may present a risk (Art. 79(1)) or a substantial modification.",
|
|
114
|
+
applies: true,
|
|
115
|
+
ok: null,
|
|
116
|
+
evidence: { findings, finding_codes: sorted(codes), models: sorted(models) },
|
|
117
|
+
},
|
|
118
|
+
{
|
|
119
|
+
id: "post-market-monitoring",
|
|
120
|
+
article: "12(2)(b)",
|
|
121
|
+
requirement: "Events that facilitate post-market monitoring (Art. 72).",
|
|
122
|
+
applies: true,
|
|
123
|
+
ok: null,
|
|
124
|
+
evidence: { providers: sorted(providers), models: sorted(models), errors, outcome },
|
|
125
|
+
},
|
|
126
|
+
{
|
|
127
|
+
id: "operation-monitoring",
|
|
128
|
+
article: "12(2)(c)",
|
|
129
|
+
requirement: "Events for deployers' monitoring of the system's operation (Art. 26(5)).",
|
|
130
|
+
applies: true,
|
|
131
|
+
ok: null,
|
|
132
|
+
evidence: { controls, unattended: open?.meta.unattended === true },
|
|
133
|
+
},
|
|
134
|
+
{
|
|
135
|
+
id: "period-of-use",
|
|
136
|
+
article: "12(3)(a)",
|
|
137
|
+
requirement: "The start and end date and time of each use.",
|
|
138
|
+
applies: annex,
|
|
139
|
+
ok: open !== undefined && close !== undefined,
|
|
140
|
+
evidence: { start: open?.ts ?? null, end: close?.ts ?? null },
|
|
141
|
+
},
|
|
142
|
+
{
|
|
143
|
+
id: "reference-database",
|
|
144
|
+
article: "12(3)(b)",
|
|
145
|
+
requirement: "The reference database against which input data was checked.",
|
|
146
|
+
applies: annex,
|
|
147
|
+
ok: null,
|
|
148
|
+
evidence: { sources: sorted(sources) },
|
|
149
|
+
},
|
|
150
|
+
{
|
|
151
|
+
id: "input-data",
|
|
152
|
+
article: "12(3)(c)",
|
|
153
|
+
requirement: "The input data for which the search led to a match.",
|
|
154
|
+
applies: annex,
|
|
155
|
+
ok: null,
|
|
156
|
+
evidence: { inputs },
|
|
157
|
+
},
|
|
158
|
+
{
|
|
159
|
+
id: "verifying-persons",
|
|
160
|
+
article: "12(3)(d), 14(5)",
|
|
161
|
+
requirement: "The natural persons who verified the results; for Art. 14(5), at least two, separately.",
|
|
162
|
+
applies: annex,
|
|
163
|
+
// every human decision names its person, and at least two different people decided
|
|
164
|
+
ok: human.length === 0 ? null : named.length === human.length && persons.size >= 2,
|
|
165
|
+
evidence: {
|
|
166
|
+
decisions: human.length,
|
|
167
|
+
persons: sorted(persons.values()),
|
|
168
|
+
authenticated,
|
|
169
|
+
asserted: named.length - authenticated,
|
|
170
|
+
unnamed: human.length - named.length,
|
|
171
|
+
},
|
|
172
|
+
},
|
|
173
|
+
{
|
|
174
|
+
id: "log-retention",
|
|
175
|
+
article: "19(1), 26(6)",
|
|
176
|
+
requirement: "Logs are kept for at least six months, unless other Union or national law says otherwise.",
|
|
177
|
+
applies: true,
|
|
178
|
+
ok: retention === null || retention >= ART12_MIN_RETENTION_DAYS,
|
|
179
|
+
evidence: { retention_days: retention, minimum_days: ART12_MIN_RETENTION_DAYS },
|
|
180
|
+
},
|
|
181
|
+
];
|
|
182
|
+
return {
|
|
183
|
+
session_id: events[0]?.session_id ?? "",
|
|
184
|
+
regulation: "Regulation (EU) 2024/1689",
|
|
185
|
+
annex_iii_1a: annex,
|
|
186
|
+
ok: checks.every((c) => !c.applies || c.ok !== false),
|
|
187
|
+
checks,
|
|
188
|
+
notes: NOTES,
|
|
189
|
+
};
|
|
190
|
+
}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
export type FactValue = number | string | null | Record<string, number>;
|
|
1
|
+
export type FactValue = number | string | null | Record<string, number | string>;
|
|
2
2
|
export type Facts = Record<string, FactValue>;
|
|
3
3
|
export interface Framework {
|
|
4
4
|
title: string;
|
|
@@ -8,6 +8,14 @@ export interface Framework {
|
|
|
8
8
|
requirement: string;
|
|
9
9
|
facts: string[];
|
|
10
10
|
}>;
|
|
11
|
+
/** spec/packs.md §1: the framework in Arabic, by section id. */
|
|
12
|
+
ar?: {
|
|
13
|
+
title: string;
|
|
14
|
+
sections: Record<string, {
|
|
15
|
+
title: string;
|
|
16
|
+
requirement: string;
|
|
17
|
+
}>;
|
|
18
|
+
};
|
|
11
19
|
}
|
|
12
20
|
export interface Report {
|
|
13
21
|
framework: string;
|
|
@@ -24,6 +32,8 @@ export interface Report {
|
|
|
24
32
|
evidence: Facts;
|
|
25
33
|
}>;
|
|
26
34
|
notes: string;
|
|
35
|
+
/** "ar" when the report is in Arabic (spec/compliance.md §3). */
|
|
36
|
+
lang?: "ar";
|
|
27
37
|
}
|
|
28
38
|
export declare function complianceReport(frameworkId: string, framework: Framework, facts: Facts, meta: {
|
|
29
39
|
tenant_id: string;
|
|
@@ -31,7 +41,8 @@ export declare function complianceReport(frameworkId: string, framework: Framewo
|
|
|
31
41
|
to: string;
|
|
32
42
|
generated_at: string;
|
|
33
43
|
notes: string;
|
|
34
|
-
|
|
44
|
+
notes_ar?: string;
|
|
45
|
+
}, lang?: "en" | "ar"): Report;
|
|
35
46
|
export declare function renderReport(r: Report): string;
|
|
36
47
|
/** spec/compliance.md §1: the facts over sessions' records (the server adds verified / anchored). */
|
|
37
48
|
export declare function complianceFacts(sessions: ReadonlyArray<{
|
|
@@ -39,3 +50,26 @@ export declare function complianceFacts(sessions: ReadonlyArray<{
|
|
|
39
50
|
verified: boolean;
|
|
40
51
|
anchored: boolean;
|
|
41
52
|
}>, dataRegion: string | null): Facts;
|
|
53
|
+
/** spec/compliance.md §1a: what a period's records say about hallucinations and their checks. */
|
|
54
|
+
export declare function hallucinationFacts(sessions: ReadonlyArray<{
|
|
55
|
+
lines: readonly string[];
|
|
56
|
+
}>): Facts;
|
|
57
|
+
/** §1a: `detector_accuracy`, the labels' overall figures (spec/findings.md §12) as percentages. */
|
|
58
|
+
export declare function accuracyFact(a: {
|
|
59
|
+
labels: number;
|
|
60
|
+
overall: {
|
|
61
|
+
precision: number | null;
|
|
62
|
+
recall: number | null;
|
|
63
|
+
precision_low: number | null;
|
|
64
|
+
recall_low: number | null;
|
|
65
|
+
};
|
|
66
|
+
}): Record<string, number | string>;
|
|
67
|
+
/** §1a: `hallucination_controls`, which checks ran (spec/findings.md §8-§12). */
|
|
68
|
+
export declare function controlsFact(o: {
|
|
69
|
+
packs: ReadonlyArray<{
|
|
70
|
+
id: string;
|
|
71
|
+
version: string;
|
|
72
|
+
}>;
|
|
73
|
+
grounding: boolean;
|
|
74
|
+
judge: boolean;
|
|
75
|
+
}): Record<string, string>;
|
package/dist/compliance/index.js
CHANGED
|
@@ -1,40 +1,53 @@
|
|
|
1
1
|
// Compliance reports (spec/compliance.md): facts from the record, mapped onto a framework's sections,
|
|
2
2
|
// and rendered as Markdown. Evidence mapping, not legal advice. Mirrors sdks/python/src/zanii_blackbox/compliance.py.
|
|
3
|
-
export function complianceReport(frameworkId, framework, facts, meta) {
|
|
3
|
+
export function complianceReport(frameworkId, framework, facts, meta, lang = "en") {
|
|
4
|
+
// spec/compliance.md §3: in Arabic when the framework has its Arabic; English otherwise
|
|
5
|
+
const ar = lang === "ar" ? framework.ar : undefined;
|
|
6
|
+
const { notes_ar, ...rest } = meta;
|
|
4
7
|
return {
|
|
5
8
|
framework: frameworkId,
|
|
6
|
-
title: framework.title,
|
|
7
|
-
...
|
|
9
|
+
title: ar?.title ?? framework.title,
|
|
10
|
+
...rest,
|
|
11
|
+
notes: ar && notes_ar ? notes_ar : meta.notes,
|
|
8
12
|
facts,
|
|
9
13
|
sections: framework.sections.map((s) => ({
|
|
10
14
|
id: s.id,
|
|
11
|
-
title: s.title,
|
|
12
|
-
requirement: s.requirement,
|
|
15
|
+
title: ar?.sections[s.id]?.title ?? s.title,
|
|
16
|
+
requirement: ar?.sections[s.id]?.requirement ?? s.requirement,
|
|
13
17
|
evidence: Object.fromEntries(s.facts.map((f) => [f, facts[f] ?? null])),
|
|
14
18
|
})),
|
|
19
|
+
...(ar ? { lang: "ar" } : {}),
|
|
15
20
|
};
|
|
16
21
|
}
|
|
17
|
-
const show = (v) => {
|
|
22
|
+
const show = (v, ar = false) => {
|
|
18
23
|
if (v === null)
|
|
19
|
-
return "not set";
|
|
24
|
+
return ar ? "غير محدد" : "not set";
|
|
20
25
|
if (typeof v === "object") {
|
|
21
26
|
const keys = Object.keys(v).sort();
|
|
22
|
-
return keys.length === 0
|
|
27
|
+
return keys.length === 0
|
|
28
|
+
? ar
|
|
29
|
+
? "لا شيء"
|
|
30
|
+
: "none"
|
|
31
|
+
: keys.map((k) => `${k}: ${v[k]}`).join(", ");
|
|
23
32
|
}
|
|
24
33
|
return String(v);
|
|
25
34
|
};
|
|
26
35
|
export function renderReport(r) {
|
|
36
|
+
const ar = r.lang === "ar";
|
|
27
37
|
const out = [
|
|
28
38
|
`# ${r.title}`,
|
|
29
39
|
"",
|
|
30
|
-
|
|
40
|
+
ar
|
|
41
|
+
? `المستأجر \`${r.tenant_id}\` · الجلسات المنشأة من ${r.from} إلى ${r.to} (غير شامل) · أُنشئ في ${r.generated_at}`
|
|
42
|
+
: `Tenant \`${r.tenant_id}\` · sessions created from ${r.from} to ${r.to} (exclusive) · generated ${r.generated_at}`,
|
|
31
43
|
"",
|
|
32
44
|
`> ${r.notes}`,
|
|
33
45
|
];
|
|
34
46
|
for (const s of r.sections) {
|
|
35
|
-
|
|
47
|
+
const head = ar ? "| الدليل | القيمة |" : "| Evidence | Value |";
|
|
48
|
+
out.push("", `## ${s.title}`, "", s.requirement, "", head, "|---|---|");
|
|
36
49
|
for (const [k, v] of Object.entries(s.evidence))
|
|
37
|
-
out.push(`| ${k} | ${show(v)} |`);
|
|
50
|
+
out.push(`| ${k} | ${show(v, ar)} |`);
|
|
38
51
|
}
|
|
39
52
|
return `${out.join("\n")}\n`;
|
|
40
53
|
}
|
|
@@ -94,3 +107,57 @@ export function complianceFacts(sessions, dataRegion) {
|
|
|
94
107
|
data_region: dataRegion,
|
|
95
108
|
};
|
|
96
109
|
}
|
|
110
|
+
// ---------------------------------------------------------------- hallucination facts (§1a, H6)
|
|
111
|
+
/** spec/compliance.md §1a: what a period's records say about hallucinations and their checks. */
|
|
112
|
+
export function hallucinationFacts(sessions) {
|
|
113
|
+
let responses = 0;
|
|
114
|
+
const codes = {};
|
|
115
|
+
const grounding = { checked: 0, outages: 0 };
|
|
116
|
+
const judge = { checked: 0, outages: 0 };
|
|
117
|
+
for (const s of sessions)
|
|
118
|
+
for (const line of s.lines) {
|
|
119
|
+
const e = JSON.parse(line);
|
|
120
|
+
const m = e.meta;
|
|
121
|
+
if (e.kind === "llm.response")
|
|
122
|
+
responses++;
|
|
123
|
+
else if (e.kind === "finding" && m.source === "hallucination" && typeof m.code === "string")
|
|
124
|
+
codes[m.code] = (codes[m.code] ?? 0) + 1;
|
|
125
|
+
else if (e.kind === "control" && (m.action === "grounding" || m.action === "judge")) {
|
|
126
|
+
const c = m.action === "grounding" ? grounding : judge;
|
|
127
|
+
if (m.outage === true)
|
|
128
|
+
c.outages++;
|
|
129
|
+
else
|
|
130
|
+
c.checked++;
|
|
131
|
+
}
|
|
132
|
+
}
|
|
133
|
+
return {
|
|
134
|
+
model_responses: responses,
|
|
135
|
+
hallucination_findings: codes,
|
|
136
|
+
grounding_checks: grounding,
|
|
137
|
+
judge_checks: judge,
|
|
138
|
+
};
|
|
139
|
+
}
|
|
140
|
+
/** A 0-1 figure as a percentage with two decimals, from whole basis points (exact in both SDKs). */
|
|
141
|
+
function percent(x) {
|
|
142
|
+
const bp = Math.floor(x * 10_000 + 0.5);
|
|
143
|
+
return `${Math.floor(bp / 100)}.${String(bp % 100).padStart(2, "0")}%`;
|
|
144
|
+
}
|
|
145
|
+
/** §1a: `detector_accuracy`, the labels' overall figures (spec/findings.md §12) as percentages. */
|
|
146
|
+
export function accuracyFact(a) {
|
|
147
|
+
const out = { labels: a.labels };
|
|
148
|
+
for (const k of ["precision", "precision_low", "recall", "recall_low"]) {
|
|
149
|
+
const v = a.overall[k];
|
|
150
|
+
if (v !== null)
|
|
151
|
+
out[k] = percent(v);
|
|
152
|
+
}
|
|
153
|
+
return out;
|
|
154
|
+
}
|
|
155
|
+
/** §1a: `hallucination_controls`, which checks ran (spec/findings.md §8-§12). */
|
|
156
|
+
export function controlsFact(o) {
|
|
157
|
+
return {
|
|
158
|
+
exact_checks: "on",
|
|
159
|
+
reference_packs: o.packs.length ? o.packs.map((p) => `${p.id}@${p.version}`).join(" ") : "none",
|
|
160
|
+
grounding_service: o.grounding ? "on" : "off",
|
|
161
|
+
judge: o.judge ? "on" : "off",
|
|
162
|
+
};
|
|
163
|
+
}
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Bundle, BundleReport } from "../verify/index.ts";
|
|
2
|
+
export declare function zaniiCompliance(bundle: Bundle, report: BundleReport, opts: {
|
|
3
|
+
generated_at: string;
|
|
4
|
+
controls?: Readonly<Record<string, string>>;
|
|
5
|
+
}): {
|
|
6
|
+
agent_id: string;
|
|
7
|
+
generated_at: string;
|
|
8
|
+
verified: boolean;
|
|
9
|
+
period: {
|
|
10
|
+
first: string | null;
|
|
11
|
+
last: string | null;
|
|
12
|
+
};
|
|
13
|
+
total_actions: number;
|
|
14
|
+
by_namespace: Record<string, number>;
|
|
15
|
+
anchored: {
|
|
16
|
+
count: number;
|
|
17
|
+
total: number;
|
|
18
|
+
complete: boolean;
|
|
19
|
+
};
|
|
20
|
+
controls: {
|
|
21
|
+
namespace: string;
|
|
22
|
+
control: string;
|
|
23
|
+
actions: number;
|
|
24
|
+
}[];
|
|
25
|
+
flags: string[];
|
|
26
|
+
};
|
|
27
|
+
export type ZaniiComplianceReport = ReturnType<typeof zaniiCompliance>;
|
|
28
|
+
/** Zanii's Markdown, as zanii.compliance.render_compliance_markdown gives it. */
|
|
29
|
+
export declare function renderZaniiCompliance(r: ZaniiComplianceReport): string;
|
|
@@ -0,0 +1,84 @@
|
|
|
1
|
+
// Zanii's compliance report, from a session (spec/compliance.md §4): @zanii/compliance's
|
|
2
|
+
// ComplianceReport fields and its Markdown, so a customer's own Zanii tooling reads our evidence.
|
|
3
|
+
// Pure; mirrors sdks/python/src/zanii_blackbox/compliance.py; pinned by
|
|
4
|
+
// spec/vectors/zanii-compliance.json.
|
|
5
|
+
function namespaceOf(e) {
|
|
6
|
+
const m = e.meta ?? {};
|
|
7
|
+
if (e.kind === "llm.request")
|
|
8
|
+
return `model:${String(m.provider ?? "unknown")}`;
|
|
9
|
+
if (e.kind === "tool.call")
|
|
10
|
+
return `tool:${String(m.server ?? m.via ?? "unknown")}`;
|
|
11
|
+
if (e.kind === "sdk.event" && m.type === "tool.call")
|
|
12
|
+
return "tool:sdk";
|
|
13
|
+
return null;
|
|
14
|
+
}
|
|
15
|
+
export function zaniiCompliance(bundle, report, opts) {
|
|
16
|
+
const events = bundle.lines.map((l) => {
|
|
17
|
+
try {
|
|
18
|
+
return JSON.parse(l);
|
|
19
|
+
}
|
|
20
|
+
catch {
|
|
21
|
+
return {};
|
|
22
|
+
}
|
|
23
|
+
});
|
|
24
|
+
const anchoredThrough = bundle.anchors.at(-1)?.count ?? 0;
|
|
25
|
+
const byNamespace = {};
|
|
26
|
+
let total = 0;
|
|
27
|
+
let anchored = 0;
|
|
28
|
+
for (const e of events) {
|
|
29
|
+
const ns = namespaceOf(e);
|
|
30
|
+
if (ns === null)
|
|
31
|
+
continue;
|
|
32
|
+
byNamespace[ns] = (byNamespace[ns] ?? 0) + 1;
|
|
33
|
+
total++;
|
|
34
|
+
if (typeof e.seq === "number" && e.seq < anchoredThrough)
|
|
35
|
+
anchored++;
|
|
36
|
+
}
|
|
37
|
+
const controls = Object.entries(byNamespace).map(([namespace, actions]) => ({
|
|
38
|
+
namespace,
|
|
39
|
+
control: opts.controls?.[namespace] ?? "(unmapped)",
|
|
40
|
+
actions,
|
|
41
|
+
}));
|
|
42
|
+
const flags = [];
|
|
43
|
+
if (!report.ok)
|
|
44
|
+
flags.push("bundle FAILED offline verification — treat all figures below as unproven");
|
|
45
|
+
if (anchored < total)
|
|
46
|
+
flags.push(`${total - anchored} of ${total} receipts are not yet anchored on-chain`);
|
|
47
|
+
if (opts.controls) {
|
|
48
|
+
const unmapped = controls.filter((c) => c.control === "(unmapped)").map((c) => c.namespace);
|
|
49
|
+
if (unmapped.length)
|
|
50
|
+
flags.push(`no control mapped for namespace(s): ${unmapped.join(", ")}`);
|
|
51
|
+
}
|
|
52
|
+
const ts = (e) => (typeof e?.ts === "string" ? e.ts : null);
|
|
53
|
+
return {
|
|
54
|
+
agent_id: bundle.anchored_by ?? "",
|
|
55
|
+
generated_at: opts.generated_at,
|
|
56
|
+
verified: report.ok,
|
|
57
|
+
period: { first: ts(events[0]), last: ts(events.at(-1)) },
|
|
58
|
+
total_actions: total,
|
|
59
|
+
by_namespace: byNamespace,
|
|
60
|
+
anchored: { count: anchored, total, complete: anchored >= total },
|
|
61
|
+
controls,
|
|
62
|
+
flags,
|
|
63
|
+
};
|
|
64
|
+
}
|
|
65
|
+
/** Zanii's Markdown, as zanii.compliance.render_compliance_markdown gives it. */
|
|
66
|
+
export function renderZaniiCompliance(r) {
|
|
67
|
+
// Python's sort is stable, as JavaScript's is: equal counts keep their order
|
|
68
|
+
const rows = [...r.controls]
|
|
69
|
+
.sort((a, b) => b.actions - a.actions)
|
|
70
|
+
.map((c) => `| \`${c.namespace}\` | ${c.control} | ${c.actions} |`)
|
|
71
|
+
.join("\n") || "| — | — | 0 |";
|
|
72
|
+
const flags = r.flags.length ? r.flags.map((f) => `- ⚠️ ${f}`).join("\n") : "- ✅ none";
|
|
73
|
+
return ("# Zanii Compliance Report\n\n" +
|
|
74
|
+
`- **Agent:** \`${r.agent_id}\`\n` +
|
|
75
|
+
`- **Generated:** ${r.generated_at}\n` +
|
|
76
|
+
`- **Verification:** ${r.verified ? "✅ PROVEN offline" : "❌ FAILED"}\n` +
|
|
77
|
+
`- **Period:** ${r.period.first ?? "—"} → ${r.period.last ?? "—"}\n` +
|
|
78
|
+
`- **Actions:** ${r.total_actions} (${r.anchored.count}/${r.anchored.total} anchored on-chain)\n\n` +
|
|
79
|
+
"## Actions by control\n\n" +
|
|
80
|
+
"| Namespace | Control | Actions |\n| --- | --- | --- |\n" +
|
|
81
|
+
`${rows}\n\n` +
|
|
82
|
+
"## Findings\n\n" +
|
|
83
|
+
`${flags}\n`);
|
|
84
|
+
}
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
export type Effect = "forbid" | "require_approval" | "allow";
|
|
2
|
+
export interface ConstitutionRule {
|
|
3
|
+
id: string;
|
|
4
|
+
effect: Effect;
|
|
5
|
+
targets: string[];
|
|
6
|
+
note?: string;
|
|
7
|
+
}
|
|
8
|
+
export interface Constitution {
|
|
9
|
+
v: 1;
|
|
10
|
+
type: "constitution";
|
|
11
|
+
issuer: string;
|
|
12
|
+
name: string;
|
|
13
|
+
rules: ConstitutionRule[];
|
|
14
|
+
default: "allow" | "forbid";
|
|
15
|
+
created_at: string;
|
|
16
|
+
sig: string;
|
|
17
|
+
}
|
|
18
|
+
/** Builds and signs the rulebook; rules are evaluated first match wins, top to bottom. */
|
|
19
|
+
export declare function createConstitution(input: {
|
|
20
|
+
issuer: string;
|
|
21
|
+
name: string;
|
|
22
|
+
rules: ConstitutionRule[];
|
|
23
|
+
defaultEffect?: "allow" | "forbid";
|
|
24
|
+
createdAt: string;
|
|
25
|
+
issuerPrivateKey: Uint8Array;
|
|
26
|
+
}): Constitution;
|
|
27
|
+
/** Structure and the issuer's signature. */
|
|
28
|
+
export declare function verifyConstitution(c: unknown): boolean;
|
|
29
|
+
/** The `manifest_hash`: RFC 8785 JSON, SHA-256. */
|
|
30
|
+
export declare const constitutionHash: (c: Constitution) => string;
|
|
31
|
+
/** First match wins, then the default. */
|
|
32
|
+
export declare function evaluateConstitution(c: Constitution, target: string): {
|
|
33
|
+
effect: Effect;
|
|
34
|
+
rule: ConstitutionRule;
|
|
35
|
+
} | {
|
|
36
|
+
effect: Effect;
|
|
37
|
+
rule: null;
|
|
38
|
+
};
|
|
39
|
+
/** spec/governance.md §3: a session's record against the rulebook. */
|
|
40
|
+
export declare function governanceOf(lines: readonly string[], c: Constitution): {
|
|
41
|
+
manifest_hash: string;
|
|
42
|
+
governed: boolean;
|
|
43
|
+
ok: boolean;
|
|
44
|
+
actions: number;
|
|
45
|
+
violations: {
|
|
46
|
+
seq: number;
|
|
47
|
+
target: string;
|
|
48
|
+
kind: string;
|
|
49
|
+
rule_id?: string;
|
|
50
|
+
}[];
|
|
51
|
+
approvals_required: {
|
|
52
|
+
seq: number;
|
|
53
|
+
target: string;
|
|
54
|
+
rule_id: string;
|
|
55
|
+
approved: boolean;
|
|
56
|
+
}[];
|
|
57
|
+
};
|
|
@@ -0,0 +1,131 @@
|
|
|
1
|
+
// The signed rulebook (spec/governance.md), in Zanii's constitution v1 format, byte for byte with
|
|
2
|
+
// @zanii/constitution / zanii.constitution, and a session's governance against it. Pure; mirrors
|
|
3
|
+
// sdks/python/src/zanii_blackbox/constitution.py; pinned by spec/vectors/governance.json.
|
|
4
|
+
import { canonicalBytes, jcsHash, publicKeyFromDid, scopeCovers } from "@zanii/core";
|
|
5
|
+
import { ed25519Sign, ed25519Verify } from "../transparency/index.js";
|
|
6
|
+
const EFFECTS = new Set(["forbid", "require_approval", "allow"]);
|
|
7
|
+
/** Builds and signs the rulebook; rules are evaluated first match wins, top to bottom. */
|
|
8
|
+
export function createConstitution(input) {
|
|
9
|
+
const { issuer, name, rules, defaultEffect = "allow", createdAt } = input;
|
|
10
|
+
if (!issuer)
|
|
11
|
+
throw new Error("issuer is required");
|
|
12
|
+
if (!name)
|
|
13
|
+
throw new Error("name is required");
|
|
14
|
+
if (!rules.length)
|
|
15
|
+
throw new Error("a constitution with no rules constrains nothing");
|
|
16
|
+
const ids = new Set();
|
|
17
|
+
for (const r of rules) {
|
|
18
|
+
if (!r.id)
|
|
19
|
+
throw new Error("every rule needs an id");
|
|
20
|
+
if (ids.has(r.id))
|
|
21
|
+
throw new Error(`duplicate rule id '${r.id}'`);
|
|
22
|
+
ids.add(r.id);
|
|
23
|
+
if (!EFFECTS.has(r.effect))
|
|
24
|
+
throw new Error(`rule '${r.id}': invalid effect '${r.effect}'`);
|
|
25
|
+
if (!r.targets?.length)
|
|
26
|
+
throw new Error(`rule '${r.id}' has no targets`);
|
|
27
|
+
}
|
|
28
|
+
if (defaultEffect !== "allow" && defaultEffect !== "forbid")
|
|
29
|
+
throw new Error("default_effect must be 'allow' or 'forbid'");
|
|
30
|
+
if (Number.isNaN(Date.parse(createdAt)))
|
|
31
|
+
throw new Error("created_at must be an ISO timestamp");
|
|
32
|
+
const unsigned = {
|
|
33
|
+
v: 1,
|
|
34
|
+
type: "constitution",
|
|
35
|
+
issuer,
|
|
36
|
+
name,
|
|
37
|
+
rules,
|
|
38
|
+
default: defaultEffect,
|
|
39
|
+
created_at: createdAt,
|
|
40
|
+
};
|
|
41
|
+
const sig = Buffer.from(ed25519Sign(input.issuerPrivateKey, canonicalBytes(unsigned))).toString("hex");
|
|
42
|
+
return { ...unsigned, sig: `ed25519:${sig}` };
|
|
43
|
+
}
|
|
44
|
+
/** Structure and the issuer's signature. */
|
|
45
|
+
export function verifyConstitution(c) {
|
|
46
|
+
const p = (c ?? {});
|
|
47
|
+
if (p.v !== 1 || p.type !== "constitution" || !p.issuer || !p.name)
|
|
48
|
+
return false;
|
|
49
|
+
if (!Array.isArray(p.rules) || p.rules.length === 0)
|
|
50
|
+
return false;
|
|
51
|
+
const ids = p.rules.map((r) => (r && typeof r === "object" ? r.id : undefined));
|
|
52
|
+
if (new Set(ids).size !== ids.length)
|
|
53
|
+
return false;
|
|
54
|
+
for (const r of p.rules)
|
|
55
|
+
if (!r || typeof r !== "object" || !EFFECTS.has(r.effect) || !r.targets?.length)
|
|
56
|
+
return false;
|
|
57
|
+
if (p.default !== "allow" && p.default !== "forbid")
|
|
58
|
+
return false;
|
|
59
|
+
const pub = publicKeyFromDid(p.issuer);
|
|
60
|
+
const m = typeof p.sig === "string" ? /^ed25519:([0-9a-f]{128})$/.exec(p.sig) : null;
|
|
61
|
+
if (!pub || !m)
|
|
62
|
+
return false;
|
|
63
|
+
const { sig: _sig, ...unsigned } = p;
|
|
64
|
+
return ed25519Verify(pub, canonicalBytes(unsigned), Buffer.from(m[1], "hex"));
|
|
65
|
+
}
|
|
66
|
+
/** The `manifest_hash`: RFC 8785 JSON, SHA-256. */
|
|
67
|
+
export const constitutionHash = (c) => jcsHash(c);
|
|
68
|
+
/** First match wins, then the default. */
|
|
69
|
+
export function evaluateConstitution(c, target) {
|
|
70
|
+
for (const r of c.rules)
|
|
71
|
+
if (r.targets.some((t) => scopeCovers(t, target)))
|
|
72
|
+
return { effect: r.effect, rule: r };
|
|
73
|
+
return { effect: c.default, rule: null };
|
|
74
|
+
}
|
|
75
|
+
/** spec/governance.md §3: the action's target, or null when the event isn't an action. */
|
|
76
|
+
function targetOf(e) {
|
|
77
|
+
const m = e.meta;
|
|
78
|
+
const policy = m.policy;
|
|
79
|
+
if (policy?.action === "deny")
|
|
80
|
+
return null; // refused: it never ran
|
|
81
|
+
if (e.kind === "tool.call" && typeof m.tool === "string")
|
|
82
|
+
return m.server !== undefined
|
|
83
|
+
? `mcp.${String(m.server)}.${m.tool}`
|
|
84
|
+
: `tool.${String(m.via)}.${m.tool}`;
|
|
85
|
+
if (e.kind === "llm.request")
|
|
86
|
+
return `model.${String(m.provider ?? "unknown")}`;
|
|
87
|
+
if (e.kind === "sdk.event" && m.type === "tool.call")
|
|
88
|
+
return `sdk.${String(m.name ?? "unknown")}`;
|
|
89
|
+
return null;
|
|
90
|
+
}
|
|
91
|
+
/** spec/governance.md §3: a session's record against the rulebook. */
|
|
92
|
+
export function governanceOf(lines, c) {
|
|
93
|
+
const manifest = constitutionHash(c);
|
|
94
|
+
const events = lines.map((l) => JSON.parse(l));
|
|
95
|
+
const open = events.find((e) => e.kind === "session.open");
|
|
96
|
+
const governed = open?.meta.manifest_hash === manifest;
|
|
97
|
+
const approved = new Set(events
|
|
98
|
+
.filter((e) => e.kind === "control" && e.meta.action === "approval" && e.meta.decision === "approve")
|
|
99
|
+
.map((e) => e.meta.call_seq));
|
|
100
|
+
const violations = [];
|
|
101
|
+
const approvals = [];
|
|
102
|
+
let actions = 0;
|
|
103
|
+
for (const e of events) {
|
|
104
|
+
const target = targetOf(e);
|
|
105
|
+
if (target === null)
|
|
106
|
+
continue;
|
|
107
|
+
actions++;
|
|
108
|
+
if (!governed) {
|
|
109
|
+
violations.push({ seq: e.seq, target, kind: "ungoverned" });
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
const ev = evaluateConstitution(c, target);
|
|
113
|
+
if (ev.effect === "forbid")
|
|
114
|
+
violations.push({
|
|
115
|
+
seq: e.seq,
|
|
116
|
+
target,
|
|
117
|
+
kind: "forbidden",
|
|
118
|
+
...(ev.rule ? { rule_id: ev.rule.id } : {}),
|
|
119
|
+
});
|
|
120
|
+
else if (ev.effect === "require_approval" && ev.rule)
|
|
121
|
+
approvals.push({ seq: e.seq, target, rule_id: ev.rule.id, approved: approved.has(e.seq) });
|
|
122
|
+
}
|
|
123
|
+
return {
|
|
124
|
+
manifest_hash: manifest,
|
|
125
|
+
governed,
|
|
126
|
+
ok: violations.length === 0,
|
|
127
|
+
actions,
|
|
128
|
+
violations,
|
|
129
|
+
approvals_required: approvals,
|
|
130
|
+
};
|
|
131
|
+
}
|