@zanii/blackbox 0.21.2 → 0.23.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/actions/index.d.ts +29 -0
- package/dist/actions/index.js +114 -0
- package/dist/admission/index.d.ts +30 -0
- package/dist/admission/index.js +106 -0
- package/dist/analysis/accepted.d.ts +26 -0
- package/dist/analysis/accepted.js +61 -0
- package/dist/analysis/detectors.js +12 -9
- package/dist/analysis/faults.js +71 -0
- package/dist/analysis/grounding.js +5 -9
- package/dist/analysis/guard.d.ts +72 -0
- package/dist/analysis/guard.js +240 -0
- package/dist/analysis/hallucination.js +2 -2
- package/dist/analysis/index.d.ts +21 -1
- package/dist/analysis/index.js +101 -19
- package/dist/analysis/injection.d.ts +2 -1
- package/dist/analysis/injection.js +16 -16
- package/dist/analysis/landing.js +60 -14
- package/dist/analysis/plan-checks.d.ts +55 -0
- package/dist/analysis/plan-checks.js +748 -0
- package/dist/analysis/taxonomy.js +1 -0
- package/dist/analysis/waste.js +2 -3
- package/dist/approvals/index.d.ts +12 -3
- package/dist/approvals/index.js +36 -10
- package/dist/authority/index.js +3 -3
- package/dist/aviation/index.d.ts +422 -0
- package/dist/aviation/index.js +1014 -0
- package/dist/aviation/qrh.d.ts +4 -0
- package/dist/aviation/qrh.js +171 -0
- package/dist/budgets/index.d.ts +63 -0
- package/dist/budgets/index.js +161 -0
- package/dist/cli.js +199 -24
- package/dist/compliance/index.d.ts +2 -0
- package/dist/compliance/index.js +26 -2
- package/dist/cost/batch.d.ts +26 -0
- package/dist/cost/batch.js +173 -0
- package/dist/cost/index.d.ts +21 -0
- package/dist/cost/index.js +115 -30
- package/dist/cv/index.js +1 -1
- package/dist/duty/index.js +2 -1
- package/dist/evals/index.d.ts +233 -0
- package/dist/evals/index.js +488 -0
- package/dist/evidence/index.d.ts +9 -0
- package/dist/evidence/index.js +50 -0
- package/dist/federation/index.d.ts +9 -0
- package/dist/federation/index.js +8 -1
- package/dist/finance/index.js +3 -4
- package/dist/fleet/index.d.ts +25 -0
- package/dist/fleet/index.js +57 -17
- package/dist/fleet/insights.js +4 -1
- package/dist/health/index.js +2 -2
- package/dist/hooks/ai-sdk.d.ts +17 -19
- package/dist/hooks/ai-sdk.js +13 -7
- package/dist/hooks/index.d.ts +1 -0
- package/dist/hooks/index.js +1 -0
- package/dist/hooks/mastra.d.ts +26 -0
- package/dist/hooks/mastra.js +63 -0
- package/dist/index.d.ts +24 -9
- package/dist/index.js +25 -9
- package/dist/mcp-requests/index.d.ts +26 -0
- package/dist/mcp-requests/index.js +154 -0
- package/dist/mcp-server/index.d.ts +12 -0
- package/dist/mcp-server/index.js +84 -2
- package/dist/mcp-wrap/index.js +8 -4
- package/dist/otlp/index.d.ts +155 -0
- package/dist/otlp/index.js +468 -0
- package/dist/otlp/protobuf.d.ts +7 -0
- package/dist/otlp/protobuf.js +314 -0
- package/dist/packs/index.d.ts +2 -0
- package/dist/packs/index.js +5 -1
- package/dist/pdf/arabic.d.ts +45 -0
- package/dist/pdf/arabic.js +373 -0
- package/dist/pdf/index.d.ts +77 -0
- package/dist/pdf/index.js +673 -0
- package/dist/policy/index.d.ts +0 -4
- package/dist/policy/index.js +13 -5
- package/dist/realtime/index.d.ts +33 -0
- package/dist/realtime/index.js +204 -0
- package/dist/reconcile/record.d.ts +14 -0
- package/dist/reconcile/record.js +161 -66
- package/dist/session/index.d.ts +43 -0
- package/dist/session/index.js +83 -0
- package/dist/undo/index.js +3 -3
- package/dist/verify/anchor.d.ts +5 -1
- package/dist/verify/anchor.js +13 -4
- package/dist/verify/chain.d.ts +4 -0
- package/dist/verify/chain.js +4 -3
- package/dist/verify/envelope.d.ts +1 -1
- package/dist/verify/envelope.js +5 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/dist/walls/index.js +2 -2
- package/dist/warehouse/index.d.ts +62 -0
- package/dist/warehouse/index.js +142 -0
- package/package.json +6 -3
package/README.md
CHANGED
|
@@ -156,6 +156,7 @@ The same checks run offline as pure functions: `toolHallucinations`, `groundingF
|
|
|
156
156
|
| OpenAI Agents SDK | `attachBlackbox(runner, s)` |
|
|
157
157
|
| Claude Agent SDK | `blackboxAgentHooks(s)` |
|
|
158
158
|
| Vercel AI SDK | `blackboxMiddleware(s)` |
|
|
159
|
+
| Mastra | `exporters: [blackboxMastraExporter(s)]` in its observability config, and `blackboxMiddleware(s, { toolCalls: false })` on the agent's model for the model-call ids |
|
|
159
160
|
| Provider clients | `wrapAnthropic`, `wrapOpenAI`, `wrapGemini`, `wrapBedrock` link each call to the gateway's record |
|
|
160
161
|
|
|
161
162
|
## Command line
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import { type Bodies, type Call } from "../reconcile/record.ts";
|
|
2
|
+
export interface Reasoning {
|
|
3
|
+
/** `text` (the model's own words), `summary` (the provider's), `encrypted` (there, unreadable) or `none`. */
|
|
4
|
+
kind: "text" | "summary" | "encrypted" | "none";
|
|
5
|
+
text: string | null;
|
|
6
|
+
}
|
|
7
|
+
export interface Action {
|
|
8
|
+
seq: number;
|
|
9
|
+
request_seq: number;
|
|
10
|
+
model: string | null;
|
|
11
|
+
tool: string;
|
|
12
|
+
id: string | null;
|
|
13
|
+
args: unknown;
|
|
14
|
+
reasoning: Reasoning;
|
|
15
|
+
/** What the model wrote before the action, in the same answer. */
|
|
16
|
+
said: string | null;
|
|
17
|
+
}
|
|
18
|
+
/** The tool calls each model answer asked for, in order. Each takes the reasoning and text that came
|
|
19
|
+
* before it in the same answer, since the action before it; one with none of its own (a second call
|
|
20
|
+
* in a row) takes the answer's latest. */
|
|
21
|
+
export declare function actionsFromCalls(calls: readonly Call[]): Action[];
|
|
22
|
+
/** The actions in a verified record, each with the reasoning before it (spec/replay.md §6). */
|
|
23
|
+
export declare function actionsOf(lines: readonly string[], bodies: Bodies): Action[];
|
|
24
|
+
/** The actions for a reader: the first `limit`, each reasoning and `said` cut to `textMax` characters. */
|
|
25
|
+
export declare function actionsPage(actions: readonly Action[], limit?: number, textMax?: number): {
|
|
26
|
+
total: number;
|
|
27
|
+
capped: boolean;
|
|
28
|
+
actions: Action[];
|
|
29
|
+
};
|
|
@@ -0,0 +1,114 @@
|
|
|
1
|
+
// Each action an agent took, with the reasoning the model gave before it (spec/replay.md §6): the
|
|
2
|
+
// thinking (Anthropic `thinking`, Gemini thoughts, a Chat Completions `reasoning_content`), the
|
|
3
|
+
// provider's reasoning summary (OpenAI Responses), or a note that it was there but encrypted. Pure;
|
|
4
|
+
// mirrors sdks/python/src/zanii_blackbox/actions.py; pinned by spec/vectors/actions.json.
|
|
5
|
+
import { callsOf } from "../reconcile/record.js";
|
|
6
|
+
const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
7
|
+
/** Text parts win over a summary, either over encrypted. */
|
|
8
|
+
function reasoningOf(parts) {
|
|
9
|
+
for (const kind of ["text", "summary"]) {
|
|
10
|
+
const texts = parts.filter(([k, t]) => k === kind && t).map(([, t]) => t);
|
|
11
|
+
if (texts.length)
|
|
12
|
+
return { kind, text: texts.join("\n\n") };
|
|
13
|
+
}
|
|
14
|
+
if (parts.some(([k]) => k === "encrypted"))
|
|
15
|
+
return { kind: "encrypted", text: null };
|
|
16
|
+
return { kind: "none", text: null };
|
|
17
|
+
}
|
|
18
|
+
const action = (c, tool, id, args, why, said) => ({
|
|
19
|
+
seq: c.endSeq,
|
|
20
|
+
request_seq: c.requestSeq,
|
|
21
|
+
model: c.model,
|
|
22
|
+
tool,
|
|
23
|
+
id: typeof id === "string" ? id : null,
|
|
24
|
+
args: args ?? {},
|
|
25
|
+
reasoning: reasoningOf(why),
|
|
26
|
+
said: said.filter(Boolean).join("\n\n") || null,
|
|
27
|
+
});
|
|
28
|
+
/** The tool calls each model answer asked for, in order. Each takes the reasoning and text that came
|
|
29
|
+
* before it in the same answer, since the action before it; one with none of its own (a second call
|
|
30
|
+
* in a row) takes the answer's latest. */
|
|
31
|
+
export function actionsFromCalls(calls) {
|
|
32
|
+
const out = [];
|
|
33
|
+
for (const c of [...calls].sort((a, b) => a.endSeq - b.endSeq)) {
|
|
34
|
+
let why = c.reasoning ? [["text", c.reasoning]] : [];
|
|
35
|
+
let said = [];
|
|
36
|
+
let last = [[], []];
|
|
37
|
+
for (const [, b] of [...c.blocks.entries()].sort(([x], [y]) => x - y)) {
|
|
38
|
+
if (b.type === "thinking")
|
|
39
|
+
why.push(typeof b.thinking === "string" && b.thinking ? ["text", b.thinking] : ["encrypted", ""]);
|
|
40
|
+
else if (b.type === "redacted_thinking")
|
|
41
|
+
why.push(["encrypted", ""]);
|
|
42
|
+
else if (b.type === "text" && typeof b.text === "string")
|
|
43
|
+
said.push(b.text);
|
|
44
|
+
else if (b.type === "tool_use" && typeof b.name === "string") {
|
|
45
|
+
if (why.length || said.length)
|
|
46
|
+
last = [why, said];
|
|
47
|
+
out.push(action(c, b.name, b.id, b.input, ...last));
|
|
48
|
+
why = [];
|
|
49
|
+
said = [];
|
|
50
|
+
}
|
|
51
|
+
}
|
|
52
|
+
for (const item of c.items) {
|
|
53
|
+
const t = item.type;
|
|
54
|
+
if (t === "reasoning") {
|
|
55
|
+
const texts = (Array.isArray(item.summary) ? item.summary : [])
|
|
56
|
+
.filter(isObj)
|
|
57
|
+
.map((s) => String(s.text ?? ""));
|
|
58
|
+
const own = (Array.isArray(item.content) ? item.content : [])
|
|
59
|
+
.filter(isObj)
|
|
60
|
+
.map((s) => String(s.text ?? ""));
|
|
61
|
+
if (own.some(Boolean))
|
|
62
|
+
why.push(...own.map((x) => ["text", x]));
|
|
63
|
+
else if (texts.some(Boolean))
|
|
64
|
+
why.push(...texts.map((x) => ["summary", x]));
|
|
65
|
+
else
|
|
66
|
+
why.push(["encrypted", ""]);
|
|
67
|
+
}
|
|
68
|
+
else if (t === "message") {
|
|
69
|
+
for (const s of Array.isArray(item.content) ? item.content : [])
|
|
70
|
+
if (isObj(s) && typeof s.text === "string")
|
|
71
|
+
said.push(s.text);
|
|
72
|
+
}
|
|
73
|
+
else if ((t === "function_call" || t === "custom_tool_call") &&
|
|
74
|
+
typeof item.name === "string") {
|
|
75
|
+
let args = t === "function_call" ? item.arguments : item.input;
|
|
76
|
+
if (typeof args === "string" && t === "function_call") {
|
|
77
|
+
try {
|
|
78
|
+
args = JSON.parse(args);
|
|
79
|
+
}
|
|
80
|
+
catch { }
|
|
81
|
+
}
|
|
82
|
+
if (why.length || said.length)
|
|
83
|
+
last = [why, said];
|
|
84
|
+
out.push(action(c, item.name, item.call_id, args, ...last));
|
|
85
|
+
why = [];
|
|
86
|
+
said = [];
|
|
87
|
+
}
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return out;
|
|
91
|
+
}
|
|
92
|
+
/** The actions in a verified record, each with the reasoning before it (spec/replay.md §6). */
|
|
93
|
+
export function actionsOf(lines, bodies) {
|
|
94
|
+
return actionsFromCalls(callsOf(lines, bodies));
|
|
95
|
+
}
|
|
96
|
+
/** Cut to `most` code points (as Python counts them), then `…`. */
|
|
97
|
+
function cut(text, most) {
|
|
98
|
+
if (text === null)
|
|
99
|
+
return null;
|
|
100
|
+
const points = Array.from(text);
|
|
101
|
+
return points.length <= most ? text : `${points.slice(0, most).join("")}…`;
|
|
102
|
+
}
|
|
103
|
+
/** The actions for a reader: the first `limit`, each reasoning and `said` cut to `textMax` characters. */
|
|
104
|
+
export function actionsPage(actions, limit = 1000, textMax = 8000) {
|
|
105
|
+
return {
|
|
106
|
+
total: actions.length,
|
|
107
|
+
capped: actions.length > limit,
|
|
108
|
+
actions: actions.slice(0, limit).map((a) => ({
|
|
109
|
+
...a,
|
|
110
|
+
reasoning: { ...a.reasoning, text: cut(a.reasoning.text, textMax) },
|
|
111
|
+
said: cut(a.said, textMax),
|
|
112
|
+
})),
|
|
113
|
+
};
|
|
114
|
+
}
|
|
@@ -0,0 +1,30 @@
|
|
|
1
|
+
export declare const REGISTER_TYPE = "blackbox.service-register";
|
|
2
|
+
export declare const ACCREDITATION_TYPE = "blackbox.accreditation";
|
|
3
|
+
export declare const SERVICE_ID: RegExp;
|
|
4
|
+
type Doc = Record<string, unknown>;
|
|
5
|
+
export interface ServiceRow {
|
|
6
|
+
id: string;
|
|
7
|
+
name: string;
|
|
8
|
+
name_ar?: string;
|
|
9
|
+
tier: number;
|
|
10
|
+
}
|
|
11
|
+
/** `sha256:` + hex of the RFC 8785 JSON of the whole signed document. */
|
|
12
|
+
export declare const docHash: (doc: Doc) => string;
|
|
13
|
+
/** §1: null when the register is well formed and signed by one of `signers`, else why not. */
|
|
14
|
+
export declare function checkRegister(doc: unknown, signers: readonly string[]): string | null;
|
|
15
|
+
/** §1: a service's tier and its row; an unknown service (or none) gets `default_tier`. */
|
|
16
|
+
export declare function serviceTier(register: Doc, serviceId: string | null | undefined): {
|
|
17
|
+
tier: number;
|
|
18
|
+
service: ServiceRow | null;
|
|
19
|
+
};
|
|
20
|
+
export type AccreditationRefusal = "malformed" | "unsigned" | "wrong_issuer" | "revoked" | "not_yet" | "expired" | "wrong_vendor" | "agent_not_listed" | "tier_too_high";
|
|
21
|
+
/** §2: null when `cert` lets this vendor's agent open a session for a service of `tier` now, else
|
|
22
|
+
* the reason. */
|
|
23
|
+
export declare function checkAccreditation(cert: unknown, issuers: readonly string[], opts: {
|
|
24
|
+
now: string;
|
|
25
|
+
vendor?: string | null;
|
|
26
|
+
agentId?: string | null;
|
|
27
|
+
tier?: number | null;
|
|
28
|
+
revoked?: readonly string[];
|
|
29
|
+
}): AccreditationRefusal | null;
|
|
30
|
+
export {};
|
|
@@ -0,0 +1,106 @@
|
|
|
1
|
+
// Who and what may open a session (spec/admission.md): the signed service risk register (each
|
|
2
|
+
// service's tier: 1 allowed, 2 a person approves each action, 3 off limits) and a vendor's signed
|
|
3
|
+
// accreditation. Both are signed like certificates (spec/fleet.md §5) and checked offline. Pure;
|
|
4
|
+
// mirrors sdks/python/src/zanii_blackbox/admission.py; pinned by spec/vectors/admission.json.
|
|
5
|
+
import { createHash } from "node:crypto";
|
|
6
|
+
import { verifyCertificate } from "../fleet/index.js";
|
|
7
|
+
import { canonical } from "../reconcile/shared.js";
|
|
8
|
+
export const REGISTER_TYPE = "blackbox.service-register";
|
|
9
|
+
export const ACCREDITATION_TYPE = "blackbox.accreditation";
|
|
10
|
+
export const SERVICE_ID = /^[a-z0-9][a-z0-9._-]{0,63}$/;
|
|
11
|
+
const VENDOR_ID = /^[a-z0-9][a-z0-9-]{0,62}$/;
|
|
12
|
+
const AGENT_ID = /^(?:[A-Za-z0-9._:@/-]{1,128}|\*)$/;
|
|
13
|
+
const MAX_SERVICES = 10_000;
|
|
14
|
+
const UTC = /^\d{4}-\d{2}-\d{2}T\d{2}:\d{2}:\d{2}(\.\d{1,6})?Z$/;
|
|
15
|
+
/** `sha256:` + hex of the RFC 8785 JSON of the whole signed document. */
|
|
16
|
+
export const docHash = (doc) => `sha256:${createHash("sha256").update(canonical(doc)).digest("hex")}`;
|
|
17
|
+
const isTier = (x) => Number.isInteger(x) && x >= 1 && x <= 3;
|
|
18
|
+
const when = (x) => {
|
|
19
|
+
if (typeof x !== "string" || !UTC.test(x))
|
|
20
|
+
return null;
|
|
21
|
+
const t = Date.parse(x);
|
|
22
|
+
return Number.isNaN(t) ? null : t;
|
|
23
|
+
};
|
|
24
|
+
const isObject = (x) => !!x && typeof x === "object" && !Array.isArray(x);
|
|
25
|
+
/** §1: null when the register is well formed and signed by one of `signers`, else why not. */
|
|
26
|
+
export function checkRegister(doc, signers) {
|
|
27
|
+
if (!isObject(doc))
|
|
28
|
+
return "the register must be a JSON object";
|
|
29
|
+
if (doc.v !== 1 || doc.type !== REGISTER_TYPE)
|
|
30
|
+
return `not a ${REGISTER_TYPE} v1`;
|
|
31
|
+
if (typeof doc.name !== "string" || !doc.name)
|
|
32
|
+
return "name is required";
|
|
33
|
+
if (!isTier(doc.default_tier))
|
|
34
|
+
return "default_tier must be 1, 2 or 3";
|
|
35
|
+
const services = doc.services;
|
|
36
|
+
if (!Array.isArray(services) || services.length > MAX_SERVICES)
|
|
37
|
+
return `services must be a list of at most ${MAX_SERVICES}`;
|
|
38
|
+
const seen = new Set();
|
|
39
|
+
for (const [i, s] of services.entries()) {
|
|
40
|
+
if (!isObject(s))
|
|
41
|
+
return `services[${i}] must be an object`;
|
|
42
|
+
if (typeof s.id !== "string" || !SERVICE_ID.test(s.id))
|
|
43
|
+
return `services[${i}].id must be 1-64 of a-z 0-9 . _ - (like visa-renewal)`;
|
|
44
|
+
if (seen.has(s.id))
|
|
45
|
+
return `services[${i}].id ${s.id} is listed twice`;
|
|
46
|
+
seen.add(s.id);
|
|
47
|
+
if (typeof s.name !== "string" || !s.name)
|
|
48
|
+
return `services[${i}].name is required`;
|
|
49
|
+
if (!isTier(s.tier))
|
|
50
|
+
return `services[${i}].tier must be 1, 2 or 3`;
|
|
51
|
+
}
|
|
52
|
+
const v = verifyCertificate(doc);
|
|
53
|
+
if (!v.ok)
|
|
54
|
+
return `the register's signature doesn't verify (${v.reason})`;
|
|
55
|
+
if (!signers.includes(v.by))
|
|
56
|
+
return `the register is signed by ${v.by}, who isn't a trusted signer`;
|
|
57
|
+
return null;
|
|
58
|
+
}
|
|
59
|
+
/** §1: a service's tier and its row; an unknown service (or none) gets `default_tier`. */
|
|
60
|
+
export function serviceTier(register, serviceId) {
|
|
61
|
+
const found = register.services.find((s) => s.id === serviceId);
|
|
62
|
+
return found
|
|
63
|
+
? { tier: found.tier, service: found }
|
|
64
|
+
: { tier: register.default_tier, service: null };
|
|
65
|
+
}
|
|
66
|
+
/** §2: null when `cert` lets this vendor's agent open a session for a service of `tier` now, else
|
|
67
|
+
* the reason. */
|
|
68
|
+
export function checkAccreditation(cert, issuers, opts) {
|
|
69
|
+
if (!isObject(cert))
|
|
70
|
+
return "malformed";
|
|
71
|
+
const agents = cert.agents;
|
|
72
|
+
const issued = when(cert.issued_at);
|
|
73
|
+
const expires = when(cert.expires_at);
|
|
74
|
+
const at = when(opts.now);
|
|
75
|
+
if (cert.v !== 1 ||
|
|
76
|
+
cert.type !== ACCREDITATION_TYPE ||
|
|
77
|
+
typeof cert.vendor !== "string" ||
|
|
78
|
+
!VENDOR_ID.test(cert.vendor) ||
|
|
79
|
+
!Array.isArray(agents) ||
|
|
80
|
+
!agents.length ||
|
|
81
|
+
!agents.every((a) => typeof a === "string" && AGENT_ID.test(a)) ||
|
|
82
|
+
(cert.max_tier !== 1 && cert.max_tier !== 2) ||
|
|
83
|
+
issued === null ||
|
|
84
|
+
expires === null ||
|
|
85
|
+
at === null)
|
|
86
|
+
return "malformed";
|
|
87
|
+
const v = verifyCertificate(cert);
|
|
88
|
+
if (!v.ok)
|
|
89
|
+
return "unsigned";
|
|
90
|
+
if (!issuers.includes(v.by))
|
|
91
|
+
return "wrong_issuer";
|
|
92
|
+
if ((opts.revoked ?? []).includes(docHash(cert)))
|
|
93
|
+
return "revoked";
|
|
94
|
+
if (at < issued)
|
|
95
|
+
return "not_yet";
|
|
96
|
+
if (at >= expires)
|
|
97
|
+
return "expired";
|
|
98
|
+
if (opts.vendor != null && cert.vendor !== opts.vendor)
|
|
99
|
+
return "wrong_vendor";
|
|
100
|
+
const listed = agents;
|
|
101
|
+
if (!listed.includes("*") && (opts.agentId == null || !listed.includes(opts.agentId)))
|
|
102
|
+
return "agent_not_listed";
|
|
103
|
+
if (opts.tier != null && opts.tier > cert.max_tier)
|
|
104
|
+
return "tier_too_high";
|
|
105
|
+
return null;
|
|
106
|
+
}
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
export interface AcceptedEntry {
|
|
2
|
+
code: string;
|
|
3
|
+
ref?: {
|
|
4
|
+
[key: string]: string | number | boolean;
|
|
5
|
+
};
|
|
6
|
+
label?: string;
|
|
7
|
+
agent?: string;
|
|
8
|
+
reason: string;
|
|
9
|
+
by: string;
|
|
10
|
+
expires_at?: string;
|
|
11
|
+
}
|
|
12
|
+
/** Why a list of accepted findings is invalid, or undefined. */
|
|
13
|
+
export declare function checkAcceptedFindings(list: unknown): string | undefined;
|
|
14
|
+
/** The list's identity, recorded with every finding it accepts: sha256 over {v: 1, accepted}. */
|
|
15
|
+
export declare function acceptedHash(list: readonly AcceptedEntry[]): string;
|
|
16
|
+
/** The index of the first entry that accepts this finding, or -1. `now` is an ISO timestamp. */
|
|
17
|
+
export declare function acceptedMatch(list: readonly AcceptedEntry[], finding: {
|
|
18
|
+
code: string;
|
|
19
|
+
ref: {
|
|
20
|
+
[key: string]: unknown;
|
|
21
|
+
};
|
|
22
|
+
}, context: {
|
|
23
|
+
label: string | null;
|
|
24
|
+
agent: string | null;
|
|
25
|
+
now: string;
|
|
26
|
+
}): number;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
// Accepted findings (spec/control.md §4b): a tenant's list of known, accepted findings. A finding
|
|
2
|
+
// that matches is still recorded, marked accepted, and raises no alert and no block. Mirrors
|
|
3
|
+
// analysis/accepted.py; pinned by spec/vectors/accepted-findings.json.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { canonical, isObj } from "../reconcile/shared.js";
|
|
6
|
+
const CODE = /^[A-Z0-9_]{1,64}\*?$/;
|
|
7
|
+
const UTC = /^\d{4}-\d\d-\d\dT\d\d:\d\d:\d\d(?:\.\d{1,3})?Z$/;
|
|
8
|
+
const FIELDS = ["code", "ref", "label", "agent", "reason", "by", "expires_at"];
|
|
9
|
+
const text = (v, max) => typeof v === "string" && v.length > 0 && Array.from(v).length <= max;
|
|
10
|
+
/** Why a list of accepted findings is invalid, or undefined. */
|
|
11
|
+
export function checkAcceptedFindings(list) {
|
|
12
|
+
if (!Array.isArray(list) || list.length > 200)
|
|
13
|
+
return "accepted must be a list of at most 200 entries";
|
|
14
|
+
for (const [i, e] of list.entries()) {
|
|
15
|
+
const at = `accepted[${i}]`;
|
|
16
|
+
if (!isObj(e))
|
|
17
|
+
return `${at} must be an object`;
|
|
18
|
+
const extra = Object.keys(e).filter((k) => !FIELDS.includes(k));
|
|
19
|
+
if (extra.length)
|
|
20
|
+
return `${at} has unknown fields: ${extra.sort().join(", ")}`;
|
|
21
|
+
if (typeof e.code !== "string" || !CODE.test(e.code))
|
|
22
|
+
return `${at}.code must be a finding code, optionally ending in *`;
|
|
23
|
+
if (e.ref !== undefined &&
|
|
24
|
+
!(isObj(e.ref) &&
|
|
25
|
+
Object.keys(e.ref).length <= 20 &&
|
|
26
|
+
Object.values(e.ref).every((v) => typeof v === "boolean" ||
|
|
27
|
+
(typeof v === "number" && Number.isSafeInteger(v)) ||
|
|
28
|
+
(typeof v === "string" && Array.from(v).length <= 256))))
|
|
29
|
+
return `${at}.ref must be an object of at most 20 strings, integers or booleans`;
|
|
30
|
+
for (const k of ["label", "agent"])
|
|
31
|
+
if (e[k] !== undefined && !text(e[k], 128))
|
|
32
|
+
return `${at}.${k} must be 1-128 characters`;
|
|
33
|
+
if (!text(e.reason, 500))
|
|
34
|
+
return `${at}.reason must be 1-500 characters`;
|
|
35
|
+
if (!text(e.by, 128))
|
|
36
|
+
return `${at}.by must be 1-128 characters`;
|
|
37
|
+
if (e.expires_at !== undefined &&
|
|
38
|
+
!(typeof e.expires_at === "string" &&
|
|
39
|
+
UTC.test(e.expires_at) &&
|
|
40
|
+
!Number.isNaN(Date.parse(e.expires_at))))
|
|
41
|
+
return `${at}.expires_at must be a UTC timestamp like 2026-12-31T00:00:00Z`;
|
|
42
|
+
}
|
|
43
|
+
return undefined;
|
|
44
|
+
}
|
|
45
|
+
/** The list's identity, recorded with every finding it accepts: sha256 over {v: 1, accepted}. */
|
|
46
|
+
export function acceptedHash(list) {
|
|
47
|
+
return `sha256:${createHash("sha256")
|
|
48
|
+
.update(canonical({ v: 1, accepted: list }))
|
|
49
|
+
.digest("hex")}`;
|
|
50
|
+
}
|
|
51
|
+
/** The index of the first entry that accepts this finding, or -1. `now` is an ISO timestamp. */
|
|
52
|
+
export function acceptedMatch(list, finding, context) {
|
|
53
|
+
const now = Date.parse(context.now);
|
|
54
|
+
return list.findIndex((e) => (e.code.endsWith("*")
|
|
55
|
+
? finding.code.startsWith(e.code.slice(0, -1))
|
|
56
|
+
: finding.code === e.code) &&
|
|
57
|
+
Object.entries(e.ref ?? {}).every(([k, v]) => Object.hasOwn(finding.ref, k) && finding.ref[k] === v) &&
|
|
58
|
+
(e.label === undefined || e.label === context.label) &&
|
|
59
|
+
(e.agent === undefined || e.agent === context.agent) &&
|
|
60
|
+
(e.expires_at === undefined || now < Date.parse(e.expires_at)));
|
|
61
|
+
}
|
|
@@ -2,7 +2,8 @@
|
|
|
2
2
|
// any agent, instrumented or not. Mirrors sdks/python/src/zanii_blackbox/analysis/detectors.py.
|
|
3
3
|
import { createHash } from "node:crypto";
|
|
4
4
|
import { priceCall, priceFor, promptTokens, tokensOf } from "../cost/index.js";
|
|
5
|
-
import {
|
|
5
|
+
import { realtimeUsage } from "../realtime/index.js";
|
|
6
|
+
import { callsOf, envelopesOf } from "../reconcile/record.js";
|
|
6
7
|
import { canonical, isObj } from "../reconcile/shared.js";
|
|
7
8
|
const VOLATILE = new Set([
|
|
8
9
|
"timestamp",
|
|
@@ -104,8 +105,7 @@ export function toolResultsOf(calls, lines) {
|
|
|
104
105
|
}
|
|
105
106
|
}
|
|
106
107
|
}
|
|
107
|
-
for (const
|
|
108
|
-
const e = JSON.parse(line);
|
|
108
|
+
for (const e of envelopesOf(lines)) {
|
|
109
109
|
if (e.kind === "tool.result" && typeof e.meta.is_error === "boolean")
|
|
110
110
|
out.push({ seq: e.seq, error: e.meta.is_error });
|
|
111
111
|
}
|
|
@@ -303,9 +303,14 @@ export function detectors(lines, options) {
|
|
|
303
303
|
if (options.prices && budget && budget > 0) {
|
|
304
304
|
let micro = 0;
|
|
305
305
|
const crossed = new Set();
|
|
306
|
-
|
|
306
|
+
// a realtime session's usage reports count too (spec/providers.md §7)
|
|
307
|
+
const spend = [
|
|
308
|
+
...priced.map((c) => ({ seq: c.endSeq, call: { ...c, usage: c.usage } })),
|
|
309
|
+
...realtimeUsage(lines).map((u) => ({ seq: u.seq, call: u })),
|
|
310
|
+
].sort((a, b) => a.seq - b.seq);
|
|
311
|
+
for (const { call } of spend) {
|
|
307
312
|
// the cost report's own pricing (spec/cost.md): tiers, provider scope, reported costs
|
|
308
|
-
const priced = priceCall(
|
|
313
|
+
const priced = priceCall(call, options.prices.table);
|
|
309
314
|
if (priced.micro_usd === null)
|
|
310
315
|
continue;
|
|
311
316
|
micro += priced.micro_usd;
|
|
@@ -355,8 +360,7 @@ export function detectors(lines, options) {
|
|
|
355
360
|
out.push(f("DANGLING_TOOL_CALL", "advisory", { tool_use_id: b.id }, "A tool call's result never reached the model; the tool may have run."));
|
|
356
361
|
}
|
|
357
362
|
// LOST_COMMS: the session went silent (squawk 7600).
|
|
358
|
-
for (const
|
|
359
|
-
const e = JSON.parse(line);
|
|
363
|
+
for (const e of envelopesOf(lines)) {
|
|
360
364
|
if (e.kind === "session.lost_contact")
|
|
361
365
|
out.push(f("LOST_COMMS", "warning", { seq: e.seq }, "The agent stopped sending heartbeats: lost contact."));
|
|
362
366
|
}
|
|
@@ -377,8 +381,7 @@ export function detectors(lines, options) {
|
|
|
377
381
|
/** The latest budget and call limit a person granted (spec/approvals.md §4), if any. */
|
|
378
382
|
export function grantedLimits(lines) {
|
|
379
383
|
const out = {};
|
|
380
|
-
for (const
|
|
381
|
-
const e = JSON.parse(line);
|
|
384
|
+
for (const e of envelopesOf(lines)) {
|
|
382
385
|
if (e.kind !== "control" || e.meta.action !== "permission")
|
|
383
386
|
continue;
|
|
384
387
|
if (Number.isSafeInteger(e.meta.budget_micro_usd))
|
package/dist/analysis/faults.js
CHANGED
|
@@ -95,6 +95,17 @@ export const FAULTS = {
|
|
|
95
95
|
ar: "بلوغ حدّ استدعاءات النموذج",
|
|
96
96
|
},
|
|
97
97
|
MODEL_DEPRECATED: { fault: "BBX-3103", en: "Deprecated model", ar: "نموذج متوقّف الدعم" },
|
|
98
|
+
AGENT_SPEND: { fault: "BBX-3104", en: "Agent spend limit", ar: "حدّ إنفاق الوكيل" },
|
|
99
|
+
AGENT_CALL_LIMIT: {
|
|
100
|
+
fault: "BBX-3105",
|
|
101
|
+
en: "Agent model-call limit reached",
|
|
102
|
+
ar: "بلوغ حدّ استدعاءات النموذج للوكيل",
|
|
103
|
+
},
|
|
104
|
+
AGENT_OUTLIER: {
|
|
105
|
+
fault: "BBX-3106",
|
|
106
|
+
en: "Agent far above its peers",
|
|
107
|
+
ar: "وكيل يتجاوز أقرانه بكثير",
|
|
108
|
+
},
|
|
98
109
|
PLAN_DEVIATION: {
|
|
99
110
|
fault: "BBX-4101",
|
|
100
111
|
en: "Deviation from the flight plan",
|
|
@@ -268,6 +279,11 @@ export const FAULTS = {
|
|
|
268
279
|
en: "A tool's definition differs from earlier sessions",
|
|
269
280
|
ar: "يختلف تعريف أداة عمّا رأته الجلسات السابقة",
|
|
270
281
|
},
|
|
282
|
+
ELICITATION_SECRET: {
|
|
283
|
+
fault: "BBX-7408",
|
|
284
|
+
en: "An MCP server asked the user for a credential",
|
|
285
|
+
ar: "طلب خادم MCP من المستخدم بيانات اعتماد",
|
|
286
|
+
},
|
|
271
287
|
REVOKED_MEMORY_READ: {
|
|
272
288
|
fault: "BBX-7101",
|
|
273
289
|
en: "A revoked memory was used",
|
|
@@ -394,6 +410,61 @@ export const FAULTS = {
|
|
|
394
410
|
en: "The agent can reach the internet around the gateway",
|
|
395
411
|
ar: "يستطيع الوكيل الوصول إلى الإنترنت متجاوزاً البوابة",
|
|
396
412
|
},
|
|
413
|
+
NO_CLEARANCE: {
|
|
414
|
+
fault: "BBX-8901",
|
|
415
|
+
en: "Changed a shared resource without a clearance",
|
|
416
|
+
ar: "غيّر موردًا مشتركًا دون تصريح",
|
|
417
|
+
},
|
|
418
|
+
NOTAM_CONFLICT: {
|
|
419
|
+
fault: "BBX-8902",
|
|
420
|
+
en: "Acted against an operator notice",
|
|
421
|
+
ar: "تصرّف بخلاف إشعار من المشغّل",
|
|
422
|
+
},
|
|
423
|
+
ENVELOPE_SHAPED: {
|
|
424
|
+
fault: "BBX-8903",
|
|
425
|
+
en: "A request was shaped to fit the operator's envelope",
|
|
426
|
+
ar: "عُدّل طلب ليبقى ضمن حدود المشغّل",
|
|
427
|
+
},
|
|
428
|
+
EMERGENCY_LOCATOR: {
|
|
429
|
+
fault: "BBX-8904",
|
|
430
|
+
en: "Contact lost with an action open",
|
|
431
|
+
ar: "انقطع الاتصال وإجراء ما زال مفتوحًا",
|
|
432
|
+
},
|
|
433
|
+
SAFETY_REPORT: {
|
|
434
|
+
fault: "BBX-8905",
|
|
435
|
+
en: "Safety report from the agent",
|
|
436
|
+
ar: "بلاغ سلامة من الوكيل",
|
|
437
|
+
},
|
|
438
|
+
MAYDAY: {
|
|
439
|
+
fault: "BBX-8906",
|
|
440
|
+
en: "Mayday: the agent asks for help",
|
|
441
|
+
ar: "نداء استغاثة: الوكيل يطلب المساعدة",
|
|
442
|
+
},
|
|
443
|
+
PAN_PAN: {
|
|
444
|
+
fault: "BBX-8907",
|
|
445
|
+
en: "Pan-pan: the agent is unsure",
|
|
446
|
+
ar: "نداء عاجل: الوكيل غير متأكد",
|
|
447
|
+
},
|
|
448
|
+
UNAPPROVED_MODIFICATION: {
|
|
449
|
+
fault: "BBX-8908",
|
|
450
|
+
en: "The agent runs a configuration with no type certificate",
|
|
451
|
+
ar: "يعمل الوكيل بإعداد بلا شهادة طراز",
|
|
452
|
+
},
|
|
453
|
+
CHECK_OVERDUE: {
|
|
454
|
+
fault: "BBX-8909",
|
|
455
|
+
en: "A scheduled check is overdue",
|
|
456
|
+
ar: "فحص مجدول متأخر",
|
|
457
|
+
},
|
|
458
|
+
RATING_LAPSED: {
|
|
459
|
+
fault: "BBX-8910",
|
|
460
|
+
en: "A type rating has lapsed",
|
|
461
|
+
ar: "انتهت صلاحية تأهيل الطراز",
|
|
462
|
+
},
|
|
463
|
+
APPROVER_FATIGUE: {
|
|
464
|
+
fault: "BBX-8911",
|
|
465
|
+
en: "An approver approves everything in seconds",
|
|
466
|
+
ar: "معتمِد يوافق على كل شيء في ثوانٍ",
|
|
467
|
+
},
|
|
397
468
|
};
|
|
398
469
|
/** The fault for a finding code; unknown codes get a generic entry, never an exception. */
|
|
399
470
|
export function faultOf(code) {
|
|
@@ -3,7 +3,7 @@
|
|
|
3
3
|
// the person's words, the system prompt, the tool results it passed back) and the tool results on
|
|
4
4
|
// the record. Exact, no model, Arabic and English. Mirrors
|
|
5
5
|
// sdks/python/src/zanii_blackbox/analysis/grounding.py; pinned by spec/vectors/grounding.json.
|
|
6
|
-
import { callsOf } from "../reconcile/record.js";
|
|
6
|
+
import { callsOf, envelopesOf } from "../reconcile/record.js";
|
|
7
7
|
import { canonical } from "../reconcile/shared.js";
|
|
8
8
|
import { toolCallsOf } from "./detectors.js";
|
|
9
9
|
import { answerText, canonicalNumber, identifierSpans, normaliseValue, western, } from "./hallucination.js";
|
|
@@ -490,8 +490,7 @@ export function answersOf(lines, bodies) {
|
|
|
490
490
|
// tool results on the record, by seq, to add those before each answer
|
|
491
491
|
const results = [];
|
|
492
492
|
// (each answer's evidence: its request, then the tool results before it, as record lines)
|
|
493
|
-
for (const
|
|
494
|
-
const e = JSON.parse(l);
|
|
493
|
+
for (const e of envelopesOf(lines)) {
|
|
495
494
|
if (e.kind === "tool.result" || (e.kind === "sdk.event" && e.meta.type === "tool.result")) {
|
|
496
495
|
const b = bodies(e.body_hash);
|
|
497
496
|
results.push({ seq: e.seq, text: b ? decoder.decode(b) : "" });
|
|
@@ -584,8 +583,7 @@ export function parseGroundingAnswer(v) {
|
|
|
584
583
|
* recorded as `control {action: "grounding", via: "model"}` with the claims as the body. */
|
|
585
584
|
export function modelGroundingFindings(lines, bodies) {
|
|
586
585
|
const out = [];
|
|
587
|
-
for (const
|
|
588
|
-
const e = JSON.parse(l);
|
|
586
|
+
for (const e of envelopesOf(lines)) {
|
|
589
587
|
if (e.kind !== "control" || e.meta.action !== "grounding" || e.meta.via !== "model")
|
|
590
588
|
continue;
|
|
591
589
|
const seq = e.meta.answer_seq;
|
|
@@ -673,8 +671,7 @@ const STEPS = 100;
|
|
|
673
671
|
export function judgeRequest(sessionId, answer, lines) {
|
|
674
672
|
const steps = [];
|
|
675
673
|
const callTools = new Map();
|
|
676
|
-
for (const
|
|
677
|
-
const e = JSON.parse(l);
|
|
674
|
+
for (const e of envelopesOf(lines)) {
|
|
678
675
|
if (e.seq >= answer.seq)
|
|
679
676
|
break;
|
|
680
677
|
const m = e.meta;
|
|
@@ -735,8 +732,7 @@ export function parseJudgeAnswer(v) {
|
|
|
735
732
|
/** §12.1: JUDGE_FAILED for each rubric step a judge failed, from its recorded verdicts. */
|
|
736
733
|
export function judgeFindings(lines, bodies) {
|
|
737
734
|
const out = [];
|
|
738
|
-
for (const
|
|
739
|
-
const e = JSON.parse(l);
|
|
735
|
+
for (const e of envelopesOf(lines)) {
|
|
740
736
|
if (e.kind !== "control" || e.meta.action !== "judge" || e.meta.via !== "model")
|
|
741
737
|
continue;
|
|
742
738
|
const seq = e.meta.answer_seq;
|
|
@@ -0,0 +1,72 @@
|
|
|
1
|
+
export type GuardVerdict = "allow" | "hold" | "block";
|
|
2
|
+
export type GuardMode = "off" | "monitor" | "enforce";
|
|
3
|
+
export interface GuardConfig {
|
|
4
|
+
canary_tools: string[];
|
|
5
|
+
canary_tokens: string[];
|
|
6
|
+
untrusted_sources: string[];
|
|
7
|
+
sensitive: string[];
|
|
8
|
+
egress: string[];
|
|
9
|
+
high_risk: string[];
|
|
10
|
+
exfil_window_seconds: number;
|
|
11
|
+
}
|
|
12
|
+
export interface GuardFinding {
|
|
13
|
+
rule: string;
|
|
14
|
+
severity: "medium" | "high" | "critical";
|
|
15
|
+
/** The OWASP Top 10 for Agentic Applications entry. */
|
|
16
|
+
asi: string;
|
|
17
|
+
detail: string;
|
|
18
|
+
}
|
|
19
|
+
export interface GuardState {
|
|
20
|
+
taint: {
|
|
21
|
+
source: string;
|
|
22
|
+
markers: number;
|
|
23
|
+
} | null;
|
|
24
|
+
sensitive: {
|
|
25
|
+
tool: string;
|
|
26
|
+
at_ms: number;
|
|
27
|
+
} | null;
|
|
28
|
+
pins: Record<string, string>;
|
|
29
|
+
changed: string[];
|
|
30
|
+
}
|
|
31
|
+
/** A tool call: its names (an MCP tool has two, `mcp__<server>__<tool>` first), its arguments. */
|
|
32
|
+
export interface GuardCall {
|
|
33
|
+
names: string[];
|
|
34
|
+
args: unknown;
|
|
35
|
+
at_ms: number;
|
|
36
|
+
}
|
|
37
|
+
/** The strictest of several modes (off < monitor < enforce); unknown values count as off. */
|
|
38
|
+
export declare function guardMode(...modes: Array<string | null | undefined>): GuardMode;
|
|
39
|
+
/** spec/guard.md §1: a guard file, checked; throws on anything wrong. */
|
|
40
|
+
export declare function loadGuardConfig(doc: unknown): GuardConfig;
|
|
41
|
+
/** A session's state before anything was seen. */
|
|
42
|
+
export declare function guardState(): GuardState;
|
|
43
|
+
/** spec/guard.md §2: the decision on one call, and the state after it. */
|
|
44
|
+
export declare function guardCheck(config: GuardConfig, state: GuardState, call: GuardCall): {
|
|
45
|
+
verdict: GuardVerdict;
|
|
46
|
+
findings: GuardFinding[];
|
|
47
|
+
state: GuardState;
|
|
48
|
+
};
|
|
49
|
+
/** spec/guard.md §3: a tool's result coming back. From an untrusted source, it taints the session. */
|
|
50
|
+
export declare function guardObserve(config: GuardConfig, state: GuardState, result: {
|
|
51
|
+
names: string[];
|
|
52
|
+
content: unknown;
|
|
53
|
+
}): GuardState;
|
|
54
|
+
/** spec/guard.md §4: a tools/list answer's definition hashes. The first one seen for each tool is its
|
|
55
|
+
* pin; a later, different one marks the tool changed for the rest of the session. */
|
|
56
|
+
export declare function guardPin(state: GuardState, defs: Record<string, string>): {
|
|
57
|
+
state: GuardState;
|
|
58
|
+
changed: string[];
|
|
59
|
+
};
|
|
60
|
+
export interface GuardToolResult {
|
|
61
|
+
/** The model's id for the call (`tool_use_id`, `tool_call_id`, `call_id`). */
|
|
62
|
+
id: string;
|
|
63
|
+
name: string;
|
|
64
|
+
args: unknown;
|
|
65
|
+
content: unknown;
|
|
66
|
+
}
|
|
67
|
+
/**
|
|
68
|
+
* spec/guard.md §5: the tool results a model request's newest turn hands back, each with the call
|
|
69
|
+
* that asked for it (from the same request). Anthropic Messages, Chat Completions and Responses.
|
|
70
|
+
* A result whose call isn't in the request (a Responses `previous_response_id`) is left out.
|
|
71
|
+
*/
|
|
72
|
+
export declare function guardToolResults(body: unknown): GuardToolResult[];
|