@zanii/blackbox 0.0.0-stage → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/LICENSE +202 -0
- package/README.md +109 -2
- package/dist/agents/index.d.ts +34 -0
- package/dist/agents/index.js +73 -0
- package/dist/analysis/detectors.d.ts +36 -0
- package/dist/analysis/detectors.js +339 -0
- package/dist/analysis/faults.d.ts +9 -0
- package/dist/analysis/faults.js +250 -0
- package/dist/analysis/index.d.ts +68 -0
- package/dist/analysis/index.js +388 -0
- package/dist/analysis/landing.d.ts +25 -0
- package/dist/analysis/landing.js +225 -0
- package/dist/analysis/memory.d.ts +13 -0
- package/dist/analysis/memory.js +33 -0
- package/dist/analysis/waste.d.ts +29 -0
- package/dist/analysis/waste.js +79 -0
- package/dist/approvals/index.d.ts +11 -0
- package/dist/approvals/index.js +27 -0
- package/dist/approvals/warnings.d.ts +2 -0
- package/dist/approvals/warnings.js +28 -0
- package/dist/attest/index.d.ts +17 -0
- package/dist/attest/index.js +106 -0
- package/dist/authority/index.d.ts +24 -0
- package/dist/authority/index.js +77 -0
- package/dist/billing/index.d.ts +99 -0
- package/dist/billing/index.js +174 -0
- package/dist/cli.d.ts +2 -0
- package/dist/cli.js +1057 -0
- package/dist/client/index.d.ts +146 -0
- package/dist/client/index.js +210 -0
- package/dist/compliance/index.d.ts +41 -0
- package/dist/compliance/index.js +96 -0
- package/dist/cost/index.d.ts +133 -0
- package/dist/cost/index.js +293 -0
- package/dist/data/index.d.ts +191 -0
- package/dist/data/index.js +762 -0
- package/dist/directives/index.d.ts +35 -0
- package/dist/directives/index.js +80 -0
- package/dist/drills/index.d.ts +43 -0
- package/dist/drills/index.js +101 -0
- package/dist/duty/index.d.ts +21 -0
- package/dist/duty/index.js +68 -0
- package/dist/fleet/index.d.ts +141 -0
- package/dist/fleet/index.js +454 -0
- package/dist/hooks/ai-sdk.d.ts +42 -0
- package/dist/hooks/ai-sdk.js +62 -0
- package/dist/hooks/claude-agent-sdk.d.ts +14 -0
- package/dist/hooks/claude-agent-sdk.js +70 -0
- package/dist/hooks/index.d.ts +7 -0
- package/dist/hooks/index.js +10 -0
- package/dist/hooks/langchain-agent.d.ts +69 -0
- package/dist/hooks/langchain-agent.js +163 -0
- package/dist/hooks/langchain.d.ts +41 -0
- package/dist/hooks/langchain.js +216 -0
- package/dist/hooks/langgraph-checkpoint.d.ts +12 -0
- package/dist/hooks/langgraph-checkpoint.js +73 -0
- package/dist/hooks/memory.d.ts +17 -0
- package/dist/hooks/memory.js +64 -0
- package/dist/hooks/openai-agents.d.ts +6 -0
- package/dist/hooks/openai-agents.js +40 -0
- package/dist/hooks/protect.d.ts +7 -0
- package/dist/hooks/protect.js +39 -0
- package/dist/hooks/providers.d.ts +16 -0
- package/dist/hooks/providers.js +149 -0
- package/dist/hooks/shared.d.ts +11 -0
- package/dist/hooks/shared.js +39 -0
- package/dist/index.d.ts +46 -0
- package/dist/index.js +48 -0
- package/dist/investigate/index.d.ts +66 -0
- package/dist/investigate/index.js +119 -0
- package/dist/mcp-server/index.d.ts +85 -0
- package/dist/mcp-server/index.js +216 -0
- package/dist/mcp-wrap/index.d.ts +17 -0
- package/dist/mcp-wrap/index.js +170 -0
- package/dist/money/index.d.ts +114 -0
- package/dist/money/index.js +622 -0
- package/dist/occurrence/index.d.ts +108 -0
- package/dist/occurrence/index.js +168 -0
- package/dist/ocsf/index.d.ts +22 -0
- package/dist/ocsf/index.js +168 -0
- package/dist/otlp/index.d.ts +24 -0
- package/dist/otlp/index.js +143 -0
- package/dist/packs/index.d.ts +48 -0
- package/dist/packs/index.js +343 -0
- package/dist/policy/delta.d.ts +11 -0
- package/dist/policy/delta.js +39 -0
- package/dist/policy/drafts.d.ts +34 -0
- package/dist/policy/drafts.js +129 -0
- package/dist/policy/index.d.ts +47 -0
- package/dist/policy/index.js +154 -0
- package/dist/precog/index.d.ts +96 -0
- package/dist/precog/index.js +167 -0
- package/dist/precog/intervention.d.ts +22 -0
- package/dist/precog/intervention.js +44 -0
- package/dist/precog/normal.d.ts +31 -0
- package/dist/precog/normal.js +89 -0
- package/dist/preflight/index.d.ts +11 -0
- package/dist/preflight/index.js +19 -0
- package/dist/ratings/index.d.ts +21 -0
- package/dist/ratings/index.js +48 -0
- package/dist/reconcile/claude-code.d.ts +19 -0
- package/dist/reconcile/claude-code.js +220 -0
- package/dist/reconcile/codex.d.ts +5 -0
- package/dist/reconcile/codex.js +191 -0
- package/dist/reconcile/index.d.ts +19 -0
- package/dist/reconcile/index.js +50 -0
- package/dist/reconcile/record.d.ts +49 -0
- package/dist/reconcile/record.js +225 -0
- package/dist/reconcile/shared.d.ts +65 -0
- package/dist/reconcile/shared.js +113 -0
- package/dist/replay/index.d.ts +11 -0
- package/dist/replay/index.js +64 -0
- package/dist/replay/repair.d.ts +10 -0
- package/dist/replay/repair.js +62 -0
- package/dist/session/drain.d.ts +13 -0
- package/dist/session/drain.js +35 -0
- package/dist/session/index.d.ts +275 -0
- package/dist/session/index.js +681 -0
- package/dist/undo/index.d.ts +45 -0
- package/dist/undo/index.js +212 -0
- package/dist/verify/anchor.d.ts +54 -0
- package/dist/verify/anchor.js +77 -0
- package/dist/verify/chain.d.ts +27 -0
- package/dist/verify/chain.js +105 -0
- package/dist/verify/envelope.d.ts +28 -0
- package/dist/verify/envelope.js +55 -0
- package/dist/verify/index.d.ts +3 -0
- package/dist/verify/index.js +3 -0
- package/dist/version.d.ts +1 -0
- package/dist/version.js +2 -0
- package/dist/weather/index.d.ts +24 -0
- package/dist/weather/index.js +45 -0
- package/dist/workspace-receipt/index.d.ts +15 -0
- package/dist/workspace-receipt/index.js +121 -0
- package/package.json +56 -3
|
@@ -0,0 +1,454 @@
|
|
|
1
|
+
// Fleet operations (spec/fleet.md): the session summary, the incident report (EN / AR), the logbook.
|
|
2
|
+
// Mirrors sdks/python/src/zanii_blackbox/fleet.py.
|
|
3
|
+
import { createHash, createPrivateKey, createPublicKey, sign as signBytes, verify as verifyBytes, } from "node:crypto";
|
|
4
|
+
import { didFromPublicKey, publicKeyFromDid } from "@zanii/core";
|
|
5
|
+
import { toolCallsOf } from "../analysis/detectors.js";
|
|
6
|
+
import { FAULTS } from "../analysis/faults.js";
|
|
7
|
+
import { eventsOf } from "../analysis/index.js";
|
|
8
|
+
import { landing } from "../analysis/landing.js";
|
|
9
|
+
import { formatAed, sessionCost, toAedFils } from "../cost/index.js";
|
|
10
|
+
import { callsOf } from "../reconcile/record.js";
|
|
11
|
+
import { canonical } from "../reconcile/shared.js";
|
|
12
|
+
const sorted = (m) => Object.fromEntries(Object.keys(m)
|
|
13
|
+
.sort()
|
|
14
|
+
.map((k) => [k, m[k]]));
|
|
15
|
+
export function sessionSummary(lines, bodies, prices) {
|
|
16
|
+
const events = lines.map((l) => JSON.parse(l));
|
|
17
|
+
const first = events[0];
|
|
18
|
+
const last = events[events.length - 1];
|
|
19
|
+
const plan = first.meta.flight_plan;
|
|
20
|
+
const tools = {};
|
|
21
|
+
let toolCalls = 0;
|
|
22
|
+
for (const t of toolCallsOf(callsOf(lines, bodies))) {
|
|
23
|
+
tools[t.tool] = (tools[t.tool] ?? 0) + 1;
|
|
24
|
+
toolCalls++;
|
|
25
|
+
}
|
|
26
|
+
const findings = { advisory: 0, caution: 0, warning: 0 };
|
|
27
|
+
const found = [];
|
|
28
|
+
const codes = {};
|
|
29
|
+
let outcome = null;
|
|
30
|
+
let blockedBy = null;
|
|
31
|
+
let nearMisses = 0;
|
|
32
|
+
for (const e of events) {
|
|
33
|
+
if (e.kind === "tool.call" &&
|
|
34
|
+
e.meta.method === "tools/call" &&
|
|
35
|
+
typeof e.meta.tool === "string") {
|
|
36
|
+
tools[e.meta.tool] = (tools[e.meta.tool] ?? 0) + 1;
|
|
37
|
+
toolCalls++;
|
|
38
|
+
}
|
|
39
|
+
else if (e.kind === "finding") {
|
|
40
|
+
const sev = e.meta.severity;
|
|
41
|
+
if (sev in findings)
|
|
42
|
+
findings[sev]++;
|
|
43
|
+
codes[String(e.meta.code)] = (codes[String(e.meta.code)] ?? 0) + 1;
|
|
44
|
+
found.push({
|
|
45
|
+
seq: e.seq,
|
|
46
|
+
code: String(e.meta.code),
|
|
47
|
+
fault: typeof e.meta.fault === "string" ? e.meta.fault : null,
|
|
48
|
+
severity: String(e.meta.severity),
|
|
49
|
+
});
|
|
50
|
+
}
|
|
51
|
+
else if (e.kind === "outcome" && typeof e.meta.outcome === "string")
|
|
52
|
+
outcome = e.meta.outcome;
|
|
53
|
+
else if (e.kind === "control" && e.meta.action === "block")
|
|
54
|
+
blockedBy = String(e.meta.code);
|
|
55
|
+
else if (e.kind === "control" && e.meta.action === "resume")
|
|
56
|
+
blockedBy = null;
|
|
57
|
+
else if (e.kind === "sdk.event" && e.meta.type === "near_miss")
|
|
58
|
+
nearMisses++;
|
|
59
|
+
}
|
|
60
|
+
const claims = { satisfied: 0, failed: 0, unverified: 0 };
|
|
61
|
+
for (const c of landing(eventsOf(lines, bodies), lines, bodies).claims)
|
|
62
|
+
claims[c.verdict]++;
|
|
63
|
+
return {
|
|
64
|
+
session_id: first.session_id,
|
|
65
|
+
label: typeof first.meta.label === "string" ? first.meta.label : null,
|
|
66
|
+
tenant: typeof first.meta.tenant === "string" ? first.meta.tenant : null,
|
|
67
|
+
objective: typeof plan?.objective === "string" ? plan.objective : null,
|
|
68
|
+
started_at: first.ts,
|
|
69
|
+
last_event_at: last.ts,
|
|
70
|
+
closed: last.kind === "session.close",
|
|
71
|
+
outcome,
|
|
72
|
+
events: events.length,
|
|
73
|
+
model_calls: events.filter((e) => e.kind === "llm.request").length,
|
|
74
|
+
tool_calls: toolCalls,
|
|
75
|
+
tools: sorted(tools),
|
|
76
|
+
spend_micro_usd: sessionCost(lines, prices).total_micro_usd,
|
|
77
|
+
findings,
|
|
78
|
+
finding_codes: sorted(codes),
|
|
79
|
+
latest_findings: found.slice(-5),
|
|
80
|
+
claims,
|
|
81
|
+
blocked_by: blockedBy,
|
|
82
|
+
near_misses: nearMisses,
|
|
83
|
+
};
|
|
84
|
+
}
|
|
85
|
+
const T = {
|
|
86
|
+
en: {
|
|
87
|
+
title: "Incident report",
|
|
88
|
+
session: "Session",
|
|
89
|
+
started: "Started",
|
|
90
|
+
last: "Last event",
|
|
91
|
+
status: "Status",
|
|
92
|
+
objective: "Objective",
|
|
93
|
+
spend: "Spend",
|
|
94
|
+
blocked: "Blocked by",
|
|
95
|
+
open: "Open",
|
|
96
|
+
closed: "Closed",
|
|
97
|
+
none: "none",
|
|
98
|
+
happened: "What happened",
|
|
99
|
+
calls: (m, t) => `${m} model calls, ${t} tool calls.`,
|
|
100
|
+
top: "Tools used",
|
|
101
|
+
findings: "Findings",
|
|
102
|
+
noFindings: "No findings.",
|
|
103
|
+
head: "| Seq | Fault | Level | What |",
|
|
104
|
+
level: { advisory: "Advisory", caution: "Caution", warning: "Warning" },
|
|
105
|
+
},
|
|
106
|
+
ar: {
|
|
107
|
+
title: "تقرير حادثة",
|
|
108
|
+
session: "الجلسة",
|
|
109
|
+
started: "البداية",
|
|
110
|
+
last: "آخر حدث",
|
|
111
|
+
status: "الحالة",
|
|
112
|
+
objective: "الهدف",
|
|
113
|
+
spend: "الإنفاق",
|
|
114
|
+
blocked: "موقوفة بسبب",
|
|
115
|
+
open: "مفتوحة",
|
|
116
|
+
closed: "مغلقة",
|
|
117
|
+
none: "لا شيء",
|
|
118
|
+
happened: "ما الذي حدث",
|
|
119
|
+
calls: (m, t) => `${m} استدعاءات للنموذج، ${t} استدعاءات للأدوات.`,
|
|
120
|
+
top: "الأدوات المستخدمة",
|
|
121
|
+
findings: "الملاحظات",
|
|
122
|
+
noFindings: "لا توجد ملاحظات.",
|
|
123
|
+
head: "| التسلسل | رمز العطل | المستوى | الوصف |",
|
|
124
|
+
level: { advisory: "إرشادي", caution: "تنبيه", warning: "تحذير" },
|
|
125
|
+
},
|
|
126
|
+
};
|
|
127
|
+
/** The incident report (spec/fleet.md §2) in Markdown, in English or Arabic. */
|
|
128
|
+
export function incidentReport(lines, bodies, prices, lang) {
|
|
129
|
+
const t = T[lang];
|
|
130
|
+
const s = sessionSummary(lines, bodies, prices);
|
|
131
|
+
const status = s.blocked_by
|
|
132
|
+
? `${t.blocked} ${s.blocked_by}`
|
|
133
|
+
: s.closed
|
|
134
|
+
? `${t.closed}${s.outcome ? ` (${s.outcome})` : ""}`
|
|
135
|
+
: t.open;
|
|
136
|
+
// Integer arithmetic, so both SDKs print the same digits.
|
|
137
|
+
const usd = `$${Math.floor(s.spend_micro_usd / 1_000_000)}.${String(s.spend_micro_usd % 1_000_000).padStart(6, "0")}`;
|
|
138
|
+
const all = lines
|
|
139
|
+
.map((l) => JSON.parse(l))
|
|
140
|
+
.filter((e) => e.kind === "finding")
|
|
141
|
+
.map((e) => {
|
|
142
|
+
const code = String(e.meta.code);
|
|
143
|
+
const f = FAULTS[code];
|
|
144
|
+
const what = f ? (lang === "ar" ? f.ar : f.en) : code;
|
|
145
|
+
return `| ${e.seq} | ${f?.fault ?? "—"} | ${t.level[String(e.meta.severity)] ?? String(e.meta.severity)} | ${what} (\`${code}\`) |`;
|
|
146
|
+
});
|
|
147
|
+
const tools = Object.entries(s.tools)
|
|
148
|
+
.map(([k, v]) => `${k} × ${v}`)
|
|
149
|
+
.join(", ");
|
|
150
|
+
const out = [
|
|
151
|
+
`# ${t.title}: ${s.label ?? s.session_id}`,
|
|
152
|
+
"",
|
|
153
|
+
"| | |",
|
|
154
|
+
"|---|---|",
|
|
155
|
+
`| ${t.session} | \`${s.session_id}\` |`,
|
|
156
|
+
`| ${t.started} | ${s.started_at} |`,
|
|
157
|
+
`| ${t.last} | ${s.last_event_at} |`,
|
|
158
|
+
`| ${t.status} | ${status} |`,
|
|
159
|
+
`| ${t.objective} | ${s.objective ?? t.none} |`,
|
|
160
|
+
`| ${t.spend} | ${usd} · ${formatAed(toAedFils(s.spend_micro_usd))} |`,
|
|
161
|
+
"",
|
|
162
|
+
`## ${t.happened}`,
|
|
163
|
+
"",
|
|
164
|
+
t.calls(s.model_calls, s.tool_calls),
|
|
165
|
+
...(tools ? ["", `${t.top}: ${tools}`] : []),
|
|
166
|
+
"",
|
|
167
|
+
`## ${t.findings}`,
|
|
168
|
+
"",
|
|
169
|
+
...(all.length === 0 ? [t.noFindings] : [t.head, "|---|---|---|---|", ...all]),
|
|
170
|
+
];
|
|
171
|
+
const body = `${out.join("\n")}\n`;
|
|
172
|
+
return lang === "ar" ? `<div dir="rtl" lang="ar">\n\n${body}\n</div>\n` : body;
|
|
173
|
+
}
|
|
174
|
+
/** Per agent label (spec/fleet.md §2): outcomes, warnings, near misses, exceedances by code (FOQA),
|
|
175
|
+
* spend, cost per success. */
|
|
176
|
+
export function logbook(summaries) {
|
|
177
|
+
const by = new Map();
|
|
178
|
+
for (const s of summaries) {
|
|
179
|
+
const k = s.label ?? "(no label)";
|
|
180
|
+
by.set(k, [...(by.get(k) ?? []), s]);
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
labels: [...by.keys()].sort().map((label) => {
|
|
184
|
+
const ss = by.get(label);
|
|
185
|
+
const outcomes = { success: 0, failure: 0, abandoned: 0, unlabelled: 0 };
|
|
186
|
+
for (const s of ss)
|
|
187
|
+
outcomes[s.outcome in outcomes
|
|
188
|
+
? s.outcome
|
|
189
|
+
: "unlabelled"]++;
|
|
190
|
+
const spend = ss.reduce((n, s) => n + s.spend_micro_usd, 0);
|
|
191
|
+
const labelled = ss
|
|
192
|
+
.filter((s) => s.outcome !== null)
|
|
193
|
+
.reduce((n, s) => n + s.spend_micro_usd, 0);
|
|
194
|
+
const wins = outcomes.success;
|
|
195
|
+
const exceedances = {};
|
|
196
|
+
for (const s of ss)
|
|
197
|
+
for (const [k, v] of Object.entries(s.finding_codes))
|
|
198
|
+
exceedances[k] = (exceedances[k] ?? 0) + v;
|
|
199
|
+
return {
|
|
200
|
+
label,
|
|
201
|
+
sessions: ss.length,
|
|
202
|
+
outcomes,
|
|
203
|
+
findings_warning: ss.reduce((n, s) => n + s.findings.warning, 0),
|
|
204
|
+
near_misses: ss.reduce((n, s) => n + s.near_misses, 0),
|
|
205
|
+
exceedances: sorted(exceedances),
|
|
206
|
+
spend_micro_usd: spend,
|
|
207
|
+
cost_per_success_micro_usd: wins
|
|
208
|
+
? Math.floor((labelled + Math.floor(wins / 2)) / wins)
|
|
209
|
+
: null,
|
|
210
|
+
};
|
|
211
|
+
}),
|
|
212
|
+
};
|
|
213
|
+
}
|
|
214
|
+
/** Tool names that change what they touch (spec/fleet.md §4): a lower-case substring match. */
|
|
215
|
+
const CHANGES = [
|
|
216
|
+
"write",
|
|
217
|
+
"edit",
|
|
218
|
+
"delete",
|
|
219
|
+
"remove",
|
|
220
|
+
"create",
|
|
221
|
+
"update",
|
|
222
|
+
"deploy",
|
|
223
|
+
"move",
|
|
224
|
+
"rename",
|
|
225
|
+
"patch",
|
|
226
|
+
"apply",
|
|
227
|
+
"put",
|
|
228
|
+
"insert",
|
|
229
|
+
"drop",
|
|
230
|
+
"upload",
|
|
231
|
+
"commit",
|
|
232
|
+
"push",
|
|
233
|
+
"merge",
|
|
234
|
+
];
|
|
235
|
+
const RESOURCE_KEYS = ["path", "file_path", "notebook_path", "url", "resource"];
|
|
236
|
+
/** An MCP tools/call body's `params.arguments`. */
|
|
237
|
+
function mcpArgs(body) {
|
|
238
|
+
try {
|
|
239
|
+
const rpc = JSON.parse(new TextDecoder().decode(body ?? new Uint8Array()));
|
|
240
|
+
return rpc?.params?.arguments;
|
|
241
|
+
}
|
|
242
|
+
catch {
|
|
243
|
+
return undefined;
|
|
244
|
+
}
|
|
245
|
+
}
|
|
246
|
+
/** The resource a changing tool call touches, or undefined. */
|
|
247
|
+
function resourceOf(tool, args) {
|
|
248
|
+
const name = tool.toLowerCase();
|
|
249
|
+
if (!CHANGES.some((w) => name.includes(w)))
|
|
250
|
+
return undefined;
|
|
251
|
+
if (typeof args !== "object" || args === null || Array.isArray(args))
|
|
252
|
+
return undefined;
|
|
253
|
+
for (const k of RESOURCE_KEYS) {
|
|
254
|
+
const v = args[k];
|
|
255
|
+
if (typeof v === "string" && v.length > 0)
|
|
256
|
+
return v;
|
|
257
|
+
}
|
|
258
|
+
return undefined;
|
|
259
|
+
}
|
|
260
|
+
/** Changes to the same resource from two or more sessions (A7), by resource; each session's first change. */
|
|
261
|
+
export function collisions(records) {
|
|
262
|
+
const by = new Map();
|
|
263
|
+
const touch = (resource, session_id, tool, seq) => {
|
|
264
|
+
const m = by.get(resource) ?? new Map();
|
|
265
|
+
by.set(resource, m);
|
|
266
|
+
if (!m.has(session_id))
|
|
267
|
+
m.set(session_id, { session_id, tool, seq });
|
|
268
|
+
};
|
|
269
|
+
for (const r of records) {
|
|
270
|
+
if (r.lines.length === 0)
|
|
271
|
+
continue;
|
|
272
|
+
const sid = JSON.parse(r.lines[0]).session_id;
|
|
273
|
+
const touches = [];
|
|
274
|
+
for (const t of toolCallsOf(callsOf(r.lines, r.bodies))) {
|
|
275
|
+
const res = resourceOf(t.tool, t.args);
|
|
276
|
+
if (res !== undefined)
|
|
277
|
+
touches.push({ seq: t.seq, tool: t.tool, resource: res });
|
|
278
|
+
}
|
|
279
|
+
for (const line of r.lines) {
|
|
280
|
+
const e = JSON.parse(line);
|
|
281
|
+
if (e.kind !== "tool.call" ||
|
|
282
|
+
e.meta.method !== "tools/call" ||
|
|
283
|
+
typeof e.meta.tool !== "string")
|
|
284
|
+
continue;
|
|
285
|
+
const res = resourceOf(e.meta.tool, mcpArgs(r.bodies(e.body_hash)));
|
|
286
|
+
if (res !== undefined)
|
|
287
|
+
touches.push({ seq: e.seq, tool: e.meta.tool, resource: res });
|
|
288
|
+
}
|
|
289
|
+
for (const t of touches.sort((a, b) => a.seq - b.seq))
|
|
290
|
+
touch(t.resource, sid, t.tool, t.seq);
|
|
291
|
+
}
|
|
292
|
+
return [...by.keys()]
|
|
293
|
+
.sort()
|
|
294
|
+
.map((resource) => ({
|
|
295
|
+
resource,
|
|
296
|
+
sessions: [...(by.get(resource)?.values() ?? [])].sort((a, b) => a.session_id < b.session_id ? -1 : a.session_id > b.session_id ? 1 : 0),
|
|
297
|
+
}))
|
|
298
|
+
.filter((c) => c.sessions.length > 1);
|
|
299
|
+
}
|
|
300
|
+
export const DEFAULT_AIRWORTHINESS = {
|
|
301
|
+
window: 20,
|
|
302
|
+
min_sessions: 5,
|
|
303
|
+
min_success_ppm: 900_000,
|
|
304
|
+
max_warnings: 0,
|
|
305
|
+
};
|
|
306
|
+
/** Is this agent (label) fit to fly? (spec/fleet.md §5). `summaries` newest first. */
|
|
307
|
+
export function airworthiness(label, summaries, criteria = DEFAULT_AIRWORTHINESS) {
|
|
308
|
+
const judged = summaries
|
|
309
|
+
.filter((s) => s.closed && (s.label ?? "(no label)") === label)
|
|
310
|
+
.slice(0, criteria.window);
|
|
311
|
+
const successes = judged.filter((s) => s.outcome === "success").length;
|
|
312
|
+
const warnings = judged.reduce((n, s) => n + s.findings.warning, 0);
|
|
313
|
+
const rate = judged.length ? Math.floor((successes * 1_000_000) / judged.length) : 0;
|
|
314
|
+
const reasons = [];
|
|
315
|
+
if (judged.length < criteria.min_sessions)
|
|
316
|
+
reasons.push(`${judged.length} closed sessions, fewer than ${criteria.min_sessions}`);
|
|
317
|
+
else {
|
|
318
|
+
if (rate < criteria.min_success_ppm)
|
|
319
|
+
reasons.push(`success rate ${rate} ppm, below ${criteria.min_success_ppm}`);
|
|
320
|
+
if (warnings > criteria.max_warnings)
|
|
321
|
+
reasons.push(`${warnings} warnings, more than ${criteria.max_warnings}`);
|
|
322
|
+
}
|
|
323
|
+
return {
|
|
324
|
+
v: 1,
|
|
325
|
+
label,
|
|
326
|
+
criteria,
|
|
327
|
+
sessions: judged.map((s) => s.session_id),
|
|
328
|
+
successes,
|
|
329
|
+
success_ppm: rate,
|
|
330
|
+
warnings,
|
|
331
|
+
near_misses: judged.reduce((n, s) => n + s.near_misses, 0),
|
|
332
|
+
verdict: judged.length < criteria.min_sessions
|
|
333
|
+
? "insufficient_data"
|
|
334
|
+
: reasons.length
|
|
335
|
+
? "not_airworthy"
|
|
336
|
+
: "airworthy",
|
|
337
|
+
reasons,
|
|
338
|
+
};
|
|
339
|
+
}
|
|
340
|
+
const NOT_FAILURES = new Set(["NEAR_MISS", "PRECOG_RISK"]);
|
|
341
|
+
/** spec/signatures.md §1: the failure pattern and its id, or null. */
|
|
342
|
+
export function signatureOf(summary) {
|
|
343
|
+
const pattern = Object.keys(summary.finding_codes)
|
|
344
|
+
.filter((c) => !NOT_FAILURES.has(c))
|
|
345
|
+
.sort();
|
|
346
|
+
if (pattern.length === 0)
|
|
347
|
+
return null;
|
|
348
|
+
const hex = createHash("sha256").update(pattern.join("+"), "utf8").digest("hex");
|
|
349
|
+
return { signature: `sig_${hex.slice(0, 16)}`, pattern };
|
|
350
|
+
}
|
|
351
|
+
/** spec/signatures.md §2: groups seen at k or more distinct tenants; counts only. */
|
|
352
|
+
export function signatures(items, k = 3) {
|
|
353
|
+
const by = new Map();
|
|
354
|
+
for (const { tenant, summary } of items) {
|
|
355
|
+
const sig = signatureOf(summary);
|
|
356
|
+
if (!sig)
|
|
357
|
+
continue;
|
|
358
|
+
const g = by.get(sig.signature) ?? {
|
|
359
|
+
pattern: sig.pattern,
|
|
360
|
+
tenants: new Set(),
|
|
361
|
+
sessions: 0,
|
|
362
|
+
failures: 0,
|
|
363
|
+
};
|
|
364
|
+
g.tenants.add(tenant);
|
|
365
|
+
g.sessions++;
|
|
366
|
+
if (summary.outcome === "failure" || summary.outcome === "abandoned")
|
|
367
|
+
g.failures++;
|
|
368
|
+
by.set(sig.signature, g);
|
|
369
|
+
}
|
|
370
|
+
return [...by.entries()]
|
|
371
|
+
.filter(([, g]) => g.tenants.size >= k)
|
|
372
|
+
.map(([signature, g]) => ({
|
|
373
|
+
signature,
|
|
374
|
+
pattern: g.pattern,
|
|
375
|
+
tenants: g.tenants.size,
|
|
376
|
+
sessions: g.sessions,
|
|
377
|
+
failures: g.failures,
|
|
378
|
+
failure_ppm: Math.floor((g.failures * 1_000_000) / g.sessions),
|
|
379
|
+
}))
|
|
380
|
+
.sort((a, b) => b.tenants - a.tenants ||
|
|
381
|
+
b.sessions - a.sessions ||
|
|
382
|
+
(a.signature < b.signature ? -1 : a.signature > b.signature ? 1 : 0));
|
|
383
|
+
}
|
|
384
|
+
// ---------------------------------------------------------------- signed certificates (spec/fleet.md §5, L6.1.6)
|
|
385
|
+
const PKCS8_ED25519 = Buffer.from("302e020100300506032b657004220420", "hex");
|
|
386
|
+
const SPKI_ED25519 = Buffer.from("302a300506032b6570032100", "hex");
|
|
387
|
+
const unsigned = (cert) => Buffer.from(canonical(Object.fromEntries(Object.entries(cert).filter(([k]) => k !== "signed"))));
|
|
388
|
+
/** The did:key of an Ed25519 identity, from its 32-byte seed. */
|
|
389
|
+
export function didFromSeed(seed) {
|
|
390
|
+
const key = createPrivateKey({
|
|
391
|
+
key: Buffer.concat([PKCS8_ED25519, Buffer.from(seed)]),
|
|
392
|
+
format: "der",
|
|
393
|
+
type: "pkcs8",
|
|
394
|
+
});
|
|
395
|
+
const x = createPublicKey(key).export({ format: "jwk" }).x;
|
|
396
|
+
return didFromPublicKey(new Uint8Array(Buffer.from(x, "base64url")));
|
|
397
|
+
}
|
|
398
|
+
/** Signs a certificate with an Ed25519 identity (the 32-byte seed): adds `signed: {by, sig}`. */
|
|
399
|
+
export function signCertificate(cert, did, seed) {
|
|
400
|
+
const key = createPrivateKey({
|
|
401
|
+
key: Buffer.concat([PKCS8_ED25519, Buffer.from(seed)]),
|
|
402
|
+
format: "der",
|
|
403
|
+
type: "pkcs8",
|
|
404
|
+
});
|
|
405
|
+
const sig = signBytes(null, unsigned(cert), key).toString("hex");
|
|
406
|
+
return { ...cert, signed: { by: did, sig: `ed25519:${sig}` } };
|
|
407
|
+
}
|
|
408
|
+
/** Checks a signed certificate offline, from the signer's did:key alone. */
|
|
409
|
+
export function verifyCertificate(cert, expectedDid) {
|
|
410
|
+
const s = cert.signed;
|
|
411
|
+
if (!s || typeof s.by !== "string" || typeof s.sig !== "string")
|
|
412
|
+
return { ok: false, by: null, reason: "unsigned" };
|
|
413
|
+
if (expectedDid && s.by !== expectedDid)
|
|
414
|
+
return { ok: false, by: s.by, reason: "wrong_signer" };
|
|
415
|
+
const pub = publicKeyFromDid(s.by);
|
|
416
|
+
const m = /^ed25519:([0-9a-f]{128})$/.exec(s.sig);
|
|
417
|
+
if (!pub || !m)
|
|
418
|
+
return { ok: false, by: s.by, reason: "malformed" };
|
|
419
|
+
const key = createPublicKey({
|
|
420
|
+
key: Buffer.concat([SPKI_ED25519, Buffer.from(pub)]),
|
|
421
|
+
format: "der",
|
|
422
|
+
type: "spki",
|
|
423
|
+
});
|
|
424
|
+
const ok = verifyBytes(null, unsigned(cert), key, Buffer.from(m[1], "hex"));
|
|
425
|
+
return ok ? { ok: true, by: s.by } : { ok: false, by: s.by, reason: "bad_signature" };
|
|
426
|
+
}
|
|
427
|
+
/** The record's findings grouped by code: how many, and when first and last, in first-seen order. */
|
|
428
|
+
export function aggregateFindings(lines) {
|
|
429
|
+
const by = new Map();
|
|
430
|
+
for (const line of lines) {
|
|
431
|
+
const e = JSON.parse(line);
|
|
432
|
+
if (e.kind !== "finding")
|
|
433
|
+
continue;
|
|
434
|
+
const code = String(e.meta.code);
|
|
435
|
+
const g = by.get(code);
|
|
436
|
+
if (g) {
|
|
437
|
+
g.count++;
|
|
438
|
+
g.last_seq = e.seq;
|
|
439
|
+
g.last_at = e.ts;
|
|
440
|
+
}
|
|
441
|
+
else
|
|
442
|
+
by.set(code, {
|
|
443
|
+
code,
|
|
444
|
+
fault: typeof e.meta.fault === "string" ? e.meta.fault : null,
|
|
445
|
+
severity: String(e.meta.severity),
|
|
446
|
+
count: 1,
|
|
447
|
+
first_seq: e.seq,
|
|
448
|
+
last_seq: e.seq,
|
|
449
|
+
first_at: e.ts,
|
|
450
|
+
last_at: e.ts,
|
|
451
|
+
});
|
|
452
|
+
}
|
|
453
|
+
return [...by.values()];
|
|
454
|
+
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import { type EventSink } from "./shared.ts";
|
|
2
|
+
interface ModelInfo {
|
|
3
|
+
provider?: string;
|
|
4
|
+
modelId?: string;
|
|
5
|
+
}
|
|
6
|
+
type Part = {
|
|
7
|
+
type?: string;
|
|
8
|
+
id?: string;
|
|
9
|
+
modelId?: string;
|
|
10
|
+
toolCallId?: string;
|
|
11
|
+
toolName?: string;
|
|
12
|
+
input?: unknown;
|
|
13
|
+
};
|
|
14
|
+
/** For `wrapLanguageModel({ model, middleware })`: every generate and stream is observed; what the
|
|
15
|
+
* model returns reaches the app unchanged. */
|
|
16
|
+
export declare function blackboxMiddleware(sink: EventSink): {
|
|
17
|
+
wrapGenerate({ doGenerate, model, }: {
|
|
18
|
+
doGenerate: () => Promise<{
|
|
19
|
+
content?: Part[];
|
|
20
|
+
response?: {
|
|
21
|
+
id?: string;
|
|
22
|
+
modelId?: string;
|
|
23
|
+
};
|
|
24
|
+
}>;
|
|
25
|
+
model?: ModelInfo;
|
|
26
|
+
}): Promise<{
|
|
27
|
+
content?: Part[];
|
|
28
|
+
response?: {
|
|
29
|
+
id?: string;
|
|
30
|
+
modelId?: string;
|
|
31
|
+
};
|
|
32
|
+
}>;
|
|
33
|
+
wrapStream({ doStream, model, }: {
|
|
34
|
+
doStream: () => Promise<{
|
|
35
|
+
stream: ReadableStream<Part>;
|
|
36
|
+
}>;
|
|
37
|
+
model?: ModelInfo;
|
|
38
|
+
}): Promise<{
|
|
39
|
+
stream: ReadableStream<Part>;
|
|
40
|
+
}>;
|
|
41
|
+
};
|
|
42
|
+
export {};
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
// Vercel AI SDK (audit K19, spec/sdk.md §5): a language-model middleware that reports each model
|
|
2
|
+
// call's ids (for the dual witness) and the tool calls the model asks for. Duck-typed on the
|
|
3
|
+
// middleware shape of AI SDK 5: no dependency on `ai`.
|
|
4
|
+
//
|
|
5
|
+
// import { wrapLanguageModel } from "ai";
|
|
6
|
+
// const model = wrapLanguageModel({ model: anthropic("claude-sonnet-4-5"), middleware: blackboxMiddleware(session) });
|
|
7
|
+
import { llmIds, neverThrow, providerOf } from "./shared.js";
|
|
8
|
+
/** The model call's ids: the provider's response id, under the key the gateway records it by. */
|
|
9
|
+
function reportCall(sink, model, id, modelId) {
|
|
10
|
+
const provider = providerOf(id) ?? model?.provider?.split(".")[0];
|
|
11
|
+
if (!provider || !id)
|
|
12
|
+
return;
|
|
13
|
+
sink.event("llm.call", provider, {
|
|
14
|
+
provider,
|
|
15
|
+
...llmIds(id, undefined, modelId ?? model?.modelId),
|
|
16
|
+
});
|
|
17
|
+
}
|
|
18
|
+
function reportTool(sink, p) {
|
|
19
|
+
if (p.type !== "tool-call" || !p.toolName)
|
|
20
|
+
return;
|
|
21
|
+
let args = p.input;
|
|
22
|
+
try {
|
|
23
|
+
args = typeof args === "string" ? JSON.parse(args) : args;
|
|
24
|
+
}
|
|
25
|
+
catch { }
|
|
26
|
+
sink.event("tool.call", p.toolName, {
|
|
27
|
+
...(p.toolCallId ? { call_id: p.toolCallId } : {}),
|
|
28
|
+
...(args === undefined ? {} : { args }),
|
|
29
|
+
});
|
|
30
|
+
}
|
|
31
|
+
/** For `wrapLanguageModel({ model, middleware })`: every generate and stream is observed; what the
|
|
32
|
+
* model returns reaches the app unchanged. */
|
|
33
|
+
export function blackboxMiddleware(sink) {
|
|
34
|
+
return {
|
|
35
|
+
async wrapGenerate({ doGenerate, model, }) {
|
|
36
|
+
const result = await doGenerate();
|
|
37
|
+
neverThrow(() => {
|
|
38
|
+
reportCall(sink, model, result.response?.id, result.response?.modelId);
|
|
39
|
+
for (const p of result.content ?? [])
|
|
40
|
+
reportTool(sink, p);
|
|
41
|
+
});
|
|
42
|
+
return result;
|
|
43
|
+
},
|
|
44
|
+
async wrapStream({ doStream, model, }) {
|
|
45
|
+
const { stream, ...rest } = await doStream();
|
|
46
|
+
let reported = false;
|
|
47
|
+
const observed = stream.pipeThrough(new TransformStream({
|
|
48
|
+
transform(chunk, controller) {
|
|
49
|
+
neverThrow(() => {
|
|
50
|
+
if (!reported && chunk.type === "response-metadata" && chunk.id) {
|
|
51
|
+
reported = true;
|
|
52
|
+
reportCall(sink, model, chunk.id, chunk.modelId);
|
|
53
|
+
}
|
|
54
|
+
reportTool(sink, chunk);
|
|
55
|
+
});
|
|
56
|
+
controller.enqueue(chunk);
|
|
57
|
+
},
|
|
58
|
+
}));
|
|
59
|
+
return { stream: observed, ...rest };
|
|
60
|
+
},
|
|
61
|
+
};
|
|
62
|
+
}
|
|
@@ -0,0 +1,14 @@
|
|
|
1
|
+
import { type EventSink } from "./shared.ts";
|
|
2
|
+
type Callback = (input: unknown, toolUseId: string | undefined, options: unknown) => Promise<object>;
|
|
3
|
+
export interface BlackboxAgentHooks {
|
|
4
|
+
/** Pass as `options.hooks`. */
|
|
5
|
+
hooks: Record<"PreToolUse" | "PostToolUse" | "PostToolUseFailure", Array<{
|
|
6
|
+
hooks: Callback[];
|
|
7
|
+
}>>;
|
|
8
|
+
/** Call with every message `query()` yields; reports each model call's id once. */
|
|
9
|
+
observe(message: unknown): void;
|
|
10
|
+
}
|
|
11
|
+
export declare function blackboxAgentHooks(session: EventSink): BlackboxAgentHooks;
|
|
12
|
+
/** Audit S6: the Python SDK's name, so docs and examples can be written once. Called, not `new`ed. */
|
|
13
|
+
export declare const BlackboxAgentHooks: typeof blackboxAgentHooks;
|
|
14
|
+
export {};
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
// Claude Agent SDK hooks (spec/sdk.md §5). Mirrors hooks/claude_agent_sdk.py.
|
|
2
|
+
//
|
|
3
|
+
// const bbx = blackboxAgentHooks(session);
|
|
4
|
+
// for await (const message of query({ prompt, options: { hooks: bbx.hooks, env } })) {
|
|
5
|
+
// bbx.observe(message);
|
|
6
|
+
// …
|
|
7
|
+
// }
|
|
8
|
+
//
|
|
9
|
+
// Tool calls come from the SDK's PreToolUse / PostToolUse / PostToolUseFailure hooks; model calls
|
|
10
|
+
// from the assistant messages `query()` yields (the Agent SDK has no model-call hook). Structural
|
|
11
|
+
// types only, so the Agent SDK stays an optional dependency.
|
|
12
|
+
import { neverThrow } from "./shared.js";
|
|
13
|
+
const MAX_ERROR = 500;
|
|
14
|
+
export function blackboxAgentHooks(session) {
|
|
15
|
+
const seen = new Set();
|
|
16
|
+
const hook = (fn) => async (input) => {
|
|
17
|
+
neverThrow(() => fn((input ?? {})));
|
|
18
|
+
return {}; // no decision: the hook only records
|
|
19
|
+
};
|
|
20
|
+
const common = (i) => ({
|
|
21
|
+
...(i.tool_use_id ? { tool_use_id: i.tool_use_id } : {}),
|
|
22
|
+
...(i.agent_id ? { agent_id: i.agent_id } : {}),
|
|
23
|
+
});
|
|
24
|
+
const name = (i) => String(i.tool_name ?? "tool").slice(0, 256);
|
|
25
|
+
return {
|
|
26
|
+
hooks: {
|
|
27
|
+
PreToolUse: [
|
|
28
|
+
{
|
|
29
|
+
hooks: [
|
|
30
|
+
hook((i) => session.event("tool.call", name(i), {
|
|
31
|
+
...common(i),
|
|
32
|
+
...(i.tool_input === undefined ? {} : { args: i.tool_input }),
|
|
33
|
+
})),
|
|
34
|
+
],
|
|
35
|
+
},
|
|
36
|
+
],
|
|
37
|
+
PostToolUse: [
|
|
38
|
+
{ hooks: [hook((i) => session.event("tool.result", name(i), { ok: true, ...common(i) }))] },
|
|
39
|
+
],
|
|
40
|
+
PostToolUseFailure: [
|
|
41
|
+
{
|
|
42
|
+
hooks: [
|
|
43
|
+
hook((i) => session.event("tool.result", name(i), {
|
|
44
|
+
ok: false,
|
|
45
|
+
...common(i),
|
|
46
|
+
error: String(i.error ?? "").slice(0, MAX_ERROR),
|
|
47
|
+
})),
|
|
48
|
+
],
|
|
49
|
+
},
|
|
50
|
+
],
|
|
51
|
+
},
|
|
52
|
+
observe(message) {
|
|
53
|
+
neverThrow(() => {
|
|
54
|
+
const m = message;
|
|
55
|
+
const id = m?.type === "assistant" ? m.message?.id : undefined;
|
|
56
|
+
// One API response can arrive as several assistant messages (one per content block).
|
|
57
|
+
if (typeof id !== "string" || seen.has(id))
|
|
58
|
+
return;
|
|
59
|
+
seen.add(id);
|
|
60
|
+
session.event("llm.call", "anthropic", {
|
|
61
|
+
provider: "anthropic",
|
|
62
|
+
message_id: id,
|
|
63
|
+
...(typeof m.message?.model === "string" ? { model: m.message.model } : {}),
|
|
64
|
+
});
|
|
65
|
+
});
|
|
66
|
+
},
|
|
67
|
+
};
|
|
68
|
+
}
|
|
69
|
+
/** Audit S6: the Python SDK's name, so docs and examples can be written once. Called, not `new`ed. */
|
|
70
|
+
export const BlackboxAgentHooks = blackboxAgentHooks;
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
export { blackboxMiddleware } from "./ai-sdk.ts";
|
|
2
|
+
export { BlackboxAgentHooks, blackboxAgentHooks } from "./claude-agent-sdk.ts";
|
|
3
|
+
export { BlackboxCallbackHandler, type BlackboxCallbacks, blackboxCallbacks } from "./langchain.ts";
|
|
4
|
+
export { type MemorySink, type TrackMemoryOptions, trackMemory } from "./memory.ts";
|
|
5
|
+
export { attachBlackbox } from "./openai-agents.ts";
|
|
6
|
+
export { llmIdsOf, wrapAnthropic, wrapBedrock, wrapGemini, wrapOpenAI } from "./providers.ts";
|
|
7
|
+
export { type EventSink, llmIds } from "./shared.ts";
|
|
@@ -0,0 +1,10 @@
|
|
|
1
|
+
// Framework hooks (spec/sdk.md §5): framework callbacks → SDK session events. Agent-asserted structure
|
|
2
|
+
// only; every callback is wrapped so a hook can never break the framework. Mirrors
|
|
3
|
+
// sdks/python/src/zanii_blackbox/hooks/.
|
|
4
|
+
export { blackboxMiddleware } from "./ai-sdk.js";
|
|
5
|
+
export { BlackboxAgentHooks, blackboxAgentHooks } from "./claude-agent-sdk.js";
|
|
6
|
+
export { BlackboxCallbackHandler, blackboxCallbacks } from "./langchain.js";
|
|
7
|
+
export { trackMemory } from "./memory.js";
|
|
8
|
+
export { attachBlackbox } from "./openai-agents.js";
|
|
9
|
+
export { llmIdsOf, wrapAnthropic, wrapBedrock, wrapGemini, wrapOpenAI } from "./providers.js";
|
|
10
|
+
export { llmIds } from "./shared.js";
|