@zanii/blackbox 0.19.0 → 0.20.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/analysis/faults.js +15 -0
- package/dist/analysis/index.d.ts +1 -0
- package/dist/analysis/index.js +4 -0
- package/dist/analysis/learning.d.ts +54 -0
- package/dist/analysis/learning.js +191 -0
- package/dist/analysis/taxonomy.js +2 -0
- package/dist/cost/index.js +5 -1
- package/dist/fleet/insights.js +5 -1
- package/dist/index.d.ts +1 -1
- package/dist/index.js +1 -1
- package/dist/otlp/index.js +11 -3
- package/dist/session/index.d.ts +21 -0
- package/dist/session/index.js +38 -0
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/dist/analysis/faults.js
CHANGED
|
@@ -232,6 +232,21 @@ export const FAULTS = {
|
|
|
232
232
|
en: "A tool's definition changed mid-session",
|
|
233
233
|
ar: "تغيّر تعريف أداة أثناء الجلسة",
|
|
234
234
|
},
|
|
235
|
+
PROMPT_CHANGED: {
|
|
236
|
+
fault: "BBX-7405",
|
|
237
|
+
en: "The system prompt changed since the last run",
|
|
238
|
+
ar: "تغيّرت تعليمات النظام منذ التشغيل السابق",
|
|
239
|
+
},
|
|
240
|
+
SELF_CHANGE: {
|
|
241
|
+
fault: "BBX-7406",
|
|
242
|
+
en: "The agent changed itself",
|
|
243
|
+
ar: "غيّر الوكيل نفسه",
|
|
244
|
+
},
|
|
245
|
+
SELF_CHANGE_UNAPPROVED: {
|
|
246
|
+
fault: "BBX-7407",
|
|
247
|
+
en: "The agent gave itself more power without approval",
|
|
248
|
+
ar: "منح الوكيل نفسه صلاحيات أكبر دون موافقة",
|
|
249
|
+
},
|
|
235
250
|
USER_DOWNVOTE: {
|
|
236
251
|
fault: "BBX-4601",
|
|
237
252
|
en: "The person served said an answer was wrong",
|
package/dist/analysis/index.d.ts
CHANGED
|
@@ -103,3 +103,4 @@ export declare function verifyRegression(events: Ev[]): Finding[];
|
|
|
103
103
|
/** Stage 2 R4, judged at close: a run labelled (or claimed) a success that its own record contradicts:
|
|
104
104
|
* a check's last result failed, or the last tool call failed with nothing succeeding after it. */
|
|
105
105
|
export declare function falseSuccess(events: Ev[], now: number): Finding[];
|
|
106
|
+
export { type Lessons, lessons, POWER_KINDS, promptChanged, SELF_CHANGE_KINDS, selfChanges, systemPromptHash, } from "./learning.ts";
|
package/dist/analysis/index.js
CHANGED
|
@@ -16,6 +16,7 @@ import { groundingFindings, judgeFindings } from "./grounding.js";
|
|
|
16
16
|
import { toolHallucinations } from "./hallucination.js";
|
|
17
17
|
import { injectionFindings } from "./injection.js";
|
|
18
18
|
import { flightPlan } from "./landing.js";
|
|
19
|
+
import { promptChanged, selfChanges } from "./learning.js";
|
|
19
20
|
import { memoryFindings } from "./memory.js";
|
|
20
21
|
export { detectorAccuracy, wilsonLow } from "./accuracy.js";
|
|
21
22
|
export { answerClaims, answerCredential, verifyAnswerCredential, } from "./credential.js";
|
|
@@ -60,6 +61,8 @@ export function analyze(lines, options) {
|
|
|
60
61
|
...toolDrifted(events),
|
|
61
62
|
...secretSent(events),
|
|
62
63
|
...userDownvotes(events),
|
|
64
|
+
...selfChanges(events),
|
|
65
|
+
...promptChanged(events),
|
|
63
66
|
...injectionFindings(lines, options.bodies),
|
|
64
67
|
...verifyRegression(events),
|
|
65
68
|
...falseSuccess(events, now),
|
|
@@ -583,3 +586,4 @@ export function falseSuccess(events, now) {
|
|
|
583
586
|
: "The agent claimed it was done, but its last tool call failed and nothing succeeded after it.");
|
|
584
587
|
return [];
|
|
585
588
|
}
|
|
589
|
+
export { lessons, POWER_KINDS, promptChanged, SELF_CHANGE_KINDS, selfChanges, systemPromptHash, } from "./learning.js";
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { Summary } from "../fleet/index.ts";
|
|
2
|
+
import type { Ev, Finding } from "./index.ts";
|
|
3
|
+
/** What a self-change may be about. Anything else is recorded as `other`. */
|
|
4
|
+
export declare const SELF_CHANGE_KINDS: readonly ["prompt", "tool", "policy", "memory", "model", "other"];
|
|
5
|
+
/** Changes that give the agent more power: each needs an approved approval first. */
|
|
6
|
+
export declare const POWER_KINDS: readonly string[];
|
|
7
|
+
/**
|
|
8
|
+
* The fingerprint of a model request's system prompt: `sha256:` + hex over the canonical JSON of
|
|
9
|
+
* every system part it carries (Anthropic `system`, OpenAI Responses `instructions`, the `system` and
|
|
10
|
+
* `developer` messages of Chat Completions or Responses `input`, Gemini `systemInstruction`), or
|
|
11
|
+
* null when it has none.
|
|
12
|
+
*/
|
|
13
|
+
export declare function systemPromptHash(body: unknown): string | null;
|
|
14
|
+
/**
|
|
15
|
+
* spec/findings.md §1c: each self-change the agent reported (`self_change`) is SELF_CHANGE; one
|
|
16
|
+
* that gives it more power (a tool, a policy or a model) with no approval approved before it is
|
|
17
|
+
* SELF_CHANGE_UNAPPROVED.
|
|
18
|
+
*/
|
|
19
|
+
export declare function selfChanges(events: Ev[]): Finding[];
|
|
20
|
+
/**
|
|
21
|
+
* spec/findings.md §1c: the gateway stamps a request's system prompt fingerprint and, on the
|
|
22
|
+
* session's first one, the fingerprint the previous session with this label began with. A
|
|
23
|
+
* difference is PROMPT_CHANGED: advisory when the agent declared a prompt self-change before it,
|
|
24
|
+
* caution when nothing did.
|
|
25
|
+
*/
|
|
26
|
+
export declare function promptChanged(events: Ev[]): Finding[];
|
|
27
|
+
export interface Lessons {
|
|
28
|
+
label: string | null;
|
|
29
|
+
sessions: number;
|
|
30
|
+
outcomes: {
|
|
31
|
+
success: number;
|
|
32
|
+
failure: number;
|
|
33
|
+
abandoned: number;
|
|
34
|
+
unknown: number;
|
|
35
|
+
};
|
|
36
|
+
codes: Array<{
|
|
37
|
+
code: string;
|
|
38
|
+
title: string | null;
|
|
39
|
+
sessions: number;
|
|
40
|
+
failed_sessions: number;
|
|
41
|
+
findings: number;
|
|
42
|
+
last_seen: string;
|
|
43
|
+
}>;
|
|
44
|
+
claims_failed_sessions: number;
|
|
45
|
+
blocked_sessions: number;
|
|
46
|
+
brief: string;
|
|
47
|
+
}
|
|
48
|
+
/**
|
|
49
|
+
* spec/fleet.md §8: what earlier runs of one agent (its label) hit, from their summaries, newest
|
|
50
|
+
* first: outcomes, the finding codes by how many runs they were in and how many of those failed,
|
|
51
|
+
* the runs whose completion claim the record contradicted, and a short brief an agent can put in its
|
|
52
|
+
* own prompt. Codes and counts only, never content.
|
|
53
|
+
*/
|
|
54
|
+
export declare function lessons(summaries: readonly Summary[], label: string | null): Lessons;
|
|
@@ -0,0 +1,191 @@
|
|
|
1
|
+
// The learning loop (spec/fleet.md §8, spec/findings.md §1c): what earlier runs of an agent hit,
|
|
2
|
+
// for it to read before a run, and the record of any change an agent makes to itself. The agent
|
|
3
|
+
// reads lessons; it never writes to the evidence. Mirrors analysis/learning.py.
|
|
4
|
+
import { createHash } from "node:crypto";
|
|
5
|
+
import { canonical } from "../reconcile/shared.js";
|
|
6
|
+
import { FAULTS } from "./faults.js";
|
|
7
|
+
/** What a self-change may be about. Anything else is recorded as `other`. */
|
|
8
|
+
export const SELF_CHANGE_KINDS = ["prompt", "tool", "policy", "memory", "model", "other"];
|
|
9
|
+
/** Changes that give the agent more power: each needs an approved approval first. */
|
|
10
|
+
export const POWER_KINDS = ["tool", "policy", "model"];
|
|
11
|
+
const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
|
|
12
|
+
/**
|
|
13
|
+
* The fingerprint of a model request's system prompt: `sha256:` + hex over the canonical JSON of
|
|
14
|
+
* every system part it carries (Anthropic `system`, OpenAI Responses `instructions`, the `system` and
|
|
15
|
+
* `developer` messages of Chat Completions or Responses `input`, Gemini `systemInstruction`), or
|
|
16
|
+
* null when it has none.
|
|
17
|
+
*/
|
|
18
|
+
export function systemPromptHash(body) {
|
|
19
|
+
if (!isObj(body))
|
|
20
|
+
return null;
|
|
21
|
+
const parts = [];
|
|
22
|
+
if (body.system !== undefined)
|
|
23
|
+
parts.push(["system", body.system]);
|
|
24
|
+
if (typeof body.instructions === "string")
|
|
25
|
+
parts.push(["instructions", body.instructions]);
|
|
26
|
+
for (const key of ["messages", "input"])
|
|
27
|
+
if (Array.isArray(body[key]))
|
|
28
|
+
for (const m of body[key])
|
|
29
|
+
if (isObj(m) && (m.role === "system" || m.role === "developer"))
|
|
30
|
+
parts.push([m.role, m.content ?? null]);
|
|
31
|
+
if ("systemInstruction" in body || "system_instruction" in body)
|
|
32
|
+
parts.push([
|
|
33
|
+
"systemInstruction",
|
|
34
|
+
"systemInstruction" in body ? body.systemInstruction : body.system_instruction,
|
|
35
|
+
]);
|
|
36
|
+
if (!parts.length)
|
|
37
|
+
return null;
|
|
38
|
+
return `sha256:${createHash("sha256").update(canonical(parts)).digest("hex")}`;
|
|
39
|
+
}
|
|
40
|
+
/**
|
|
41
|
+
* spec/findings.md §1c: each self-change the agent reported (`self_change`) is SELF_CHANGE; one
|
|
42
|
+
* that gives it more power (a tool, a policy or a model) with no approval approved before it is
|
|
43
|
+
* SELF_CHANGE_UNAPPROVED.
|
|
44
|
+
*/
|
|
45
|
+
export function selfChanges(events) {
|
|
46
|
+
const out = [];
|
|
47
|
+
const approved = new Map();
|
|
48
|
+
for (const e of events)
|
|
49
|
+
if (e.kind === "control" && e.meta.action === "approval" && e.meta.decision === "approve")
|
|
50
|
+
if (typeof e.meta.approval_id === "string" && !approved.has(e.meta.approval_id))
|
|
51
|
+
approved.set(e.meta.approval_id, e.seq);
|
|
52
|
+
for (const e of events) {
|
|
53
|
+
if (e.kind !== "sdk.event" || e.meta.type !== "self_change")
|
|
54
|
+
continue;
|
|
55
|
+
const d = e.data ?? {};
|
|
56
|
+
const kind = typeof d.kind === "string" && d.kind ? d.kind : "other";
|
|
57
|
+
out.push({
|
|
58
|
+
code: "SELF_CHANGE",
|
|
59
|
+
source: "detectors",
|
|
60
|
+
severity: "advisory",
|
|
61
|
+
ref: { seq: e.seq, kind },
|
|
62
|
+
detail: "The agent reported a change to itself.",
|
|
63
|
+
});
|
|
64
|
+
if (!POWER_KINDS.includes(kind))
|
|
65
|
+
continue;
|
|
66
|
+
const id = typeof d.approval_id === "string" ? d.approval_id : null;
|
|
67
|
+
const at = id === null ? undefined : approved.get(id);
|
|
68
|
+
if (at !== undefined && at < e.seq)
|
|
69
|
+
continue;
|
|
70
|
+
out.push({
|
|
71
|
+
code: "SELF_CHANGE_UNAPPROVED",
|
|
72
|
+
source: "detectors",
|
|
73
|
+
severity: "warning",
|
|
74
|
+
ref: { seq: e.seq, kind, ...(id ? { approval_id: id } : {}) },
|
|
75
|
+
detail: "The agent gave itself more power (a tool, a policy or a model) with no approval approved before it.",
|
|
76
|
+
});
|
|
77
|
+
}
|
|
78
|
+
return out;
|
|
79
|
+
}
|
|
80
|
+
/**
|
|
81
|
+
* spec/findings.md §1c: the gateway stamps a request's system prompt fingerprint and, on the
|
|
82
|
+
* session's first one, the fingerprint the previous session with this label began with. A
|
|
83
|
+
* difference is PROMPT_CHANGED: advisory when the agent declared a prompt self-change before it,
|
|
84
|
+
* caution when nothing did.
|
|
85
|
+
*/
|
|
86
|
+
export function promptChanged(events) {
|
|
87
|
+
for (const e of events) {
|
|
88
|
+
const { system_sha256: after, system_sha256_prev: before } = e.meta;
|
|
89
|
+
if (e.kind !== "llm.request" || typeof after !== "string" || typeof before !== "string")
|
|
90
|
+
continue;
|
|
91
|
+
if (after === before)
|
|
92
|
+
return [];
|
|
93
|
+
const declared = events.some((s) => s.seq < e.seq &&
|
|
94
|
+
s.kind === "sdk.event" &&
|
|
95
|
+
s.meta.type === "self_change" &&
|
|
96
|
+
s.data?.kind === "prompt");
|
|
97
|
+
return [
|
|
98
|
+
{
|
|
99
|
+
code: "PROMPT_CHANGED",
|
|
100
|
+
source: "detectors",
|
|
101
|
+
severity: declared ? "advisory" : "caution",
|
|
102
|
+
ref: { seq: e.seq, before, after, declared },
|
|
103
|
+
detail: declared
|
|
104
|
+
? "The system prompt differs from the one the last run of this agent began with; the agent declared the change."
|
|
105
|
+
: "The system prompt differs from the one the last run of this agent began with, and nothing declared the change.",
|
|
106
|
+
},
|
|
107
|
+
];
|
|
108
|
+
}
|
|
109
|
+
return [];
|
|
110
|
+
}
|
|
111
|
+
/** At most this many codes in a lesson: the ones seen in the most runs. */
|
|
112
|
+
const TOP = 10;
|
|
113
|
+
/**
|
|
114
|
+
* spec/fleet.md §8: what earlier runs of one agent (its label) hit, from their summaries, newest
|
|
115
|
+
* first: outcomes, the finding codes by how many runs they were in and how many of those failed,
|
|
116
|
+
* the runs whose completion claim the record contradicted, and a short brief an agent can put in its
|
|
117
|
+
* own prompt. Codes and counts only, never content.
|
|
118
|
+
*/
|
|
119
|
+
export function lessons(summaries, label) {
|
|
120
|
+
const outcomes = { success: 0, failure: 0, abandoned: 0, unknown: 0 };
|
|
121
|
+
const codes = new Map();
|
|
122
|
+
let claimsFailed = 0;
|
|
123
|
+
let blocked = 0;
|
|
124
|
+
for (const s of summaries) {
|
|
125
|
+
const o = s.outcome;
|
|
126
|
+
const failed = o === "failure";
|
|
127
|
+
if (o === "success" || o === "failure" || o === "abandoned")
|
|
128
|
+
outcomes[o]++;
|
|
129
|
+
else
|
|
130
|
+
outcomes.unknown++;
|
|
131
|
+
if ((s.claims?.failed ?? 0) > 0)
|
|
132
|
+
claimsFailed++;
|
|
133
|
+
if (s.blocked_by)
|
|
134
|
+
blocked++;
|
|
135
|
+
for (const [code, n] of Object.entries(s.finding_codes ?? {})) {
|
|
136
|
+
const c = codes.get(code) ?? { sessions: 0, failed_sessions: 0, findings: 0, last_seen: "" };
|
|
137
|
+
c.sessions++;
|
|
138
|
+
if (failed)
|
|
139
|
+
c.failed_sessions++;
|
|
140
|
+
c.findings += n;
|
|
141
|
+
if (s.started_at > c.last_seen)
|
|
142
|
+
c.last_seen = s.started_at;
|
|
143
|
+
codes.set(code, c);
|
|
144
|
+
}
|
|
145
|
+
}
|
|
146
|
+
const top = [...codes.entries()]
|
|
147
|
+
.sort((a, b) => b[1].sessions - a[1].sessions || (a[0] < b[0] ? -1 : 1))
|
|
148
|
+
.slice(0, TOP)
|
|
149
|
+
.map(([code, c]) => ({
|
|
150
|
+
code,
|
|
151
|
+
title: FAULTS[code]?.en ?? null,
|
|
152
|
+
...c,
|
|
153
|
+
}));
|
|
154
|
+
const n = summaries.length;
|
|
155
|
+
const name = label === null ? "this agent" : `"${label}"`;
|
|
156
|
+
let brief;
|
|
157
|
+
if (n === 0)
|
|
158
|
+
brief = `No earlier runs of ${name} on record.`;
|
|
159
|
+
else {
|
|
160
|
+
const parts = [`Lessons from the last ${n} run${n === 1 ? "" : "s"} of ${name}:`];
|
|
161
|
+
const counts = [`${outcomes.success} succeeded`, `${outcomes.failure} failed`];
|
|
162
|
+
if (outcomes.abandoned)
|
|
163
|
+
counts.push(`${outcomes.abandoned} abandoned`);
|
|
164
|
+
if (outcomes.unknown)
|
|
165
|
+
counts.push(`${outcomes.unknown} without an outcome`);
|
|
166
|
+
parts.push(`${counts.join(", ")}.`);
|
|
167
|
+
if (top.length)
|
|
168
|
+
parts.push(`Problems seen: ${top
|
|
169
|
+
.slice(0, 5)
|
|
170
|
+
.map((c) => `${c.code}${c.title ? ` (${c.title.toLowerCase()})` : ""} in ${c.sessions} run${c.sessions === 1 ? "" : "s"}${c.failed_sessions ? (c.sessions === 1 ? ", which failed" : `, ${c.failed_sessions} of them failed`) : ""}`)
|
|
171
|
+
.join("; ")}.`);
|
|
172
|
+
else
|
|
173
|
+
parts.push("No problems were found in them.");
|
|
174
|
+
if (claimsFailed)
|
|
175
|
+
parts.push(`${claimsFailed} run${claimsFailed === 1 ? "" : "s"} claimed to be done and the record disagreed.`);
|
|
176
|
+
if (blocked)
|
|
177
|
+
parts.push(`${blocked} run${blocked === 1 ? " was" : "s were"} stopped by the gateway.`);
|
|
178
|
+
if (top.length || claimsFailed)
|
|
179
|
+
parts.push("Avoid repeating these.");
|
|
180
|
+
brief = parts.join(" ");
|
|
181
|
+
}
|
|
182
|
+
return {
|
|
183
|
+
label,
|
|
184
|
+
sessions: n,
|
|
185
|
+
outcomes,
|
|
186
|
+
codes: top,
|
|
187
|
+
claims_failed_sessions: claimsFailed,
|
|
188
|
+
blocked_sessions: blocked,
|
|
189
|
+
brief,
|
|
190
|
+
};
|
|
191
|
+
}
|
|
@@ -56,6 +56,8 @@ export const TAXONOMY = {
|
|
|
56
56
|
TOOL_CHANGED: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },
|
|
57
57
|
TOOL_DRIFTED: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },
|
|
58
58
|
PROMPT_INJECTION: { owasp: ["ASI01"], atlas: ["AML.T0051"] },
|
|
59
|
+
PROMPT_CHANGED: { owasp: ["ASI04"] },
|
|
60
|
+
SELF_CHANGE_UNAPPROVED: { owasp: ["ASI10"] },
|
|
59
61
|
SECRET_SENT: { owasp: ["ASI03"], atlas: ["AML.T0057"] },
|
|
60
62
|
TOOL_OUTPUT_MISMATCH: { owasp: ["ASI04", "ASI06"], atlas: ["AML.T0110.002", "AML.T0099"] },
|
|
61
63
|
TOOL_WITNESS_INVALID: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },
|
package/dist/cost/index.js
CHANGED
|
@@ -127,7 +127,11 @@ export function tokensOf(usage) {
|
|
|
127
127
|
cache_write: written,
|
|
128
128
|
}, { input_audio: audioIn, output_audio: audioOut });
|
|
129
129
|
}
|
|
130
|
-
|
|
130
|
+
// Anthropic's cache names are its own: they win over the Responses shape, since Anthropic also
|
|
131
|
+
// sends output_tokens_details (thinking_tokens) since Claude Sonnet 5
|
|
132
|
+
const anthropicCache = "cache_read_input_tokens" in usage || "cache_creation_input_tokens" in usage;
|
|
133
|
+
if (!anthropicCache &&
|
|
134
|
+
Object.keys(usage).some((k) => k.startsWith("input_tokens_details_") || k.startsWith("output_tokens_details_"))) {
|
|
131
135
|
const cached = n("input_tokens_details_cached_tokens");
|
|
132
136
|
const written = n("input_tokens_details_cache_write_tokens");
|
|
133
137
|
const audioIn = n("input_tokens_details_audio_tokens");
|
package/dist/fleet/insights.js
CHANGED
|
@@ -179,7 +179,11 @@ function value(v) {
|
|
|
179
179
|
return JSON.stringify(short);
|
|
180
180
|
}
|
|
181
181
|
const isMoment = (e) => {
|
|
182
|
-
const
|
|
182
|
+
const meta = e.meta;
|
|
183
|
+
const status = meta?.status;
|
|
184
|
+
// an allowed policy check (one per local tool call) is routine, not a moment
|
|
185
|
+
if (e.kind === "control" && meta?.action === "policy_check" && meta.decision === "allow")
|
|
186
|
+
return false;
|
|
183
187
|
return (e.kind === "finding" ||
|
|
184
188
|
e.kind === "control" ||
|
|
185
189
|
e.kind === "outcome" ||
|
package/dist/index.d.ts
CHANGED
|
@@ -2,7 +2,7 @@ export { A2A_STREAMING, type AgentCardReport, a2aMethod, a2aRequestMeta, a2aResp
|
|
|
2
2
|
export { causality, memoryXray } from "./agents/index.ts";
|
|
3
3
|
export { type DetectorOptions, detectors, fingerprint } from "./analysis/detectors.ts";
|
|
4
4
|
export { FAULTS, type Fault, faultOf } from "./analysis/faults.ts";
|
|
5
|
-
export { type Accuracy, type AnalyzeOptions, type AnswerClaims, analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, type ClaimEntry, type ClaimVerdict, type CriterionResult, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, type Fact, type Finding as AnalysisFinding, factsIn, findingKey, type GroundedClaim, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, type Label as AccuracyLabel, type Landing, landing, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, type Outcome, type OutcomeRollup, packsFor, parseGroundingAnswer, parseJudgeAnswer, type ReferenceFact, type ReferencePack, type ReferenceVerdict, referenceContext, rollup, type Severity, schemaProblem, secretSent, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, type WasteReason, type WasteReport, waste, wilsonLow, } from "./analysis/index.ts";
|
|
5
|
+
export { type Accuracy, type AnalyzeOptions, type AnswerClaims, analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, type ClaimEntry, type ClaimVerdict, type CriterionResult, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, type Fact, type Finding as AnalysisFinding, factsIn, findingKey, type GroundedClaim, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, type Label as AccuracyLabel, type Landing, landing, lessons, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, type Outcome, type OutcomeRollup, packsFor, parseGroundingAnswer, parseJudgeAnswer, promptChanged, type ReferenceFact, type ReferencePack, type ReferenceVerdict, referenceContext, rollup, type Severity, schemaProblem, secretSent, selfChanges, systemPromptHash, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, type WasteReason, type WasteReport, waste, wilsonLow, } from "./analysis/index.ts";
|
|
6
6
|
export { injectionFindings, injectionRule } from "./analysis/injection.ts";
|
|
7
7
|
export { MITRE_ATLAS, OWASP_ASI, TAXONOMY, type Taxonomy, taxonomyOf, } from "./analysis/taxonomy.ts";
|
|
8
8
|
export { type ApprovalFinding, approvalConfirmation, approvalFindings, argsHash, verifyConfirmation, } from "./approvals/index.ts";
|
package/dist/index.js
CHANGED
|
@@ -4,7 +4,7 @@ export { A2A_STREAMING, a2aMethod, a2aRequestMeta, a2aResponseMeta, a2aRestMetho
|
|
|
4
4
|
export { causality, memoryXray } from "./agents/index.js";
|
|
5
5
|
export { detectors, fingerprint } from "./analysis/detectors.js";
|
|
6
6
|
export { FAULTS, faultOf } from "./analysis/faults.js";
|
|
7
|
-
export { analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, factsIn, findingKey, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, landing, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, packsFor, parseGroundingAnswer, parseJudgeAnswer, referenceContext, rollup, schemaProblem, secretSent, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, waste, wilsonLow, } from "./analysis/index.js";
|
|
7
|
+
export { analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, factsIn, findingKey, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, landing, lessons, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, packsFor, parseGroundingAnswer, parseJudgeAnswer, promptChanged, referenceContext, rollup, schemaProblem, secretSent, selfChanges, systemPromptHash, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, waste, wilsonLow, } from "./analysis/index.js";
|
|
8
8
|
export { injectionFindings, injectionRule } from "./analysis/injection.js";
|
|
9
9
|
export { MITRE_ATLAS, OWASP_ASI, TAXONOMY, taxonomyOf, } from "./analysis/taxonomy.js";
|
|
10
10
|
export { approvalConfirmation, approvalFindings, argsHash, verifyConfirmation, } from "./approvals/index.js";
|
package/dist/otlp/index.js
CHANGED
|
@@ -398,7 +398,8 @@ function toOtlpV2(lines) {
|
|
|
398
398
|
],
|
|
399
399
|
};
|
|
400
400
|
}
|
|
401
|
-
|
|
401
|
+
// Spring AI says `embedding` (singular) for the semconv `embeddings`
|
|
402
|
+
const LLM_OPS = new Set(["chat", "text_completion", "generate_content", "embeddings", "embedding"]);
|
|
402
403
|
const MAX_SPANS = 5000;
|
|
403
404
|
const MAX_DATA = 60_000;
|
|
404
405
|
function anyValue(v) {
|
|
@@ -433,13 +434,20 @@ function attributes(list) {
|
|
|
433
434
|
function spanType(a) {
|
|
434
435
|
const op = a["gen_ai.operation.name"];
|
|
435
436
|
const kind = a["openinference.span.kind"];
|
|
437
|
+
const spring = a["spring.ai.kind"]; // Spring AI's own span kinds
|
|
436
438
|
if ((typeof op === "string" && LLM_OPS.has(op)) || kind === "LLM" || kind === "EMBEDDING")
|
|
437
439
|
return "llm.call";
|
|
438
440
|
if (op === "execute_tool" || kind === "TOOL")
|
|
439
441
|
return "tool.call";
|
|
440
|
-
if (kind === "RETRIEVER")
|
|
442
|
+
if (kind === "RETRIEVER" || spring === "vector_store")
|
|
441
443
|
return "retrieval";
|
|
442
|
-
if (op === "invoke_agent" ||
|
|
444
|
+
if (op === "invoke_agent" ||
|
|
445
|
+
op === "create_agent" ||
|
|
446
|
+
op === "framework" ||
|
|
447
|
+
kind === "AGENT" ||
|
|
448
|
+
kind === "CHAIN" ||
|
|
449
|
+
spring === "chat_client" ||
|
|
450
|
+
spring === "advisor")
|
|
443
451
|
return "step";
|
|
444
452
|
return "otel.span";
|
|
445
453
|
}
|
package/dist/session/index.d.ts
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { type Lessons } from "../analysis/learning.ts";
|
|
1
2
|
import { type Attestation } from "../attest/index.ts";
|
|
2
3
|
import { buildPayment, type screenCounterparty } from "../finance/index.ts";
|
|
3
4
|
export interface SessionOptions {
|
|
@@ -368,6 +369,26 @@ export declare class BlackboxSession {
|
|
|
368
369
|
/** Audit S17: the gateway's view of this session (blocked? findings?), or null if it can't be
|
|
369
370
|
* read. Lets an agent react to its own block before its next model call. Never throws. */
|
|
370
371
|
state(): Promise<SessionState | null>;
|
|
372
|
+
/**
|
|
373
|
+
* spec/fleet.md §8: what earlier runs of this agent (this session's label) hit, to read before a
|
|
374
|
+
* run: outcomes, finding codes and a `brief` to put in the agent's own prompt. Codes and counts
|
|
375
|
+
* only. The gateway records that the agent read them. `null` when it can't be asked.
|
|
376
|
+
*/
|
|
377
|
+
lessons(options?: {
|
|
378
|
+
limit?: number;
|
|
379
|
+
}): Promise<Lessons | null>;
|
|
380
|
+
/**
|
|
381
|
+
* spec/findings.md §1c: the agent changed itself (its prompt, a tool, its policy, its memory, its
|
|
382
|
+
* model). Only hashes of `before` and `after` are sent, never the content. A change that gives
|
|
383
|
+
* more power (tool, policy, model) needs an approval approved first (`requestApproval`, then
|
|
384
|
+
* pass its id), or it is SELF_CHANGE_UNAPPROVED. Never throws.
|
|
385
|
+
*/
|
|
386
|
+
selfChange(kind: string, options?: {
|
|
387
|
+
before?: string;
|
|
388
|
+
after?: string;
|
|
389
|
+
reason?: string;
|
|
390
|
+
approvalId?: string;
|
|
391
|
+
}): void;
|
|
371
392
|
/** spec/findings.md §13.1: what in an answer (the latest, or the one at `seq`) would hold it,
|
|
372
393
|
* for an app that streams the answer by itself: `{seq, risks, hold}`. `hold` says whether the
|
|
373
394
|
* server would hold it. `null` when there's no answer yet or the gateway can't be asked. */
|
package/dist/session/index.js
CHANGED
|
@@ -6,6 +6,7 @@ import { request as httpRequest } from "node:http";
|
|
|
6
6
|
import { request as httpsRequest } from "node:https";
|
|
7
7
|
import { tmpdir } from "node:os";
|
|
8
8
|
import { join } from "node:path";
|
|
9
|
+
import { SELF_CHANGE_KINDS } from "../analysis/learning.js";
|
|
9
10
|
import { argsHash } from "../approvals/index.js";
|
|
10
11
|
import { attest, isReadOnly, parseCommand } from "../attest/index.js";
|
|
11
12
|
import { buildPayment, ftaPayload, screeningPayload, } from "../finance/index.js";
|
|
@@ -596,6 +597,43 @@ export class BlackboxSession {
|
|
|
596
597
|
}
|
|
597
598
|
return null;
|
|
598
599
|
}
|
|
600
|
+
/**
|
|
601
|
+
* spec/fleet.md §8: what earlier runs of this agent (this session's label) hit, to read before a
|
|
602
|
+
* run: outcomes, finding codes and a `brief` to put in the agent's own prompt. Codes and counts
|
|
603
|
+
* only. The gateway records that the agent read them. `null` when it can't be asked.
|
|
604
|
+
*/
|
|
605
|
+
async lessons(options = {}) {
|
|
606
|
+
if (!this.enabled)
|
|
607
|
+
return null;
|
|
608
|
+
const q = options.limit === undefined ? "" : `?limit=${Math.trunc(options.limit)}`;
|
|
609
|
+
try {
|
|
610
|
+
const r = await send("GET", this.options.url, `/v1/sessions/${this.id}/lessons${q}`, this.token);
|
|
611
|
+
if (r.status === 200)
|
|
612
|
+
return r.json;
|
|
613
|
+
this.fail("rejected", `reading the lessons: the gateway answered ${r.status}`);
|
|
614
|
+
}
|
|
615
|
+
catch (error) {
|
|
616
|
+
this.fail("network", `reading the lessons: ${message(error)}`);
|
|
617
|
+
}
|
|
618
|
+
return null;
|
|
619
|
+
}
|
|
620
|
+
/**
|
|
621
|
+
* spec/findings.md §1c: the agent changed itself (its prompt, a tool, its policy, its memory, its
|
|
622
|
+
* model). Only hashes of `before` and `after` are sent, never the content. A change that gives
|
|
623
|
+
* more power (tool, policy, model) needs an approval approved first (`requestApproval`, then
|
|
624
|
+
* pass its id), or it is SELF_CHANGE_UNAPPROVED. Never throws.
|
|
625
|
+
*/
|
|
626
|
+
selfChange(kind, options = {}) {
|
|
627
|
+
const k = SELF_CHANGE_KINDS.includes(kind) ? kind : "other";
|
|
628
|
+
const sha = (text) => `sha256:${createHash("sha256").update(text).digest("hex")}`;
|
|
629
|
+
this.event("self_change", k, {
|
|
630
|
+
kind: k,
|
|
631
|
+
...(options.before === undefined ? {} : { before_sha256: sha(options.before) }),
|
|
632
|
+
...(options.after === undefined ? {} : { after_sha256: sha(options.after) }),
|
|
633
|
+
...(options.reason ? { reason: options.reason.slice(0, 500) } : {}),
|
|
634
|
+
...(options.approvalId ? { approval_id: options.approvalId.slice(0, 128) } : {}),
|
|
635
|
+
});
|
|
636
|
+
}
|
|
599
637
|
/** spec/findings.md §13.1: what in an answer (the latest, or the one at `seq`) would hold it,
|
|
600
638
|
* for an app that streams the answer by itself: `{seq, risks, hold}`. `hold` says whether the
|
|
601
639
|
* server would hold it. `null` when there's no answer yet or the gateway can't be asked. */
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "0.20.1";
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// The SDK version, on its own so the CLI can print it without loading the whole SDK.
|
|
2
|
-
export const VERSION = "0.
|
|
2
|
+
export const VERSION = "0.20.1";
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zanii/blackbox",
|
|
3
3
|
"license": "Apache-2.0",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.20.1",
|
|
5
5
|
"description": "The flight recorder for AI agents: sessions, a zero-loss spool, framework hooks, approvals, and offline verification of the gateway's hash-chained record.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"ai-agents",
|