@zanii/blackbox 0.5.0 → 0.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +10 -0
- package/dist/analysis/detectors.js +56 -5
- package/dist/analysis/hallucination.js +16 -1
- package/dist/session/index.d.ts +4 -1
- package/dist/session/index.js +60 -9
- package/dist/version.d.ts +1 -1
- package/dist/version.js +1 -1
- package/package.json +1 -1
package/README.md
CHANGED
|
@@ -83,6 +83,16 @@ verifyAnswerCredential(credential, lines, bodies, packs); // { ok, problems }: t
|
|
|
83
83
|
|
|
84
84
|
The same checks run offline as pure functions: `toolHallucinations`, `groundingFindings`, `factsIn`, `loadReferencePacks`, `answerRisk`, `detectorAccuracy`.
|
|
85
85
|
|
|
86
|
+
## New in 0.7.0
|
|
87
|
+
|
|
88
|
+
- Any number of serverless functions or workers can write one session at the same moment: a joined session sends its events unnumbered, each with an id, and a v0.9+ gateway numbers them in arrival order (a resend adds nothing).
|
|
89
|
+
- The repeat-call detector counts within one turn: a chat assistant or support bot asked for the same lookup in separate messages isn't stopped as a loop. A real loop inside one task still is.
|
|
90
|
+
|
|
91
|
+
## New in 0.6.0
|
|
92
|
+
|
|
93
|
+
- Serverless: a session joined with its id and token in a fresh function (Vercel, Lambda, Cloud Run) numbers its events from the server's `sdk_next`, so none are lost. Join in each invocation and flush before returning.
|
|
94
|
+
- Fewer false alarms: an ISO date-time (`2026-10-03T09:30:00+04:00`) in a tool's arguments is a date, not an invented number, and an MCP call carrying out the model's own tool call isn't flagged a second time for the same value.
|
|
95
|
+
|
|
86
96
|
## New in 0.5.0
|
|
87
97
|
|
|
88
98
|
- Held streams: open a session with `holdStreams: true` (or set `BLACKBOX_HOLD_STREAMS=all` on the gateway) and a risky streamed answer waits for a person, as a non-streamed one does. The client gets keep-alives, then the whole stream, or one error in the provider's own format.
|
|
@@ -135,27 +135,78 @@ function changedSegment(before, after) {
|
|
|
135
135
|
return b.length >= a.length ? "none" : a[b.length][0];
|
|
136
136
|
}
|
|
137
137
|
const f = (code, severity, ref, detail) => ({ code, source: "detectors", severity, ref, detail });
|
|
138
|
+
/** spec/findings.md §4: what the person has said in a model request (Anthropic, Chat Completions,
|
|
139
|
+
* Responses): how many messages, and the newest. Tool results aren't the person speaking. */
|
|
140
|
+
function humanTail(request) {
|
|
141
|
+
const raw = request?.messages ?? request?.input;
|
|
142
|
+
const list = Array.isArray(raw)
|
|
143
|
+
? raw
|
|
144
|
+
: typeof raw === "string"
|
|
145
|
+
? [{ role: "user", content: raw }]
|
|
146
|
+
: [];
|
|
147
|
+
let count = 0;
|
|
148
|
+
let last = "";
|
|
149
|
+
for (const m of list) {
|
|
150
|
+
if (!isObj(m) || m.role !== "user")
|
|
151
|
+
continue;
|
|
152
|
+
const c = m.content;
|
|
153
|
+
if (Array.isArray(c)) {
|
|
154
|
+
const blocks = c.filter(isObj);
|
|
155
|
+
if (blocks.some((b) => b.type === "tool_result"))
|
|
156
|
+
continue;
|
|
157
|
+
last = blocks
|
|
158
|
+
.map((b) => (typeof b.text === "string" ? b.text : ""))
|
|
159
|
+
.filter(Boolean)
|
|
160
|
+
.join("\n");
|
|
161
|
+
}
|
|
162
|
+
else
|
|
163
|
+
last = typeof c === "string" ? c : "";
|
|
164
|
+
count++;
|
|
165
|
+
}
|
|
166
|
+
return `${count}\n${last}`;
|
|
167
|
+
}
|
|
168
|
+
/** §4: each tool call's turn. A turn starts when the person says something new (a new message,
|
|
169
|
+
* or different newest words); a client resending the same request is still the same turn. */
|
|
170
|
+
function turnsOf(calls, tools) {
|
|
171
|
+
const byEnd = new Map();
|
|
172
|
+
let turn = -1;
|
|
173
|
+
let previous = null;
|
|
174
|
+
for (const c of calls) {
|
|
175
|
+
const tail = humanTail(c.request);
|
|
176
|
+
if (tail !== previous)
|
|
177
|
+
turn++;
|
|
178
|
+
previous = tail;
|
|
179
|
+
byEnd.set(c.endSeq, Math.max(turn, 0));
|
|
180
|
+
}
|
|
181
|
+
return tools.map((t) => byEnd.get(t.seq) ?? 0);
|
|
182
|
+
}
|
|
138
183
|
export function detectors(lines, options) {
|
|
139
184
|
const calls = callsOf(lines, options.bodies);
|
|
140
185
|
const byEnd = [...calls].sort((a, b) => a.endSeq - b.endSeq);
|
|
141
186
|
const tools = toolCallsOf(calls);
|
|
142
187
|
const results = toolResultsOf(calls, lines);
|
|
143
188
|
const keys = tools.map((t) => `${t.tool}\n${t.fp}`);
|
|
189
|
+
const turn = turnsOf(calls, tools);
|
|
144
190
|
const first = lines[0] ? JSON.parse(lines[0]) : null;
|
|
145
191
|
const open = first?.kind === "session.open" ? first.meta : {};
|
|
146
192
|
const out = [];
|
|
147
|
-
// D1: the 3rd / 5th occurrence of the same call within the last 20 tool calls
|
|
193
|
+
// D1: the 3rd / 5th occurrence of the same call within the last 20 tool calls of one turn: the
|
|
194
|
+
// same lookup asked for again in a person's next message isn't a loop.
|
|
148
195
|
const d1 = new Set();
|
|
149
196
|
const repeat = [];
|
|
197
|
+
let turnStart = 0;
|
|
150
198
|
keys.forEach((key, i) => {
|
|
151
|
-
|
|
199
|
+
if (i > 0 && turn[i] !== turn[i - 1])
|
|
200
|
+
turnStart = i;
|
|
201
|
+
const from = Math.max(turnStart, i - WINDOW + 1);
|
|
202
|
+
const count = keys.slice(from, i + 1).filter((k) => k === key).length;
|
|
152
203
|
repeat.push(count >= 2);
|
|
153
204
|
for (const [at, severity] of [
|
|
154
205
|
[3, "caution"],
|
|
155
206
|
[5, "warning"],
|
|
156
207
|
])
|
|
157
|
-
if (count === at && !d1.has(`${key}\n${at}`)) {
|
|
158
|
-
d1.add(`${key}\n${at}`);
|
|
208
|
+
if (count === at && !d1.has(`${turn[i]}\n${key}\n${at}`)) {
|
|
209
|
+
d1.add(`${turn[i]}\n${key}\n${at}`);
|
|
159
210
|
const t = tools[i];
|
|
160
211
|
out.push(f("D1_REPEAT_CALL", severity, { tool: t.tool, fingerprint: t.fp, count: at }, `The same ${t.tool} call ${at} times in the last ${WINDOW} tool calls.`));
|
|
161
212
|
}
|
|
@@ -165,7 +216,7 @@ export function detectors(lines, options) {
|
|
|
165
216
|
for (let i = 0; i < keys.length;) {
|
|
166
217
|
let found = 0;
|
|
167
218
|
for (let n = 2; n <= 4 && !found; n++) {
|
|
168
|
-
if (i + 3 * n > keys.length)
|
|
219
|
+
if (i + 3 * n > keys.length || turn[i] !== turn[i + 3 * n - 1])
|
|
169
220
|
break;
|
|
170
221
|
const gram = keys.slice(i, i + n);
|
|
171
222
|
if (new Set(gram).size === 1)
|
|
@@ -114,6 +114,7 @@ export function normaliseValue(s) {
|
|
|
114
114
|
.replace(/[\s\-_./()+:,]/g, "");
|
|
115
115
|
}
|
|
116
116
|
const DATE = /^[0-9]{4}-[0-9]{2}-[0-9]{2}$|^[0-9]{2}-[0-9]{2}-[0-9]{4}$/;
|
|
117
|
+
const DATE_TIME = /(?<![0-9])[0-9]{4}-[0-9]{2}-[0-9]{2}(?:[T ][0-9]{2}:[0-9]{2}(?::[0-9]{2}(?:\.[0-9]+)?)?(?:Z|[+-][0-9]{2}:?[0-9]{2})?)?(?![0-9])/g;
|
|
117
118
|
const VALUE_KINDS = [
|
|
118
119
|
["email", /[A-Za-z0-9._%+-]+@[A-Za-z0-9.-]+\.[A-Za-z]{2,}/g],
|
|
119
120
|
["iban", /\b[A-Z]{2}[0-9]{2}(?:[ ]?[A-Z0-9]){11,30}\b/g],
|
|
@@ -124,6 +125,12 @@ const VALUE_KINDS = [
|
|
|
124
125
|
/** The identifying values in a string (Western digits) and where they are, skipping `taken` spans. */
|
|
125
126
|
export function identifierSpans(s, taken = []) {
|
|
126
127
|
const out = [];
|
|
128
|
+
// a date or an ISO 8601 date-time (2026-10-03T00:00:00+04:00) is a date, not part of a number
|
|
129
|
+
for (const m of s.matchAll(DATE_TIME)) {
|
|
130
|
+
const at = m.index ?? 0;
|
|
131
|
+
if (!taken.some(([a, b]) => at < b && at + m[0].length > a))
|
|
132
|
+
taken.push([at, at + m[0].length]);
|
|
133
|
+
}
|
|
127
134
|
for (const [kind, re] of VALUE_KINDS)
|
|
128
135
|
for (const m of s.matchAll(re)) {
|
|
129
136
|
const at = m.index ?? 0;
|
|
@@ -245,9 +252,14 @@ export function toolHallucinations(lines, bodies) {
|
|
|
245
252
|
let request = "";
|
|
246
253
|
let toolText = "";
|
|
247
254
|
let sawModel = false;
|
|
248
|
-
for
|
|
255
|
+
// §8: what the model asked for is where an agent's MCP call got its values; the model's call is
|
|
256
|
+
// checked above, so the MCP call it led to isn't flagged a second time for the same act
|
|
257
|
+
const modelArgs = new Map();
|
|
258
|
+
for (const t of modelCalls) {
|
|
249
259
|
if (!called.has(t.tool))
|
|
250
260
|
called.set(t.tool, t.seq);
|
|
261
|
+
modelArgs.set(t.seq, `${modelArgs.get(t.seq) ?? ""}\n${normaliseValue(canonical(t.args ?? {}))}`);
|
|
262
|
+
}
|
|
251
263
|
for (const e of events) {
|
|
252
264
|
const m = e.meta;
|
|
253
265
|
const text = () => {
|
|
@@ -258,6 +270,9 @@ export function toolHallucinations(lines, bodies) {
|
|
|
258
270
|
sawModel = true;
|
|
259
271
|
request = normaliseValue(text());
|
|
260
272
|
}
|
|
273
|
+
else if (e.kind === "llm.response" && modelArgs.has(e.seq)) {
|
|
274
|
+
toolText += modelArgs.get(e.seq);
|
|
275
|
+
}
|
|
261
276
|
else if (e.kind === "tool.result" && typeof m.server === "string" && isObj(m.tool_defs)) {
|
|
262
277
|
const tools = rpcOf(bodies(e.body_hash))?.result;
|
|
263
278
|
const schemas = new Map();
|
package/dist/session/index.d.ts
CHANGED
|
@@ -167,11 +167,14 @@ export declare class BlackboxSession {
|
|
|
167
167
|
/** A session that records nothing. `reason`, when opening failed, goes to `stats.lastError` (audit K1). */
|
|
168
168
|
static disabled(options: SessionOptions, reason?: string): BlackboxSession;
|
|
169
169
|
private readonly options;
|
|
170
|
+
private readonly serverNumbering;
|
|
170
171
|
readonly id: string;
|
|
171
172
|
readonly token: string;
|
|
172
173
|
constructor(options: SessionOptions, id: string, token: string, enabled?: boolean,
|
|
173
174
|
/** Audit K5: this SDK opened the session, so only it knows the token: keep it by the spool. */
|
|
174
|
-
keepToken?: boolean
|
|
175
|
+
keepToken?: boolean,
|
|
176
|
+
/** spec/sdk.md §1: events go unnumbered, each with an event_id; the server numbers them. */
|
|
177
|
+
serverNumbering?: boolean);
|
|
175
178
|
event(type: string, name?: string, data?: Record<string, unknown>, options?: {
|
|
176
179
|
eventId?: string;
|
|
177
180
|
/** spec/sdk.md §1.1: an image the event's body is (see `screen`). */
|
package/dist/session/index.js
CHANGED
|
@@ -58,6 +58,32 @@ export function imageType(image) {
|
|
|
58
58
|
}
|
|
59
59
|
const LOCK_WAIT_MS = 2_000;
|
|
60
60
|
const LOCK_STALE_MS = 10_000;
|
|
61
|
+
const spoolDirOf = (options) => options.spoolDir ?? process.env.BLACKBOX_SPOOL_DIR ?? join(tmpdir(), "zanii-blackbox-spool");
|
|
62
|
+
/** spec/sdk.md §1: joining a session another process may write to (a serverless function, a
|
|
63
|
+
* second worker). When the server numbers events itself, this SDK sends them unnumbered with an
|
|
64
|
+
* event_id, so two writers never collide: true then. Otherwise, without a spool, it numbers from
|
|
65
|
+
* the server's `next` (from 0, its events would be skipped as already recorded). */
|
|
66
|
+
async function joinSession(options, id, token) {
|
|
67
|
+
const dir = spoolDirOf(options);
|
|
68
|
+
try {
|
|
69
|
+
const r = await send("GET", options.url, `/v1/sessions/${id}/state`, token);
|
|
70
|
+
if (r.status !== 200)
|
|
71
|
+
return false;
|
|
72
|
+
const numbering = r.json.sdk_numbering;
|
|
73
|
+
if (Array.isArray(numbering) && numbering.includes("server"))
|
|
74
|
+
return true;
|
|
75
|
+
const n = r.json.sdk_next;
|
|
76
|
+
const have = existsSync(join(dir, `${id}.jsonl`)) || existsSync(join(dir, `${id}.ack`));
|
|
77
|
+
if (typeof n === "number" && Number.isInteger(n) && n > 0 && !have) {
|
|
78
|
+
mkdirSync(dir, { recursive: true });
|
|
79
|
+
writeFileSync(join(dir, `${id}.ack`), JSON.stringify({ next: n, offset: 0, gen: 0 }));
|
|
80
|
+
}
|
|
81
|
+
}
|
|
82
|
+
catch {
|
|
83
|
+
// offline: the spool keeps the events, numbered locally
|
|
84
|
+
}
|
|
85
|
+
return false;
|
|
86
|
+
}
|
|
61
87
|
/** Opens (or joins) a session. Never throws: on failure it returns a disabled session and logs why. */
|
|
62
88
|
export async function session(options) {
|
|
63
89
|
const log = options.logger ?? (() => { });
|
|
@@ -95,7 +121,8 @@ export async function session(options) {
|
|
|
95
121
|
token = String(r.json.token);
|
|
96
122
|
return new BlackboxSession(options, id, token, true, true);
|
|
97
123
|
}
|
|
98
|
-
|
|
124
|
+
const serverNumbering = await joinSession(options, id, token);
|
|
125
|
+
return new BlackboxSession(options, id, token, true, false, serverNumbering);
|
|
99
126
|
}
|
|
100
127
|
catch (error) {
|
|
101
128
|
log(`blackbox: session disabled: ${message(error)}`);
|
|
@@ -134,16 +161,20 @@ export class BlackboxSession {
|
|
|
134
161
|
return s;
|
|
135
162
|
}
|
|
136
163
|
options;
|
|
164
|
+
serverNumbering;
|
|
137
165
|
id;
|
|
138
166
|
token;
|
|
139
167
|
constructor(options, id, token, enabled = true,
|
|
140
168
|
/** Audit K5: this SDK opened the session, so only it knows the token: keep it by the spool. */
|
|
141
|
-
keepToken = false
|
|
169
|
+
keepToken = false,
|
|
170
|
+
/** spec/sdk.md §1: events go unnumbered, each with an event_id; the server numbers them. */
|
|
171
|
+
serverNumbering = false) {
|
|
172
|
+
this.serverNumbering = serverNumbering;
|
|
142
173
|
this.options = options;
|
|
143
174
|
this.id = id;
|
|
144
175
|
this.token = token;
|
|
145
176
|
this.enabled = enabled;
|
|
146
|
-
const dir = options
|
|
177
|
+
const dir = spoolDirOf(options);
|
|
147
178
|
this.spool = join(dir, `${id}.jsonl`);
|
|
148
179
|
this.ackFile = join(dir, `${id}.ack`);
|
|
149
180
|
if (!enabled)
|
|
@@ -199,7 +230,11 @@ export class BlackboxSession {
|
|
|
199
230
|
sdk_seq: this.nextSeq,
|
|
200
231
|
type,
|
|
201
232
|
...(name === undefined ? {} : { name }),
|
|
202
|
-
...(eventId
|
|
233
|
+
...(eventId !== undefined
|
|
234
|
+
? { event_id: eventId }
|
|
235
|
+
: this.serverNumbering
|
|
236
|
+
? { event_id: `evt_${randomUUID().replaceAll("-", "")}` } // a resend is harmless
|
|
237
|
+
: {}),
|
|
203
238
|
ts: new Date().toISOString(),
|
|
204
239
|
...(kept === undefined ? {} : { data: kept }),
|
|
205
240
|
...(options.attachment
|
|
@@ -672,18 +707,34 @@ export class BlackboxSession {
|
|
|
672
707
|
});
|
|
673
708
|
if (lines.length === 0)
|
|
674
709
|
return;
|
|
675
|
-
|
|
710
|
+
let events = lines.map((l) => JSON.parse(l.text));
|
|
711
|
+
// spec/sdk.md §1: lines with an event_id go unnumbered; a line without one (left in the spool
|
|
712
|
+
// by a run that numbered its own) goes numbered, as it was written. Never mixed in one batch.
|
|
713
|
+
const withId = (e) => e.event_id !== undefined;
|
|
714
|
+
const unnumbered = this.serverNumbering && withId(events[0]);
|
|
715
|
+
const cut = events.findIndex((e) => withId(e) !== withId(events[0]));
|
|
716
|
+
if (cut > 0) {
|
|
717
|
+
lines.splice(cut);
|
|
718
|
+
events = events.slice(0, cut);
|
|
719
|
+
}
|
|
720
|
+
const sent = unnumbered ? events.map(({ sdk_seq: _, ...rest }) => rest) : events;
|
|
676
721
|
const r = await post(this.options.url, `/v1/sessions/${this.id}/sdk-events`, this.token, {
|
|
677
|
-
events,
|
|
722
|
+
events: sent,
|
|
678
723
|
});
|
|
679
724
|
const next = typeof r.json.next === "number" ? r.json.next : null;
|
|
680
725
|
if (next === null)
|
|
681
726
|
throw new Rejected(r.status);
|
|
682
727
|
this.withLock(() => {
|
|
683
728
|
// Another process may have acknowledged or compacted meanwhile: then our offsets are stale.
|
|
684
|
-
|
|
685
|
-
|
|
686
|
-
|
|
729
|
+
const fresh = (this.ack.gen ?? 0) === from.gen && this.ack.offset === from.offset;
|
|
730
|
+
if (fresh && unnumbered && r.status === 200) {
|
|
731
|
+
// spec/sdk.md §1: the server numbered them; every line sent is recorded (or was)
|
|
732
|
+
const done = Math.max(...events.map((e) => e.sdk_seq)) + 1;
|
|
733
|
+
const offset = from.offset + lines.reduce((n, l) => n + l.bytes, 0);
|
|
734
|
+
this.ack = { next: done, offset, gen: from.gen };
|
|
735
|
+
writeFileSync(this.ackFile, JSON.stringify(this.ack));
|
|
736
|
+
}
|
|
737
|
+
else if (fresh && !unnumbered && next > this.ack.next) {
|
|
687
738
|
// Advance past every sent line the server now has (sdk_seq < next).
|
|
688
739
|
let offset = from.offset;
|
|
689
740
|
for (const [i, e] of events.entries())
|
package/dist/version.d.ts
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const VERSION = "0.
|
|
1
|
+
export declare const VERSION = "0.7.0";
|
package/dist/version.js
CHANGED
|
@@ -1,2 +1,2 @@
|
|
|
1
1
|
// The SDK version, on its own so the CLI can print it without loading the whole SDK.
|
|
2
|
-
export const VERSION = "0.
|
|
2
|
+
export const VERSION = "0.7.0";
|
package/package.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@zanii/blackbox",
|
|
3
3
|
"license": "Apache-2.0",
|
|
4
|
-
"version": "0.
|
|
4
|
+
"version": "0.7.0",
|
|
5
5
|
"description": "The flight recorder for AI agents: sessions, a zero-loss spool, framework hooks, approvals, and offline verification of the gateway's hash-chained record.",
|
|
6
6
|
"keywords": [
|
|
7
7
|
"ai-agents",
|