@zanii/blackbox 0.18.0 → 0.20.0

package/dist/analysis/faults.js

@@ -232,6 +232,21 @@ export const FAULTS = {
         en: "A tool's definition changed mid-session",
         ar: "تغيّر تعريف أداة أثناء الجلسة",
     },
+    PROMPT_CHANGED: {
+        fault: "BBX-7405",
+        en: "The system prompt changed since the last run",
+        ar: "تغيّرت تعليمات النظام منذ التشغيل السابق",
+    },
+    SELF_CHANGE: {
+        fault: "BBX-7406",
+        en: "The agent changed itself",
+        ar: "غيّر الوكيل نفسه",
+    },
+    SELF_CHANGE_UNAPPROVED: {
+        fault: "BBX-7407",
+        en: "The agent gave itself more power without approval",
+        ar: "منح الوكيل نفسه صلاحيات أكبر دون موافقة",
+    },
     USER_DOWNVOTE: {
         fault: "BBX-4601",
         en: "The person served said an answer was wrong",

package/dist/analysis/index.d.ts

@@ -103,3 +103,4 @@ export declare function verifyRegression(events: Ev[]): Finding[];
 /** Stage 2 R4, judged at close: a run labelled (or claimed) a success that its own record contradicts:
  * a check's last result failed, or the last tool call failed with nothing succeeding after it. */
 export declare function falseSuccess(events: Ev[], now: number): Finding[];
+export { type Lessons, lessons, POWER_KINDS, promptChanged, SELF_CHANGE_KINDS, selfChanges, systemPromptHash, } from "./learning.ts";

package/dist/analysis/index.js

@@ -16,6 +16,7 @@ import { groundingFindings, judgeFindings } from "./grounding.js";
 import { toolHallucinations } from "./hallucination.js";
 import { injectionFindings } from "./injection.js";
 import { flightPlan } from "./landing.js";
+import { promptChanged, selfChanges } from "./learning.js";
 import { memoryFindings } from "./memory.js";
 export { detectorAccuracy, wilsonLow } from "./accuracy.js";
 export { answerClaims, answerCredential, verifyAnswerCredential, } from "./credential.js";
@@ -60,6 +61,8 @@ export function analyze(lines, options) {
         ...toolDrifted(events),
         ...secretSent(events),
         ...userDownvotes(events),
+        ...selfChanges(events),
+        ...promptChanged(events),
         ...injectionFindings(lines, options.bodies),
         ...verifyRegression(events),
         ...falseSuccess(events, now),
@@ -583,3 +586,4 @@ export function falseSuccess(events, now) {
             : "The agent claimed it was done, but its last tool call failed and nothing succeeded after it.");
     return [];
 }
+export { lessons, POWER_KINDS, promptChanged, SELF_CHANGE_KINDS, selfChanges, systemPromptHash, } from "./learning.js";

package/dist/analysis/learning.d.ts

@@ -0,0 +1,54 @@
+import type { Summary } from "../fleet/index.ts";
+import type { Ev, Finding } from "./index.ts";
+/** What a self-change may be about. Anything else is recorded as `other`. */
+export declare const SELF_CHANGE_KINDS: readonly ["prompt", "tool", "policy", "memory", "model", "other"];
+/** Changes that give the agent more power: each needs an approved approval first. */
+export declare const POWER_KINDS: readonly string[];
+/**
+ * The fingerprint of a model request's system prompt: `sha256:` + hex over the canonical JSON of
+ * every system part it carries (Anthropic `system`, OpenAI Responses `instructions`, the `system` and
+ * `developer` messages of Chat Completions or Responses `input`, Gemini `systemInstruction`), or
+ * null when it has none.
+ */
+export declare function systemPromptHash(body: unknown): string | null;
+/**
+ * spec/findings.md §1c: each self-change the agent reported (`self_change`) is SELF_CHANGE; one
+ * that gives it more power (a tool, a policy or a model) with no approval approved before it is
+ * SELF_CHANGE_UNAPPROVED.
+ */
+export declare function selfChanges(events: Ev[]): Finding[];
+/**
+ * spec/findings.md §1c: the gateway stamps a request's system prompt fingerprint and, on the
+ * session's first one, the fingerprint the previous session with this label began with. A
+ * difference is PROMPT_CHANGED: advisory when the agent declared a prompt self-change before it,
+ * caution when nothing did.
+ */
+export declare function promptChanged(events: Ev[]): Finding[];
+export interface Lessons {
+    label: string | null;
+    sessions: number;
+    outcomes: {
+        success: number;
+        failure: number;
+        abandoned: number;
+        unknown: number;
+    };
+    codes: Array<{
+        code: string;
+        title: string | null;
+        sessions: number;
+        failed_sessions: number;
+        findings: number;
+        last_seen: string;
+    }>;
+    claims_failed_sessions: number;
+    blocked_sessions: number;
+    brief: string;
+}
+/**
+ * spec/fleet.md §8: what earlier runs of one agent (its label) hit, from their summaries, newest
+ * first: outcomes, the finding codes by how many runs they were in and how many of those failed,
+ * the runs whose completion claim the record contradicted, and a short brief an agent can put in its
+ * own prompt. Codes and counts only, never content.
+ */
+export declare function lessons(summaries: readonly Summary[], label: string | null): Lessons;

package/dist/analysis/learning.js

@@ -0,0 +1,191 @@
+// The learning loop (spec/fleet.md §8, spec/findings.md §1c): what earlier runs of an agent hit,
+// for it to read before a run, and the record of any change an agent makes to itself. The agent
+// reads lessons; it never writes to the evidence. Mirrors analysis/learning.py.
+import { createHash } from "node:crypto";
+import { canonical } from "../reconcile/shared.js";
+import { FAULTS } from "./faults.js";
+/** What a self-change may be about. Anything else is recorded as `other`. */
+export const SELF_CHANGE_KINDS = ["prompt", "tool", "policy", "memory", "model", "other"];
+/** Changes that give the agent more power: each needs an approved approval first. */
+export const POWER_KINDS = ["tool", "policy", "model"];
+const isObj = (v) => typeof v === "object" && v !== null && !Array.isArray(v);
+/**
+ * The fingerprint of a model request's system prompt: `sha256:` + hex over the canonical JSON of
+ * every system part it carries (Anthropic `system`, OpenAI Responses `instructions`, the `system` and
+ * `developer` messages of Chat Completions or Responses `input`, Gemini `systemInstruction`), or
+ * null when it has none.
+ */
+export function systemPromptHash(body) {
+    if (!isObj(body))
+        return null;
+    const parts = [];
+    if (body.system !== undefined)
+        parts.push(["system", body.system]);
+    if (typeof body.instructions === "string")
+        parts.push(["instructions", body.instructions]);
+    for (const key of ["messages", "input"])
+        if (Array.isArray(body[key]))
+            for (const m of body[key])
+                if (isObj(m) && (m.role === "system" || m.role === "developer"))
+                    parts.push([m.role, m.content ?? null]);
+    if ("systemInstruction" in body || "system_instruction" in body)
+        parts.push([
+            "systemInstruction",
+            "systemInstruction" in body ? body.systemInstruction : body.system_instruction,
+        ]);
+    if (!parts.length)
+        return null;
+    return `sha256:${createHash("sha256").update(canonical(parts)).digest("hex")}`;
+}
+/**
+ * spec/findings.md §1c: each self-change the agent reported (`self_change`) is SELF_CHANGE; one
+ * that gives it more power (a tool, a policy or a model) with no approval approved before it is
+ * SELF_CHANGE_UNAPPROVED.
+ */
+export function selfChanges(events) {
+    const out = [];
+    const approved = new Map();
+    for (const e of events)
+        if (e.kind === "control" && e.meta.action === "approval" && e.meta.decision === "approve")
+            if (typeof e.meta.approval_id === "string" && !approved.has(e.meta.approval_id))
+                approved.set(e.meta.approval_id, e.seq);
+    for (const e of events) {
+        if (e.kind !== "sdk.event" || e.meta.type !== "self_change")
+            continue;
+        const d = e.data ?? {};
+        const kind = typeof d.kind === "string" && d.kind ? d.kind : "other";
+        out.push({
+            code: "SELF_CHANGE",
+            source: "detectors",
+            severity: "advisory",
+            ref: { seq: e.seq, kind },
+            detail: "The agent reported a change to itself.",
+        });
+        if (!POWER_KINDS.includes(kind))
+            continue;
+        const id = typeof d.approval_id === "string" ? d.approval_id : null;
+        const at = id === null ? undefined : approved.get(id);
+        if (at !== undefined && at < e.seq)
+            continue;
+        out.push({
+            code: "SELF_CHANGE_UNAPPROVED",
+            source: "detectors",
+            severity: "warning",
+            ref: { seq: e.seq, kind, ...(id ? { approval_id: id } : {}) },
+            detail: "The agent gave itself more power (a tool, a policy or a model) with no approval approved before it.",
+        });
+    }
+    return out;
+}
+/**
+ * spec/findings.md §1c: the gateway stamps a request's system prompt fingerprint and, on the
+ * session's first one, the fingerprint the previous session with this label began with. A
+ * difference is PROMPT_CHANGED: advisory when the agent declared a prompt self-change before it,
+ * caution when nothing did.
+ */
+export function promptChanged(events) {
+    for (const e of events) {
+        const { system_sha256: after, system_sha256_prev: before } = e.meta;
+        if (e.kind !== "llm.request" || typeof after !== "string" || typeof before !== "string")
+            continue;
+        if (after === before)
+            return [];
+        const declared = events.some((s) => s.seq < e.seq &&
+            s.kind === "sdk.event" &&
+            s.meta.type === "self_change" &&
+            s.data?.kind === "prompt");
+        return [
+            {
+                code: "PROMPT_CHANGED",
+                source: "detectors",
+                severity: declared ? "advisory" : "caution",
+                ref: { seq: e.seq, before, after, declared },
+                detail: declared
+                    ? "The system prompt differs from the one the last run of this agent began with; the agent declared the change."
+                    : "The system prompt differs from the one the last run of this agent began with, and nothing declared the change.",
+            },
+        ];
+    }
+    return [];
+}
+/** At most this many codes in a lesson: the ones seen in the most runs. */
+const TOP = 10;
+/**
+ * spec/fleet.md §8: what earlier runs of one agent (its label) hit, from their summaries, newest
+ * first: outcomes, the finding codes by how many runs they were in and how many of those failed,
+ * the runs whose completion claim the record contradicted, and a short brief an agent can put in its
+ * own prompt. Codes and counts only, never content.
+ */
+export function lessons(summaries, label) {
+    const outcomes = { success: 0, failure: 0, abandoned: 0, unknown: 0 };
+    const codes = new Map();
+    let claimsFailed = 0;
+    let blocked = 0;
+    for (const s of summaries) {
+        const o = s.outcome;
+        const failed = o === "failure";
+        if (o === "success" || o === "failure" || o === "abandoned")
+            outcomes[o]++;
+        else
+            outcomes.unknown++;
+        if ((s.claims?.failed ?? 0) > 0)
+            claimsFailed++;
+        if (s.blocked_by)
+            blocked++;
+        for (const [code, n] of Object.entries(s.finding_codes ?? {})) {
+            const c = codes.get(code) ?? { sessions: 0, failed_sessions: 0, findings: 0, last_seen: "" };
+            c.sessions++;
+            if (failed)
+                c.failed_sessions++;
+            c.findings += n;
+            if (s.started_at > c.last_seen)
+                c.last_seen = s.started_at;
+            codes.set(code, c);
+        }
+    }
+    const top = [...codes.entries()]
+        .sort((a, b) => b[1].sessions - a[1].sessions || (a[0] < b[0] ? -1 : 1))
+        .slice(0, TOP)
+        .map(([code, c]) => ({
+        code,
+        title: FAULTS[code]?.en ?? null,
+        ...c,
+    }));
+    const n = summaries.length;
+    const name = label === null ? "this agent" : `"${label}"`;
+    let brief;
+    if (n === 0)
+        brief = `No earlier runs of ${name} on record.`;
+    else {
+        const parts = [`Lessons from the last ${n} run${n === 1 ? "" : "s"} of ${name}:`];
+        const counts = [`${outcomes.success} succeeded`, `${outcomes.failure} failed`];
+        if (outcomes.abandoned)
+            counts.push(`${outcomes.abandoned} abandoned`);
+        if (outcomes.unknown)
+            counts.push(`${outcomes.unknown} without an outcome`);
+        parts.push(`${counts.join(", ")}.`);
+        if (top.length)
+            parts.push(`Problems seen: ${top
+                .slice(0, 5)
+                .map((c) => `${c.code}${c.title ? ` (${c.title.toLowerCase()})` : ""} in ${c.sessions} run${c.sessions === 1 ? "" : "s"}${c.failed_sessions ? (c.sessions === 1 ? ", which failed" : `, ${c.failed_sessions} of them failed`) : ""}`)
+                .join("; ")}.`);
+        else
+            parts.push("No problems were found in them.");
+        if (claimsFailed)
+            parts.push(`${claimsFailed} run${claimsFailed === 1 ? "" : "s"} claimed to be done and the record disagreed.`);
+        if (blocked)
+            parts.push(`${blocked} run${blocked === 1 ? " was" : "s were"} stopped by the gateway.`);
+        if (top.length || claimsFailed)
+            parts.push("Avoid repeating these.");
+        brief = parts.join(" ");
+    }
+    return {
+        label,
+        sessions: n,
+        outcomes,
+        codes: top,
+        claims_failed_sessions: claimsFailed,
+        blocked_sessions: blocked,
+        brief,
+    };
+}

package/dist/analysis/taxonomy.js

@@ -56,6 +56,8 @@ export const TAXONOMY = {
     TOOL_CHANGED: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },
     TOOL_DRIFTED: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },
     PROMPT_INJECTION: { owasp: ["ASI01"], atlas: ["AML.T0051"] },
+    PROMPT_CHANGED: { owasp: ["ASI04"] },
+    SELF_CHANGE_UNAPPROVED: { owasp: ["ASI10"] },
     SECRET_SENT: { owasp: ["ASI03"], atlas: ["AML.T0057"] },
     TOOL_OUTPUT_MISMATCH: { owasp: ["ASI04", "ASI06"], atlas: ["AML.T0110.002", "AML.T0099"] },
     TOOL_WITNESS_INVALID: { owasp: ["ASI04"], atlas: ["AML.T0110.002"] },

package/dist/cli.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 // The `blackbox` command. Mirrors the Python SDK's `python -m zanii_blackbox.<command>` entry points.
+import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -25,6 +26,8 @@ const USAGE = `usage (spec/cli.md):
   blackbox occurrence-pack <session_id> --framework <id>
   blackbox file <session_id> --framework <id> --reference <ref> [--channel <c>] [--filed-at <iso>]
   blackbox replay <session_id> [--fork] [--fork-after N] [--label L] [--drill <drill.json>]
+  blackbox scenario save <session_id> [--out <file>]
+  blackbox scenario run <scenario.json>... -- <agent command>   (1 when an incident's findings recur)
   blackbox tail <session_id> [--after-seq N]
   blackbox close <session_id> [--outcome success|failure|abandoned] [--note <text>]
   blackbox resume <session_id> [--token <resume token>] [--note <text>] [--force]   (a paused session needs no token)
@@ -94,6 +97,8 @@ async function main(argv) {
         return occurrenceCommand(rest);
     if (command === "replay")
         return replayCommand(rest);
+    if (command === "scenario")
+        return scenarioCommand(rest);
     if (command === "analyze")
         return analyzeCommand(rest);
     if (command === "verify-certificate")
@@ -405,9 +410,10 @@ class ServerError extends Error {
 /** GET from the gateway's admin API (BLACKBOX_URL + BLACKBOX_ADMIN_KEY). */
 async function admin(path, body, allow = []) {
     const url = process.env.BLACKBOX_URL;
-    const key = process.env.BLACKBOX_ADMIN_KEY;
+    // a tenant's key works too, for what a tenant may read (the scenarios in CI)
+    const key = process.env.BLACKBOX_ADMIN_KEY || process.env.BLACKBOX_API_KEY;
     if (!url || !key)
-        throw new ServerError("BLACKBOX_URL and BLACKBOX_ADMIN_KEY must be set");
+        throw new ServerError("BLACKBOX_URL and BLACKBOX_ADMIN_KEY must be set (or a tenant's BLACKBOX_API_KEY)");
     let res;
     try {
         res = await fetch(`${url.replace(/\/+$/, "")}${path}`, {
@@ -641,6 +647,81 @@ async function replayCommand(rest) {
     process.stdout.write(`${JSON.stringify(doc)}\n`);
     return 0;
 }
+/** spec/replay.md §3a: saves an incident as a regression test, or reruns the agent against each
+ * saved one and fails when any of the incident's findings comes back. */
+async function scenarioCommand(rest) {
+    const at = rest.indexOf("--");
+    const command = at === -1 ? [] : rest.slice(at + 1);
+    const { flags, positional } = parse(at === -1 ? rest : rest.slice(0, at), []);
+    const [verb, ...args] = positional;
+    if (verb === "save" && args.length === 1 && SESSION_ID.test(args[0])) {
+        const doc = await (await admin(`/v1/sessions/${args[0]}/scenario`)).json();
+        const out = flags.out || `${args[0]}.scenario.json`;
+        writeFileSync(out, `${JSON.stringify(doc)}\n`);
+        process.stdout.write(`${out}\n`);
+        return 0;
+    }
+    if (verb !== "run" || args.length === 0 || command.length === 0)
+        return usage();
+    let failed = false;
+    for (const file of args) {
+        const result = await scenarioRun(file, command);
+        process.stdout.write(`${JSON.stringify(result)}\n`);
+        failed ||= !result.ok;
+    }
+    return failed ? 1 : 0;
+}
+/** One scenario: a fork of its incident, the agent run against it (its output on stderr, so
+ * stdout stays the results), closed, and its findings compared with the incident's. */
+async function scenarioRun(file, command) {
+    let scenario;
+    try {
+        scenario = JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch (error) {
+        throw new ServerError(`${file}: ${error instanceof Error ? error.message : error}`);
+    }
+    const source = scenario?.source;
+    const expect = scenario?.expect?.finding_codes;
+    if (scenario?.v !== 1 || typeof source !== "string" || !SESSION_ID.test(source))
+        throw new ServerError(`${file}: not a scenario (spec/replay.md §3)`);
+    const before = (expect && typeof expect === "object" ? expect : {});
+    const label = `regression: ${scenario.label || source}`.slice(0, 200);
+    const opened = (await (await admin("/v1/sessions", { label, replay: { of: source, fork: true } })).json());
+    const sid = String(opened.session_id);
+    const token = String(opened.token);
+    const url = String(process.env.BLACKBOX_URL).replace(/\/+$/, "");
+    const run = spawnSync(command[0], command.slice(1), {
+        stdio: ["inherit", 2, "inherit"], // the agent prints to stderr: stdout is the results
+        env: {
+            ...process.env,
+            BLACKBOX_SESSION_ID: sid,
+            BLACKBOX_TOKEN: token,
+            BLACKBOX_SCENARIO: file,
+            // the official clients read these: an agent that doesn't set its own goes through the fork
+            ANTHROPIC_BASE_URL: `${url}/anthropic`,
+            ANTHROPIC_API_KEY: token,
+            OPENAI_BASE_URL: `${url}/openai/v1`,
+            OPENAI_API_KEY: token,
+        },
+    });
+    if (run.error)
+        process.stderr.write(`blackbox: can't run ${command[0]}: ${run.error.message}\n`);
+    const code = run.error ? 127 : (run.status ?? 1);
+    await admin(`/v1/sessions/${sid}/close`, { outcome: code === 0 ? "success" : "failure" });
+    const after = (await (await admin(`/v1/sessions/${sid}/scenario`)).json());
+    const got = (after?.expect?.finding_codes ?? {});
+    const recurred = Object.fromEntries(Object.keys(before).flatMap((c) => (got[c] ? [[c, got[c]]] : [])));
+    return {
+        scenario: file,
+        source,
+        session_id: sid,
+        exit_code: code,
+        ok: code === 0 && Object.keys(recurred).length === 0,
+        recurred,
+        new: Object.fromEntries(Object.entries(got).filter(([c]) => !(c in before))),
+    };
+}
 /** Audit S12: an exported bundle, analysed offline: it must verify first, then the summary, the
  * findings it recorded, and the findings this SDK's analyses find in it now. */
 async function analyzeCommand(rest) {

package/dist/hooks/langchain-agent.d.ts

@@ -1,7 +1,8 @@
 import { type CompiledPolicy } from "../policy/index.ts";
 import { type EventSink } from "./shared.ts";
+import { type WitnessOptions, type WitnessSink } from "./witness.ts";
 /** A BlackboxSession, or anything with the same methods. */
-export interface AgentSink extends EventSink {
+export interface AgentSink extends EventSink, WitnessSink {
     state?(): Promise<{
         control?: {
             state?: string;
@@ -29,7 +30,7 @@ export declare function blackboxAgentMiddleware(session: AgentSink, options?: {
     checkBlock?: boolean;
     /** Files with a machine-managed block (idea S5): file tools may not change it. */
     protect?: readonly string[];
-}): {
+} & WitnessOptions): {
     [x: symbol]: true;
     name: string;
     beforeAgent(): Promise<undefined>;

package/dist/hooks/langchain-agent.js

@@ -11,6 +11,7 @@ import { decide } from "../policy/index.js";
 import { stableEventId } from "../session/index.js";
 import { protectedEdit } from "./protect.js";
 import { llmIds, neverThrow, providerOf } from "./shared.js";
+import { shellCommand, toolText } from "./witness.js";
 /** Thrown by `beforeAgent` when the gateway has stopped this session (spec/control.md). */
 export class BlackboxStoppedError extends Error {
     constructor(state) {
@@ -117,13 +118,28 @@ export function blackboxAgentMiddleware(session, options = {}) {
                     return (await refusal(call, `a second person's approval is needed (${answer})`));
                 }
             }
+            const receipt = async (phase) => {
+                if (!options.workspace || !key || !session.workspaceReceipt)
+                    return;
+                try {
+                    await session.workspaceReceipt(phase, key, { tool: call.name, cwd: options.workspace });
+                }
+                catch { } // a receipt never replaces the tool's own result or error
+            };
+            await receipt("pre");
             try {
                 const out = await handler(request);
+                await receipt("post");
+                const command = options.witnessShell ? shellCommand(call.args) : null;
+                const text = command ? toolText(out) : null;
+                if (command && text !== null)
+                    neverThrow(() => session.witness?.(command, text, { cwd: options.workspace }));
                 const status = out?.status;
                 emit("tool.result", call.name, { ok: status !== "error", ...(key ? { run_id: key } : {}) }, key);
                 return out;
             }
             catch (error) {
+                await receipt("post");
                 if (isControl(error)) {
                     emit("interrupt", call.name, { ...(key ? { run_id: key } : {}) }, key);
                 }

package/dist/hooks/openai-agents.d.ts

@@ -1,6 +1,10 @@
 import { type EventSink } from "./shared.ts";
+import { type WitnessOptions, type WitnessSink } from "./witness.ts";
 interface HookEmitter {
     on(event: any, listener: any): unknown;
 }
-export declare function attachBlackbox(runner: HookEmitter, session: EventSink): void;
+/** `options.workspace`: a receipt of that git workspace before and after each tool call;
+ *  `options.witnessShell`: re-run each read-only shell command and compare its output. The pre
+ *  receipt's tree is taken synchronously in the listener, so before the tool runs. */
+export declare function attachBlackbox(runner: HookEmitter, session: EventSink & WitnessSink, options?: WitnessOptions): void;
 export {};

package/dist/hooks/openai-agents.js

@@ -5,23 +5,37 @@
 // The JS SDK's hooks are events on the Runner (agent / handoff / tool); it has no model-call event,
 // so model-call ids come from the gateway alone here.
 import { neverThrow } from "./shared.js";
+import { shellCommand } from "./witness.js";
 const nameOf = (x) => String(x?.name ?? "agent").slice(0, 256);
 /** Audit K23: the SDK turns a tool's exception into this answer (its default error function), so a
  *  tool that failed isn't recorded as a success. */
 const TOOL_ERROR = "An error occurred while running the tool.";
 const failed = (result) => typeof result === "string" && result.startsWith(TOOL_ERROR);
-export function attachBlackbox(runner, session) {
+const argsOf = (details) => {
+    const raw = details?.toolCall?.arguments;
+    try {
+        return typeof raw === "string" ? JSON.parse(raw) : raw;
+    }
+    catch {
+        return raw;
+    }
+};
+/** `options.workspace`: a receipt of that git workspace before and after each tool call;
+ *  `options.witnessShell`: re-run each read-only shell command and compare its output. The pre
+ *  receipt's tree is taken synchronously in the listener, so before the tool runs. */
+export function attachBlackbox(runner, session, options = {}) {
+    const receipt = (phase, tool, details) => {
+        const callId = details?.toolCall?.callId;
+        if (options.workspace && callId && session.workspaceReceipt)
+            Promise.resolve(session.workspaceReceipt(phase, callId, { tool: nameOf(tool), cwd: options.workspace })).catch(() => { }); // never an unhandled rejection in the agent
+    };
     const on = (event, listener) => runner.on(event, ((...args) => neverThrow(() => listener(...args))));
     on("agent_start", (_context, agent) => session.event("step", nameOf(agent), {}));
     on("agent_handoff", (_context, from, to) => session.event("step", nameOf(to), { handoff_from: nameOf(from) }));
     on("agent_tool_start", (_context, agent, tool, details) => {
-        const call = details
-            ?.toolCall;
-        let args = call?.arguments;
-        try {
-            args = typeof args === "string" ? JSON.parse(args) : args;
-        }
-        catch { }
+        const call = details?.toolCall;
+        const args = argsOf(details);
+        receipt("pre", tool, details);
         session.event("tool.call", nameOf(tool), {
             agent: nameOf(agent),
             ...(call?.callId ? { call_id: call.callId } : {}),
@@ -30,11 +44,15 @@ export function attachBlackbox(runner, session) {
     });
     on("agent_tool_end", (_context, agent, tool, result, details) => {
         const callId = details?.toolCall?.callId;
+        receipt("post", tool, details);
         session.event("tool.result", nameOf(tool), {
             ok: !failed(result),
             agent: nameOf(agent),
             ...(callId ? { call_id: callId } : {}),
             ...(failed(result) ? { error: String(result).slice(0, 500) } : {}),
         });
+        const command = options.witnessShell ? shellCommand(argsOf(details)) : null;
+        if (command && typeof result === "string" && !failed(result))
+            session.witness?.(command, result, { cwd: options.workspace });
     });
 }

package/dist/hooks/witness.d.ts

@@ -0,0 +1,22 @@
+/** What a framework hook passes for the receipts and the witness. Both off unless set. */
+export interface WitnessOptions {
+    /** A directory in a git repository: a receipt of its files before and after each tool call. */
+    workspace?: string;
+    /** Re-run each read-only shell command the agent ran and compare its output (attestation.md). */
+    witnessShell?: boolean;
+}
+/** The session methods the hooks call, when the sink has them (a BlackboxSession does). */
+export interface WitnessSink {
+    workspaceReceipt?(phase: "pre" | "post", toolUseId: string, options?: {
+        tool?: string;
+        cwd?: string;
+    }): Promise<string | null>;
+    witness?(command: string, claimed: string, options?: {
+        cwd?: string | undefined;
+        exitCode?: number;
+    }): unknown;
+}
+/** The shell command a tool call ran, from its arguments (`command`, `cmd`, or `commands`). */
+export declare function shellCommand(args: unknown): string | null;
+/** A tool's answer as text: a string, or a LangChain message's content (string or text parts). */
+export declare function toolText(out: unknown): string | null;

package/dist/hooks/witness.js

@@ -0,0 +1,28 @@
+// File receipts and a shell witness from any framework's tool hooks (spec/sdk.md §5.3). The
+// session does the work (`workspaceReceipt`, `witness`); this finds what to give it in a tool call.
+// Mirrors hooks/witness.py.
+/** The shell command a tool call ran, from its arguments (`command`, `cmd`, or `commands`). */
+export function shellCommand(args) {
+    const a = (args ?? {});
+    for (const k of ["command", "cmd", "commands"]) {
+        const v = a[k];
+        if (typeof v === "string" && v.trim())
+            return v;
+        if (Array.isArray(v) && v.length === 1 && typeof v[0] === "string")
+            return v[0];
+    }
+    return null;
+}
+/** A tool's answer as text: a string, or a LangChain message's content (string or text parts). */
+export function toolText(out) {
+    if (typeof out === "string")
+        return out;
+    const content = out?.content;
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content))
+        return content
+            .map((p) => (typeof p === "string" ? p : (p?.text ?? "")))
+            .join("");
+    return null;
+}

package/dist/index.d.ts

@@ -2,7 +2,7 @@ export { A2A_STREAMING, type AgentCardReport, a2aMethod, a2aRequestMeta, a2aResp
 export { causality, memoryXray } from "./agents/index.ts";
 export { type DetectorOptions, detectors, fingerprint } from "./analysis/detectors.ts";
 export { FAULTS, type Fault, faultOf } from "./analysis/faults.ts";
-export { type Accuracy, type AnalyzeOptions, type AnswerClaims, analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, type ClaimEntry, type ClaimVerdict, type CriterionResult, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, type Fact, type Finding as AnalysisFinding, factsIn, findingKey, type GroundedClaim, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, type Label as AccuracyLabel, type Landing, landing, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, type Outcome, type OutcomeRollup, packsFor, parseGroundingAnswer, parseJudgeAnswer, type ReferenceFact, type ReferencePack, type ReferenceVerdict, referenceContext, rollup, type Severity, schemaProblem, secretSent, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, type WasteReason, type WasteReport, waste, wilsonLow, } from "./analysis/index.ts";
+export { type Accuracy, type AnalyzeOptions, type AnswerClaims, analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, type ClaimEntry, type ClaimVerdict, type CriterionResult, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, type Fact, type Finding as AnalysisFinding, factsIn, findingKey, type GroundedClaim, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, type Label as AccuracyLabel, type Landing, landing, lessons, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, type Outcome, type OutcomeRollup, packsFor, parseGroundingAnswer, parseJudgeAnswer, promptChanged, type ReferenceFact, type ReferencePack, type ReferenceVerdict, referenceContext, rollup, type Severity, schemaProblem, secretSent, selfChanges, systemPromptHash, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, type WasteReason, type WasteReport, waste, wilsonLow, } from "./analysis/index.ts";
 export { injectionFindings, injectionRule } from "./analysis/injection.ts";
 export { MITRE_ATLAS, OWASP_ASI, TAXONOMY, type Taxonomy, taxonomyOf, } from "./analysis/taxonomy.ts";
 export { type ApprovalFinding, approvalConfirmation, approvalFindings, argsHash, verifyConfirmation, } from "./approvals/index.ts";

package/dist/index.js

@@ -4,7 +4,7 @@ export { A2A_STREAMING, a2aMethod, a2aRequestMeta, a2aResponseMeta, a2aRestMetho
 export { causality, memoryXray } from "./agents/index.js";
 export { detectors, fingerprint } from "./analysis/detectors.js";
 export { FAULTS, faultOf } from "./analysis/faults.js";
-export { analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, factsIn, findingKey, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, landing, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, packsFor, parseGroundingAnswer, parseJudgeAnswer, referenceContext, rollup, schemaProblem, secretSent, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, waste, wilsonLow, } from "./analysis/index.js";
+export { analyze, answerClaims, answerCredential, answerRisk, answersOf, answerText, canonicalNumber, checkAgainstPacks, checkFlightPlan, checkpointReplay, detectorAccuracy, dualWitness, Evidence, eventsOf, factsIn, findingKey, groundingFindings, groundingRequest, HOLD_TRIGGERS, identifiersIn, judgeFindings, judgeRequest, landing, lessons, loadReferencePacks, modelGroundingFindings, normaliseValue, OUTCOMES, packsFor, parseGroundingAnswer, parseJudgeAnswer, promptChanged, referenceContext, rollup, schemaProblem, secretSent, selfChanges, systemPromptHash, toolDrifted, toolHallucinations, userDownvotes, verifyAnswerCredential, waste, wilsonLow, } from "./analysis/index.js";
 export { injectionFindings, injectionRule } from "./analysis/injection.js";
 export { MITRE_ATLAS, OWASP_ASI, TAXONOMY, taxonomyOf, } from "./analysis/taxonomy.js";
 export { approvalConfirmation, approvalFindings, argsHash, verifyConfirmation, } from "./approvals/index.js";

package/dist/otlp/index.js

@@ -398,7 +398,8 @@ function toOtlpV2(lines) {
         ],
     };
 }
-const LLM_OPS = new Set(["chat", "text_completion", "generate_content", "embeddings"]);
+// Spring AI says `embedding` (singular) for the semconv `embeddings`
+const LLM_OPS = new Set(["chat", "text_completion", "generate_content", "embeddings", "embedding"]);
 const MAX_SPANS = 5000;
 const MAX_DATA = 60_000;
 function anyValue(v) {
@@ -433,13 +434,20 @@ function attributes(list) {
 function spanType(a) {
     const op = a["gen_ai.operation.name"];
     const kind = a["openinference.span.kind"];
+    const spring = a["spring.ai.kind"]; // Spring AI's own span kinds
     if ((typeof op === "string" && LLM_OPS.has(op)) || kind === "LLM" || kind === "EMBEDDING")
         return "llm.call";
     if (op === "execute_tool" || kind === "TOOL")
         return "tool.call";
-    if (kind === "RETRIEVER")
+    if (kind === "RETRIEVER" || spring === "vector_store")
         return "retrieval";
-    if (op === "invoke_agent" || op === "create_agent" || kind === "AGENT" || kind === "CHAIN")
+    if (op === "invoke_agent" ||
+        op === "create_agent" ||
+        op === "framework" ||
+        kind === "AGENT" ||
+        kind === "CHAIN" ||
+        spring === "chat_client" ||
+        spring === "advisor")
         return "step";
     return "otel.span";
 }

package/dist/session/index.d.ts

@@ -1,3 +1,4 @@
+import { type Lessons } from "../analysis/learning.ts";
 import { type Attestation } from "../attest/index.ts";
 import { buildPayment, type screenCounterparty } from "../finance/index.ts";
 export interface SessionOptions {
@@ -315,6 +316,25 @@ export declare class BlackboxSession {
     nearMiss(description: string): void;
     /** spec/attestation.md: re-runs a read-only command and records whether the output matches. */
     attest(argv: string[], claimed: string, cwd?: string, claimedExit?: number): Attestation | undefined;
+    /**
+     * The witness for a framework's shell tool (spec/attestation.md §2): parses `command` and, when
+     * it's one simple read-only command, re-runs it and records whether its output matches `claimed`.
+     * Anything else (a pipe, a write, a parse failure) records nothing. Never throws.
+     */
+    witness(command: string, claimed: string, options?: {
+        cwd?: string | undefined;
+        exitCode?: number;
+    }): Attestation | undefined;
+    /**
+     * A workspace receipt (spec/reconcile.md §7) from any framework's tool hook: the git tree of
+     * `cwd` before (`pre`) or after (`post`) a tool call, so a change no tool claimed is
+     * UNATTRIBUTED_CHANGE. Resolves with null when posted, else the reason; never rejects.
+     */
+    workspaceReceipt(phase: "pre" | "post", toolUseId: string, options?: {
+        tool?: string;
+        cwd?: string;
+    }): Promise<string | null>;
+    private receipts;
     /**
      * spec/sdk.md §1.1: a computer-use screenshot (PNG, JPEG or WebP, at most 2 MB), recorded as the
      * body of a `screen` event, so its hash is in the chain. `action` says what the agent did or is
@@ -349,6 +369,26 @@ export declare class BlackboxSession {
     /** Audit S17: the gateway's view of this session (blocked? findings?), or null if it can't be
      * read. Lets an agent react to its own block before its next model call. Never throws. */
     state(): Promise<SessionState | null>;
+    /**
+     * spec/fleet.md §8: what earlier runs of this agent (this session's label) hit, to read before a
+     * run: outcomes, finding codes and a `brief` to put in the agent's own prompt. Codes and counts
+     * only. The gateway records that the agent read them. `null` when it can't be asked.
+     */
+    lessons(options?: {
+        limit?: number;
+    }): Promise<Lessons | null>;
+    /**
+     * spec/findings.md §1c: the agent changed itself (its prompt, a tool, its policy, its memory, its
+     * model). Only hashes of `before` and `after` are sent, never the content. A change that gives
+     * more power (tool, policy, model) needs an approval approved first (`requestApproval`, then
+     * pass its id), or it is SELF_CHANGE_UNAPPROVED. Never throws.
+     */
+    selfChange(kind: string, options?: {
+        before?: string;
+        after?: string;
+        reason?: string;
+        approvalId?: string;
+    }): void;
     /** spec/findings.md §13.1: what in an answer (the latest, or the one at `seq`) would hold it,
      * for an app that streams the answer by itself: `{seq, risks, hold}`. `hold` says whether the
      * server would hold it. `null` when there's no answer yet or the gateway can't be asked. */

package/dist/session/index.js

@@ -6,12 +6,14 @@ import { request as httpRequest } from "node:http";
 import { request as httpsRequest } from "node:https";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
+import { SELF_CHANGE_KINDS } from "../analysis/learning.js";
 import { argsHash } from "../approvals/index.js";
-import { attest, isReadOnly } from "../attest/index.js";
+import { attest, isReadOnly, parseCommand } from "../attest/index.js";
 import { buildPayment, ftaPayload, screeningPayload, } from "../finance/index.js";
 import { decisionPayload } from "../gov/index.js";
 import { accessPayload, checkEpisodeTag, recommendationPayload, verifyHealthSignature, } from "../health/index.js";
 import { memoryEntry } from "../memory/index.js";
+import { postReceipt, receiptFor, workspaceTree } from "../workspace-receipt/index.js";
 /** spec/preflight.md §1 (L3): with an `egress` item, the SDK checks it here, on the agent's host. */
 async function withEgress(p) {
     if (![...(p.require ?? []), ...(p.optional ?? [])].includes("egress"))
@@ -464,6 +466,44 @@ export class BlackboxSession {
         this.event("attestation", argv[0], { ...result });
         return result;
     }
+    /**
+     * The witness for a framework's shell tool (spec/attestation.md §2): parses `command` and, when
+     * it's one simple read-only command, re-runs it and records whether its output matches `claimed`.
+     * Anything else (a pipe, a write, a parse failure) records nothing. Never throws.
+     */
+    witness(command, claimed, options = {}) {
+        const argv = parseCommand(command);
+        if (!argv)
+            return undefined;
+        try {
+            return this.attest(argv, claimed, options.cwd, options.exitCode);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * A workspace receipt (spec/reconcile.md §7) from any framework's tool hook: the git tree of
+     * `cwd` before (`pre`) or after (`post`) a tool call, so a change no tool claimed is
+     * UNATTRIBUTED_CHANGE. Resolves with null when posted, else the reason; never rejects.
+     */
+    async workspaceReceipt(phase, toolUseId, options = {}) {
+        if (!this.enabled || this.closed)
+            return "the session is not recording";
+        const receipt = receiptFor({
+            hook_event_name: phase === "pre" ? "PreToolUse" : "PostToolUse",
+            tool_use_id: toolUseId.slice(0, 256),
+            ...(options.tool ? { tool_name: options.tool } : {}),
+        }, workspaceTree(options.cwd ?? process.cwd()));
+        if (!receipt)
+            return "no tool use id";
+        // The tree is taken now; the posts go one after another, so a hook that doesn't wait (the
+        // OpenAI Agents runner's events) can't have a call's post land before its pre.
+        const posted = this.receipts.then(() => postReceipt(this.options.url, this.token, receipt));
+        this.receipts = posted;
+        return posted;
+    }
+    receipts = Promise.resolve();
     /**
      * spec/sdk.md §1.1: a computer-use screenshot (PNG, JPEG or WebP, at most 2 MB), recorded as the
      * body of a `screen` event, so its hash is in the chain. `action` says what the agent did or is
@@ -557,6 +597,43 @@ export class BlackboxSession {
         }
         return null;
     }
+    /**
+     * spec/fleet.md §8: what earlier runs of this agent (this session's label) hit, to read before a
+     * run: outcomes, finding codes and a `brief` to put in the agent's own prompt. Codes and counts
+     * only. The gateway records that the agent read them. `null` when it can't be asked.
+     */
+    async lessons(options = {}) {
+        if (!this.enabled)
+            return null;
+        const q = options.limit === undefined ? "" : `?limit=${Math.trunc(options.limit)}`;
+        try {
+            const r = await send("GET", this.options.url, `/v1/sessions/${this.id}/lessons${q}`, this.token);
+            if (r.status === 200)
+                return r.json;
+            this.fail("rejected", `reading the lessons: the gateway answered ${r.status}`);
+        }
+        catch (error) {
+            this.fail("network", `reading the lessons: ${message(error)}`);
+        }
+        return null;
+    }
+    /**
+     * spec/findings.md §1c: the agent changed itself (its prompt, a tool, its policy, its memory, its
+     * model). Only hashes of `before` and `after` are sent, never the content. A change that gives
+     * more power (tool, policy, model) needs an approval approved first (`requestApproval`, then
+     * pass its id), or it is SELF_CHANGE_UNAPPROVED. Never throws.
+     */
+    selfChange(kind, options = {}) {
+        const k = SELF_CHANGE_KINDS.includes(kind) ? kind : "other";
+        const sha = (text) => `sha256:${createHash("sha256").update(text).digest("hex")}`;
+        this.event("self_change", k, {
+            kind: k,
+            ...(options.before === undefined ? {} : { before_sha256: sha(options.before) }),
+            ...(options.after === undefined ? {} : { after_sha256: sha(options.after) }),
+            ...(options.reason ? { reason: options.reason.slice(0, 500) } : {}),
+            ...(options.approvalId ? { approval_id: options.approvalId.slice(0, 128) } : {}),
+        });
+    }
     /** spec/findings.md §13.1: what in an answer (the latest, or the one at `seq`) would hold it,
      * for an app that streams the answer by itself: `{seq, risks, hold}`. `hold` says whether the
      * server would hold it. `null` when there's no answer yet or the gateway can't be asked. */

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.18.0";
+export declare const VERSION = "0.20.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 // The SDK version, on its own so the CLI can print it without loading the whole SDK.
-export const VERSION = "0.18.0";
+export const VERSION = "0.20.0";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@zanii/blackbox",
   "license": "Apache-2.0",
-  "version": "0.18.0",
+  "version": "0.20.0",
   "description": "The flight recorder for AI agents: sessions, a zero-loss spool, framework hooks, approvals, and offline verification of the gateway's hash-chained record.",
   "keywords": [
     "ai-agents",