@zanii/blackbox 0.18.0 → 0.19.0

package/dist/cli.js

@@ -1,5 +1,6 @@
 #!/usr/bin/env node
 // The `blackbox` command. Mirrors the Python SDK's `python -m zanii_blackbox.<command>` entry points.
+import { spawnSync } from "node:child_process";
 import { createHash } from "node:crypto";
 import { appendFileSync, existsSync, mkdirSync, readFileSync, writeFileSync } from "node:fs";
 import { join } from "node:path";
@@ -25,6 +26,8 @@ const USAGE = `usage (spec/cli.md):
   blackbox occurrence-pack <session_id> --framework <id>
   blackbox file <session_id> --framework <id> --reference <ref> [--channel <c>] [--filed-at <iso>]
   blackbox replay <session_id> [--fork] [--fork-after N] [--label L] [--drill <drill.json>]
+  blackbox scenario save <session_id> [--out <file>]
+  blackbox scenario run <scenario.json>... -- <agent command>   (1 when an incident's findings recur)
   blackbox tail <session_id> [--after-seq N]
   blackbox close <session_id> [--outcome success|failure|abandoned] [--note <text>]
   blackbox resume <session_id> [--token <resume token>] [--note <text>] [--force]   (a paused session needs no token)
@@ -94,6 +97,8 @@ async function main(argv) {
         return occurrenceCommand(rest);
     if (command === "replay")
         return replayCommand(rest);
+    if (command === "scenario")
+        return scenarioCommand(rest);
     if (command === "analyze")
         return analyzeCommand(rest);
     if (command === "verify-certificate")
@@ -405,9 +410,10 @@ class ServerError extends Error {
 /** GET from the gateway's admin API (BLACKBOX_URL + BLACKBOX_ADMIN_KEY). */
 async function admin(path, body, allow = []) {
     const url = process.env.BLACKBOX_URL;
-    const key = process.env.BLACKBOX_ADMIN_KEY;
+    // a tenant's key works too, for what a tenant may read (the scenarios in CI)
+    const key = process.env.BLACKBOX_ADMIN_KEY || process.env.BLACKBOX_API_KEY;
     if (!url || !key)
-        throw new ServerError("BLACKBOX_URL and BLACKBOX_ADMIN_KEY must be set");
+        throw new ServerError("BLACKBOX_URL and BLACKBOX_ADMIN_KEY must be set (or a tenant's BLACKBOX_API_KEY)");
     let res;
     try {
         res = await fetch(`${url.replace(/\/+$/, "")}${path}`, {
@@ -641,6 +647,81 @@ async function replayCommand(rest) {
     process.stdout.write(`${JSON.stringify(doc)}\n`);
     return 0;
 }
+/** spec/replay.md §3a: saves an incident as a regression test, or reruns the agent against each
+ * saved one and fails when any of the incident's findings comes back. */
+async function scenarioCommand(rest) {
+    const at = rest.indexOf("--");
+    const command = at === -1 ? [] : rest.slice(at + 1);
+    const { flags, positional } = parse(at === -1 ? rest : rest.slice(0, at), []);
+    const [verb, ...args] = positional;
+    if (verb === "save" && args.length === 1 && SESSION_ID.test(args[0])) {
+        const doc = await (await admin(`/v1/sessions/${args[0]}/scenario`)).json();
+        const out = flags.out || `${args[0]}.scenario.json`;
+        writeFileSync(out, `${JSON.stringify(doc)}\n`);
+        process.stdout.write(`${out}\n`);
+        return 0;
+    }
+    if (verb !== "run" || args.length === 0 || command.length === 0)
+        return usage();
+    let failed = false;
+    for (const file of args) {
+        const result = await scenarioRun(file, command);
+        process.stdout.write(`${JSON.stringify(result)}\n`);
+        failed ||= !result.ok;
+    }
+    return failed ? 1 : 0;
+}
+/** One scenario: a fork of its incident, the agent run against it (its output on stderr, so
+ * stdout stays the results), closed, and its findings compared with the incident's. */
+async function scenarioRun(file, command) {
+    let scenario;
+    try {
+        scenario = JSON.parse(readFileSync(file, "utf8"));
+    }
+    catch (error) {
+        throw new ServerError(`${file}: ${error instanceof Error ? error.message : error}`);
+    }
+    const source = scenario?.source;
+    const expect = scenario?.expect?.finding_codes;
+    if (scenario?.v !== 1 || typeof source !== "string" || !SESSION_ID.test(source))
+        throw new ServerError(`${file}: not a scenario (spec/replay.md §3)`);
+    const before = (expect && typeof expect === "object" ? expect : {});
+    const label = `regression: ${scenario.label || source}`.slice(0, 200);
+    const opened = (await (await admin("/v1/sessions", { label, replay: { of: source, fork: true } })).json());
+    const sid = String(opened.session_id);
+    const token = String(opened.token);
+    const url = String(process.env.BLACKBOX_URL).replace(/\/+$/, "");
+    const run = spawnSync(command[0], command.slice(1), {
+        stdio: ["inherit", 2, "inherit"], // the agent prints to stderr: stdout is the results
+        env: {
+            ...process.env,
+            BLACKBOX_SESSION_ID: sid,
+            BLACKBOX_TOKEN: token,
+            BLACKBOX_SCENARIO: file,
+            // the official clients read these: an agent that doesn't set its own goes through the fork
+            ANTHROPIC_BASE_URL: `${url}/anthropic`,
+            ANTHROPIC_API_KEY: token,
+            OPENAI_BASE_URL: `${url}/openai/v1`,
+            OPENAI_API_KEY: token,
+        },
+    });
+    if (run.error)
+        process.stderr.write(`blackbox: can't run ${command[0]}: ${run.error.message}\n`);
+    const code = run.error ? 127 : (run.status ?? 1);
+    await admin(`/v1/sessions/${sid}/close`, { outcome: code === 0 ? "success" : "failure" });
+    const after = (await (await admin(`/v1/sessions/${sid}/scenario`)).json());
+    const got = (after?.expect?.finding_codes ?? {});
+    const recurred = Object.fromEntries(Object.keys(before).flatMap((c) => (got[c] ? [[c, got[c]]] : [])));
+    return {
+        scenario: file,
+        source,
+        session_id: sid,
+        exit_code: code,
+        ok: code === 0 && Object.keys(recurred).length === 0,
+        recurred,
+        new: Object.fromEntries(Object.entries(got).filter(([c]) => !(c in before))),
+    };
+}
 /** Audit S12: an exported bundle, analysed offline: it must verify first, then the summary, the
  * findings it recorded, and the findings this SDK's analyses find in it now. */
 async function analyzeCommand(rest) {

package/dist/hooks/langchain-agent.d.ts

@@ -1,7 +1,8 @@
 import { type CompiledPolicy } from "../policy/index.ts";
 import { type EventSink } from "./shared.ts";
+import { type WitnessOptions, type WitnessSink } from "./witness.ts";
 /** A BlackboxSession, or anything with the same methods. */
-export interface AgentSink extends EventSink {
+export interface AgentSink extends EventSink, WitnessSink {
     state?(): Promise<{
         control?: {
             state?: string;
@@ -29,7 +30,7 @@ export declare function blackboxAgentMiddleware(session: AgentSink, options?: {
     checkBlock?: boolean;
     /** Files with a machine-managed block (idea S5): file tools may not change it. */
     protect?: readonly string[];
-}): {
+} & WitnessOptions): {
     [x: symbol]: true;
     name: string;
     beforeAgent(): Promise<undefined>;

package/dist/hooks/langchain-agent.js

@@ -11,6 +11,7 @@ import { decide } from "../policy/index.js";
 import { stableEventId } from "../session/index.js";
 import { protectedEdit } from "./protect.js";
 import { llmIds, neverThrow, providerOf } from "./shared.js";
+import { shellCommand, toolText } from "./witness.js";
 /** Thrown by `beforeAgent` when the gateway has stopped this session (spec/control.md). */
 export class BlackboxStoppedError extends Error {
     constructor(state) {
@@ -117,13 +118,28 @@ export function blackboxAgentMiddleware(session, options = {}) {
                     return (await refusal(call, `a second person's approval is needed (${answer})`));
                 }
             }
+            const receipt = async (phase) => {
+                if (!options.workspace || !key || !session.workspaceReceipt)
+                    return;
+                try {
+                    await session.workspaceReceipt(phase, key, { tool: call.name, cwd: options.workspace });
+                }
+                catch { } // a receipt never replaces the tool's own result or error
+            };
+            await receipt("pre");
             try {
                 const out = await handler(request);
+                await receipt("post");
+                const command = options.witnessShell ? shellCommand(call.args) : null;
+                const text = command ? toolText(out) : null;
+                if (command && text !== null)
+                    neverThrow(() => session.witness?.(command, text, { cwd: options.workspace }));
                 const status = out?.status;
                 emit("tool.result", call.name, { ok: status !== "error", ...(key ? { run_id: key } : {}) }, key);
                 return out;
             }
             catch (error) {
+                await receipt("post");
                 if (isControl(error)) {
                     emit("interrupt", call.name, { ...(key ? { run_id: key } : {}) }, key);
                 }

package/dist/hooks/openai-agents.d.ts

@@ -1,6 +1,10 @@
 import { type EventSink } from "./shared.ts";
+import { type WitnessOptions, type WitnessSink } from "./witness.ts";
 interface HookEmitter {
     on(event: any, listener: any): unknown;
 }
-export declare function attachBlackbox(runner: HookEmitter, session: EventSink): void;
+/** `options.workspace`: a receipt of that git workspace before and after each tool call;
+ *  `options.witnessShell`: re-run each read-only shell command and compare its output. The pre
+ *  receipt's tree is taken synchronously in the listener, so before the tool runs. */
+export declare function attachBlackbox(runner: HookEmitter, session: EventSink & WitnessSink, options?: WitnessOptions): void;
 export {};

package/dist/hooks/openai-agents.js

@@ -5,23 +5,37 @@
 // The JS SDK's hooks are events on the Runner (agent / handoff / tool); it has no model-call event,
 // so model-call ids come from the gateway alone here.
 import { neverThrow } from "./shared.js";
+import { shellCommand } from "./witness.js";
 const nameOf = (x) => String(x?.name ?? "agent").slice(0, 256);
 /** Audit K23: the SDK turns a tool's exception into this answer (its default error function), so a
  *  tool that failed isn't recorded as a success. */
 const TOOL_ERROR = "An error occurred while running the tool.";
 const failed = (result) => typeof result === "string" && result.startsWith(TOOL_ERROR);
-export function attachBlackbox(runner, session) {
+const argsOf = (details) => {
+    const raw = details?.toolCall?.arguments;
+    try {
+        return typeof raw === "string" ? JSON.parse(raw) : raw;
+    }
+    catch {
+        return raw;
+    }
+};
+/** `options.workspace`: a receipt of that git workspace before and after each tool call;
+ *  `options.witnessShell`: re-run each read-only shell command and compare its output. The pre
+ *  receipt's tree is taken synchronously in the listener, so before the tool runs. */
+export function attachBlackbox(runner, session, options = {}) {
+    const receipt = (phase, tool, details) => {
+        const callId = details?.toolCall?.callId;
+        if (options.workspace && callId && session.workspaceReceipt)
+            Promise.resolve(session.workspaceReceipt(phase, callId, { tool: nameOf(tool), cwd: options.workspace })).catch(() => { }); // never an unhandled rejection in the agent
+    };
     const on = (event, listener) => runner.on(event, ((...args) => neverThrow(() => listener(...args))));
     on("agent_start", (_context, agent) => session.event("step", nameOf(agent), {}));
     on("agent_handoff", (_context, from, to) => session.event("step", nameOf(to), { handoff_from: nameOf(from) }));
     on("agent_tool_start", (_context, agent, tool, details) => {
-        const call = details
-            ?.toolCall;
-        let args = call?.arguments;
-        try {
-            args = typeof args === "string" ? JSON.parse(args) : args;
-        }
-        catch { }
+        const call = details?.toolCall;
+        const args = argsOf(details);
+        receipt("pre", tool, details);
         session.event("tool.call", nameOf(tool), {
             agent: nameOf(agent),
             ...(call?.callId ? { call_id: call.callId } : {}),
@@ -30,11 +44,15 @@ export function attachBlackbox(runner, session) {
     });
     on("agent_tool_end", (_context, agent, tool, result, details) => {
         const callId = details?.toolCall?.callId;
+        receipt("post", tool, details);
         session.event("tool.result", nameOf(tool), {
             ok: !failed(result),
             agent: nameOf(agent),
             ...(callId ? { call_id: callId } : {}),
             ...(failed(result) ? { error: String(result).slice(0, 500) } : {}),
         });
+        const command = options.witnessShell ? shellCommand(argsOf(details)) : null;
+        if (command && typeof result === "string" && !failed(result))
+            session.witness?.(command, result, { cwd: options.workspace });
     });
 }

package/dist/hooks/witness.d.ts

@@ -0,0 +1,22 @@
+/** What a framework hook passes for the receipts and the witness. Both off unless set. */
+export interface WitnessOptions {
+    /** A directory in a git repository: a receipt of its files before and after each tool call. */
+    workspace?: string;
+    /** Re-run each read-only shell command the agent ran and compare its output (attestation.md). */
+    witnessShell?: boolean;
+}
+/** The session methods the hooks call, when the sink has them (a BlackboxSession does). */
+export interface WitnessSink {
+    workspaceReceipt?(phase: "pre" | "post", toolUseId: string, options?: {
+        tool?: string;
+        cwd?: string;
+    }): Promise<string | null>;
+    witness?(command: string, claimed: string, options?: {
+        cwd?: string | undefined;
+        exitCode?: number;
+    }): unknown;
+}
+/** The shell command a tool call ran, from its arguments (`command`, `cmd`, or `commands`). */
+export declare function shellCommand(args: unknown): string | null;
+/** A tool's answer as text: a string, or a LangChain message's content (string or text parts). */
+export declare function toolText(out: unknown): string | null;

package/dist/hooks/witness.js

@@ -0,0 +1,28 @@
+// File receipts and a shell witness from any framework's tool hooks (spec/sdk.md §5.3). The
+// session does the work (`workspaceReceipt`, `witness`); this finds what to give it in a tool call.
+// Mirrors hooks/witness.py.
+/** The shell command a tool call ran, from its arguments (`command`, `cmd`, or `commands`). */
+export function shellCommand(args) {
+    const a = (args ?? {});
+    for (const k of ["command", "cmd", "commands"]) {
+        const v = a[k];
+        if (typeof v === "string" && v.trim())
+            return v;
+        if (Array.isArray(v) && v.length === 1 && typeof v[0] === "string")
+            return v[0];
+    }
+    return null;
+}
+/** A tool's answer as text: a string, or a LangChain message's content (string or text parts). */
+export function toolText(out) {
+    if (typeof out === "string")
+        return out;
+    const content = out?.content;
+    if (typeof content === "string")
+        return content;
+    if (Array.isArray(content))
+        return content
+            .map((p) => (typeof p === "string" ? p : (p?.text ?? "")))
+            .join("");
+    return null;
+}

package/dist/session/index.d.ts

@@ -315,6 +315,25 @@ export declare class BlackboxSession {
     nearMiss(description: string): void;
     /** spec/attestation.md: re-runs a read-only command and records whether the output matches. */
     attest(argv: string[], claimed: string, cwd?: string, claimedExit?: number): Attestation | undefined;
+    /**
+     * The witness for a framework's shell tool (spec/attestation.md §2): parses `command` and, when
+     * it's one simple read-only command, re-runs it and records whether its output matches `claimed`.
+     * Anything else (a pipe, a write, a parse failure) records nothing. Never throws.
+     */
+    witness(command: string, claimed: string, options?: {
+        cwd?: string | undefined;
+        exitCode?: number;
+    }): Attestation | undefined;
+    /**
+     * A workspace receipt (spec/reconcile.md §7) from any framework's tool hook: the git tree of
+     * `cwd` before (`pre`) or after (`post`) a tool call, so a change no tool claimed is
+     * UNATTRIBUTED_CHANGE. Resolves with null when posted, else the reason; never rejects.
+     */
+    workspaceReceipt(phase: "pre" | "post", toolUseId: string, options?: {
+        tool?: string;
+        cwd?: string;
+    }): Promise<string | null>;
+    private receipts;
     /**
      * spec/sdk.md §1.1: a computer-use screenshot (PNG, JPEG or WebP, at most 2 MB), recorded as the
      * body of a `screen` event, so its hash is in the chain. `action` says what the agent did or is

package/dist/session/index.js

@@ -7,11 +7,12 @@ import { request as httpsRequest } from "node:https";
 import { tmpdir } from "node:os";
 import { join } from "node:path";
 import { argsHash } from "../approvals/index.js";
-import { attest, isReadOnly } from "../attest/index.js";
+import { attest, isReadOnly, parseCommand } from "../attest/index.js";
 import { buildPayment, ftaPayload, screeningPayload, } from "../finance/index.js";
 import { decisionPayload } from "../gov/index.js";
 import { accessPayload, checkEpisodeTag, recommendationPayload, verifyHealthSignature, } from "../health/index.js";
 import { memoryEntry } from "../memory/index.js";
+import { postReceipt, receiptFor, workspaceTree } from "../workspace-receipt/index.js";
 /** spec/preflight.md §1 (L3): with an `egress` item, the SDK checks it here, on the agent's host. */
 async function withEgress(p) {
     if (![...(p.require ?? []), ...(p.optional ?? [])].includes("egress"))
@@ -464,6 +465,44 @@ export class BlackboxSession {
         this.event("attestation", argv[0], { ...result });
         return result;
     }
+    /**
+     * The witness for a framework's shell tool (spec/attestation.md §2): parses `command` and, when
+     * it's one simple read-only command, re-runs it and records whether its output matches `claimed`.
+     * Anything else (a pipe, a write, a parse failure) records nothing. Never throws.
+     */
+    witness(command, claimed, options = {}) {
+        const argv = parseCommand(command);
+        if (!argv)
+            return undefined;
+        try {
+            return this.attest(argv, claimed, options.cwd, options.exitCode);
+        }
+        catch {
+            return undefined;
+        }
+    }
+    /**
+     * A workspace receipt (spec/reconcile.md §7) from any framework's tool hook: the git tree of
+     * `cwd` before (`pre`) or after (`post`) a tool call, so a change no tool claimed is
+     * UNATTRIBUTED_CHANGE. Resolves with null when posted, else the reason; never rejects.
+     */
+    async workspaceReceipt(phase, toolUseId, options = {}) {
+        if (!this.enabled || this.closed)
+            return "the session is not recording";
+        const receipt = receiptFor({
+            hook_event_name: phase === "pre" ? "PreToolUse" : "PostToolUse",
+            tool_use_id: toolUseId.slice(0, 256),
+            ...(options.tool ? { tool_name: options.tool } : {}),
+        }, workspaceTree(options.cwd ?? process.cwd()));
+        if (!receipt)
+            return "no tool use id";
+        // The tree is taken now; the posts go one after another, so a hook that doesn't wait (the
+        // OpenAI Agents runner's events) can't have a call's post land before its pre.
+        const posted = this.receipts.then(() => postReceipt(this.options.url, this.token, receipt));
+        this.receipts = posted;
+        return posted;
+    }
+    receipts = Promise.resolve();
     /**
      * spec/sdk.md §1.1: a computer-use screenshot (PNG, JPEG or WebP, at most 2 MB), recorded as the
      * body of a `screen` event, so its hash is in the chain. `action` says what the agent did or is

package/dist/version.d.ts

@@ -1 +1 @@
-export declare const VERSION = "0.18.0";
+export declare const VERSION = "0.19.0";

package/dist/version.js

@@ -1,2 +1,2 @@
 // The SDK version, on its own so the CLI can print it without loading the whole SDK.
-export const VERSION = "0.18.0";
+export const VERSION = "0.19.0";

package/package.json

@@ -1,7 +1,7 @@
 {
   "name": "@zanii/blackbox",
   "license": "Apache-2.0",
-  "version": "0.18.0",
+  "version": "0.19.0",
   "description": "The flight recorder for AI agents: sessions, a zero-loss spool, framework hooks, approvals, and offline verification of the gateway's hash-chained record.",
   "keywords": [
     "ai-agents",