@zanii/blackbox 0.2.0 → 0.4.0

package/dist/otlp/index.js

@@ -1,12 +1,48 @@
 // A session as OpenTelemetry traces (spec/otlp.md): OTLP/JSON with the GenAI semantic conventions.
 // Deterministic (ids derived from the session), so both SDKs produce the same bytes. Mirrors otlp.py.
 import { createHash } from "node:crypto";
+import { taxonomyOf } from "../analysis/taxonomy.js";
 import { tokensOf } from "../cost/index.js";
 const hex = (s, n) => createHash("sha256").update(s).digest("hex").slice(0, n);
 const nanos = (ts) => `${Date.parse(ts)}000000`;
 const str = (key, v) => typeof v === "string" ? [{ key, value: { stringValue: v } }] : [];
 const int = (key, v) => typeof v === "number" && Number.isSafeInteger(v) ? [{ key, value: { intValue: String(v) } }] : [];
-export function toOtlp(lines) {
+/**
+ * spec/otlp.md: a session as OTLP/JSON. `semconv: 2` (the default) follows the GenAI semantic
+ * conventions as of v1.42 (`gen_ai.provider.name`, `invoke_agent`, MCP `tools/call`); `semconv: 1`
+ * is the first version's output, pinned by spec/vectors/otlp.json.
+ */
+export function toOtlp(lines, options = {}) {
+    return linkCallers(options.semconv === 1 ? toOtlpV1(lines) : toOtlpV2(lines), lines);
+}
+const TRACEPARENT = /^00-([0-9a-f]{32})-([0-9a-f]{16})-[0-9a-f]{2}$/;
+/** spec/otlp.md §3: a call that carried the caller's W3C `traceparent` links to the caller's span. */
+function linkCallers(out, lines) {
+    const links = new Map();
+    let session = "";
+    for (const l of lines) {
+        const e = JSON.parse(l);
+        session ||= e.session_id;
+        const tp = e.meta.headers?.traceparent;
+        const m = typeof tp === "string" ? TRACEPARENT.exec(tp) : null;
+        if (m && !/^0+$/.test(m[1]) && !/^0+$/.test(m[2]))
+            links.set(hex(`${session}:${e.seq}`, 16), {
+                traceId: m[1],
+                spanId: m[2],
+            });
+    }
+    if (links.size === 0)
+        return out;
+    for (const rs of out.resourceSpans)
+        for (const ss of rs.scopeSpans)
+            for (const sp of ss.spans) {
+                const link = links.get(sp.spanId);
+                if (link)
+                    sp.links = [link];
+            }
+    return out;
+}
+function toOtlpV1(lines) {
     const events = lines.map((l) => JSON.parse(l));
     const first = events[0];
     if (!first)
@@ -141,3 +177,224 @@ export function toOtlp(lines) {
         ],
     };
 }
+// ---------------------------------------------------------------- semconv 2 (GenAI v1.42)
+/** Provider kinds as `gen_ai.provider.name` well-known values; any other provider id as it is. */
+const PROVIDER_NAMES = {
+    anthropic: "anthropic",
+    openai: "openai",
+    "azure-openai": "azure.ai.openai",
+    gemini: "gcp.gemini",
+    vertex: "gcp.vertex_ai",
+    bedrock: "aws.bedrock",
+};
+const list = (key, vs) => vs.length
+    ? [{ key, value: { arrayValue: { values: vs.map((v) => ({ stringValue: v })) } } }]
+    : [];
+const strs = (key, v) => typeof v === "string" ? [{ key, value: { arrayValue: { values: [{ stringValue: v }] } } }] : [];
+const failure = (type, message) => ({
+    attrs: str("error.type", type),
+    status: { status: { code: 2, message } },
+});
+function toOtlpV2(lines) {
+    const events = lines.map((l) => JSON.parse(l));
+    const first = events[0];
+    if (!first)
+        return { resourceSpans: [] };
+    const session = first.session_id;
+    const traceId = hex(session, 32);
+    const spanId = (seq) => hex(`${session}:${seq}`, 16);
+    const root = spanId(0);
+    const last = events[events.length - 1];
+    const label = typeof first.meta.label === "string" ? first.meta.label : undefined;
+    const conversation = str("gen_ai.conversation.id", session);
+    const spans = [];
+    // The session is the agent's invocation, with the findings as its events.
+    spans.push({
+        traceId,
+        spanId: root,
+        name: label ? `invoke_agent ${label}` : "invoke_agent",
+        kind: 1,
+        startTimeUnixNano: nanos(first.ts),
+        endTimeUnixNano: nanos(last.ts),
+        attributes: [
+            ...str("gen_ai.operation.name", "invoke_agent"),
+            ...str("gen_ai.agent.name", label),
+            ...conversation,
+            ...str("blackbox.session_id", session),
+            ...str("blackbox.mode", first.meta.mode),
+            ...str("blackbox.tenant", first.meta.tenant),
+            ...int("blackbox.events", events.length),
+        ],
+        events: events
+            .filter((e) => e.kind === "finding")
+            .map((e) => ({
+            timeUnixNano: nanos(e.ts),
+            name: String(e.meta.code),
+            attributes: [
+                ...str("blackbox.severity", e.meta.severity),
+                ...str("blackbox.fault", e.meta.fault),
+                ...list("blackbox.owasp_asi", taxonomyOf(String(e.meta.code)).owasp.map((o) => o.id)),
+                ...list("blackbox.mitre_atlas", taxonomyOf(String(e.meta.code)).atlas.map((a) => a.id)),
+                ...int("blackbox.seq", e.seq),
+            ],
+        })),
+    });
+    // Model calls: CLIENT spans named `chat {model}`.
+    const ends = new Map();
+    for (const e of events)
+        if ((e.kind === "llm.response" || e.kind === "llm.incomplete") &&
+            typeof e.meta.request_seq === "number")
+            ends.set(e.meta.request_seq, e);
+    for (const e of events.filter((x) => x.kind === "llm.request")) {
+        const end = ends.get(e.seq);
+        const usage = end?.meta.usage && typeof end.meta.usage === "object"
+            ? tokensOf(end.meta.usage)
+            : null;
+        const status = end?.meta.status;
+        const model = typeof end?.meta.model === "string" ? end.meta.model : undefined;
+        const provider = typeof e.meta.provider === "string" ? e.meta.provider : undefined;
+        const reason = String(end?.meta.reason ?? "incomplete");
+        const fail = !end
+            ? failure("incomplete", "no response recorded")
+            : end.kind === "llm.incomplete"
+                ? failure(reason, reason)
+                : typeof status === "number" && status >= 400
+                    ? failure(String(status), `status ${status}`)
+                    : null;
+        spans.push({
+            traceId,
+            spanId: spanId(e.seq),
+            parentSpanId: root,
+            name: model ? `chat ${model}` : "chat",
+            kind: 3,
+            startTimeUnixNano: nanos(e.ts),
+            endTimeUnixNano: nanos((end ?? e).ts),
+            attributes: [
+                ...str("gen_ai.operation.name", "chat"),
+                ...str("gen_ai.provider.name", provider ? (PROVIDER_NAMES[provider] ?? provider) : undefined),
+                ...str("gen_ai.response.model", model),
+                ...str("gen_ai.response.id", end?.meta.message_id ?? end?.meta.response_id),
+                ...strs("gen_ai.response.finish_reasons", end?.meta.stop_reason ?? end?.meta.finish_reason),
+                ...(usage
+                    ? [
+                        ...int("gen_ai.usage.input_tokens", usage.input + usage.cache_read + usage.cache_write),
+                        ...int("gen_ai.usage.output_tokens", usage.output),
+                        ...(usage.cache_read
+                            ? int("gen_ai.usage.cache_read.input_tokens", usage.cache_read)
+                            : []),
+                        ...(usage.cache_write
+                            ? int("gen_ai.usage.cache_write.input_tokens", usage.cache_write)
+                            : []),
+                    ]
+                    : []),
+                ...conversation,
+                ...str("blackbox.provider", provider),
+                ...str("blackbox.region", e.meta.region),
+                ...int("http.response.status_code", status),
+                ...(fail ? fail.attrs : []),
+                ...int("blackbox.seq", e.seq),
+            ],
+            ...(fail ? fail.status : {}),
+        });
+    }
+    // MCP and broker calls, closed by their result (call_seq). MCP spans follow the MCP conventions.
+    const results = new Map();
+    for (const e of events)
+        if (e.kind === "tool.result" && typeof e.meta.call_seq === "number")
+            results.set(e.meta.call_seq, e);
+    for (const e of events.filter((x) => x.kind === "tool.call")) {
+        const end = results.get(e.seq);
+        const broker = e.meta.via === "broker";
+        const method = typeof e.meta.method === "string" ? e.meta.method : undefined;
+        const tool = typeof e.meta.tool === "string" ? e.meta.tool : undefined;
+        const isCall = broker || method === "tools/call";
+        const fail = e.meta.policy !== undefined
+            ? failure("policy_denied", "denied by policy")
+            : end?.meta.rpc_error !== undefined
+                ? failure(String(end.meta.rpc_error), "rpc error")
+                : end?.meta.is_error === true
+                    ? failure("tool_error", "tool error")
+                    : broker && typeof end?.meta.status === "number" && end.meta.status >= 400
+                        ? failure(String(end.meta.status), `status ${end.meta.status}`)
+                        : null;
+        const rpcId = e.meta.rpc_id;
+        spans.push({
+            traceId,
+            spanId: spanId(e.seq),
+            parentSpanId: root,
+            name: broker
+                ? `execute_tool ${tool ?? "api"}`
+                : `${method ?? "mcp"}${tool && isCall ? ` ${tool}` : ""}`,
+            kind: 3,
+            startTimeUnixNano: nanos(e.ts),
+            endTimeUnixNano: nanos((end ?? e).ts),
+            attributes: [
+                ...(isCall ? str("gen_ai.operation.name", "execute_tool") : []),
+                ...str("gen_ai.tool.name", tool),
+                ...(broker ? str("gen_ai.tool.type", "extension") : []),
+                ...(broker ? [] : str("mcp.method.name", method)),
+                ...(broker ? [] : str("mcp.protocol.version", e.meta.protocol_version)),
+                ...(typeof rpcId === "string" || typeof rpcId === "number"
+                    ? str("jsonrpc.request.id", String(rpcId))
+                    : []),
+                ...(broker ? str("http.request.method", e.meta.method) : []),
+                ...(broker ? str("url.path", e.meta.path) : []),
+                ...str("blackbox.mcp_server", e.meta.server),
+                ...int("rpc.response.status_code", end?.meta.rpc_error),
+                ...conversation,
+                ...(fail ? fail.attrs : []),
+                ...int("blackbox.seq", e.seq),
+            ],
+            ...(fail ? fail.status : {}),
+        });
+    }
+    // The agent's own tools (SDK events): INTERNAL spans, from a call to the next result of its name.
+    const open = new Map();
+    const sdk = (e, type) => e.kind === "sdk.event" && e.meta.type === type;
+    for (const e of events) {
+        const name = typeof e.meta.name === "string" ? e.meta.name : undefined;
+        if (!name)
+            continue;
+        if (sdk(e, "tool.call"))
+            open.set(name, [...(open.get(name) ?? []), e]);
+        if (!sdk(e, "tool.result"))
+            continue;
+        const call = open.get(name)?.shift();
+        spans.push({
+            traceId,
+            spanId: spanId(e.seq),
+            parentSpanId: root,
+            name: `execute_tool ${name}`,
+            kind: 1,
+            startTimeUnixNano: nanos((call ?? e).ts),
+            endTimeUnixNano: nanos(e.ts),
+            attributes: [
+                ...str("gen_ai.operation.name", "execute_tool"),
+                ...str("gen_ai.tool.name", name),
+                ...str("gen_ai.tool.type", "function"),
+                ...conversation,
+                ...str("blackbox.reported_by", "agent"),
+                ...int("blackbox.seq", e.seq),
+            ],
+        });
+    }
+    return {
+        resourceSpans: [
+            {
+                resource: {
+                    attributes: [
+                        ...str("service.name", "zanii-blackbox"),
+                        ...str("blackbox.session_id", session),
+                    ],
+                },
+                scopeSpans: [
+                    {
+                        scope: { name: "zanii-blackbox", version: "2" },
+                        schemaUrl: "https://opentelemetry.io/schemas/1.44.0",
+                        spans,
+                    },
+                ],
+            },
+        ],
+    };
+}

package/dist/packs/index.js

@@ -16,6 +16,13 @@ const FACTS = new Set([
     "resumes_with_note",
     "outcomes",
     "data_region",
+    // spec/compliance.md §1a (H6)
+    "model_responses",
+    "hallucination_findings",
+    "grounding_checks",
+    "judge_checks",
+    "hallucination_controls",
+    "detector_accuracy",
 ]);
 const FILLS = new Set(["record", "findings", "redactions", "operator"]);
 const PACK_ID = /^[a-z0-9-]{1,64}$/;
@@ -63,8 +70,8 @@ function checkPart(part, kind) {
         if (!isObj(fw) || !text(fw.title, 500))
             return `${at}.title must be 1-500 characters`;
         if (kind === "compliance") {
-            if (!only(fw, ["title", "sections"]))
-                return `${at} may only have title and sections`;
+            if (!only(fw, ["title", "sections", "ar"]))
+                return `${at} may only have title, sections and ar`;
             const bad = checkSections(fw.sections, 50, at, (s, sat) => {
                 if (!only(s, ["id", "title", "requirement", "facts"]))
                     return `${sat} may only have id, title, requirement, facts`;
@@ -79,10 +86,15 @@ function checkPart(part, kind) {
             });
             if (bad)
                 return bad;
+            if (fw.ar !== undefined) {
+                const badAr = checkComplianceArabic(fw.ar, fw, `${at}.ar`);
+                if (badAr)
+                    return badAr;
+            }
         }
         else {
-            if (!only(fw, ["title", "authority", "deadlines", "sections", "ar"]))
-                return `${at} may only have title, authority, deadlines, sections`;
+            if (!only(fw, ["title", "authority", "deadlines", "clock_hours", "sections", "ar"]))
+                return `${at} may only have title, authority, deadlines, clock_hours, sections`;
             if (!text(fw.authority, 500))
                 return `${at}.authority must be 1-500 characters`;
             if (!Array.isArray(fw.deadlines) ||
@@ -90,6 +102,13 @@ function checkPart(part, kind) {
                 fw.deadlines.length > 10 ||
                 !fw.deadlines.every((d) => text(d, 500)))
                 return `${at}.deadlines must hold 1-10 texts of 1-500 characters`;
+            // spec/packs.md §1: each deadline's hours after awareness, or null when it runs from another event
+            if (fw.clock_hours !== undefined &&
+                !(Array.isArray(fw.clock_hours) &&
+                    fw.clock_hours.length === fw.deadlines.length &&
+                    fw.clock_hours.every((h) => h === null ||
+                        (Number.isSafeInteger(h) && h >= 1 && h <= 8760))))
+                return `${at}.clock_hours must give each deadline 1-8760 hours, or null`;
             const bad = checkSections(fw.sections, 30, at, (s, sat) => {
                 if (!only(s, ["id", "title", "fill"]))
                     return `${sat} may only have id, title, fill`;
@@ -108,6 +127,27 @@ function checkPart(part, kind) {
     }
     return undefined;
 }
+/** spec/packs.md §1: a compliance framework's Arabic: its title, and each section's title and
+ * requirement, by id. */
+function checkComplianceArabic(ar, fw, at) {
+    if (!isObj(ar) || !only(ar, ["title", "sections"]))
+        return `${at} must be {title, sections}`;
+    if (!text(ar.title, 500))
+        return `${at}.title must be 1-500 characters`;
+    const ids = fw.sections.map((s) => s.id);
+    const secs = ar.sections;
+    if (!isObj(secs) ||
+        Object.keys(secs).length !== ids.length ||
+        !ids.every((id) => {
+            const s = secs[id];
+            return (isObj(s) &&
+                only(s, ["title", "requirement"]) &&
+                text(s.title, 500) &&
+                text(s.requirement, 2000));
+        }))
+        return `${at}.sections must translate each section's title and requirement, by id`;
+    return undefined;
+}
 /** spec/packs.md §1: an occurrence framework's Arabic: the same deadlines and sections, translated. */
 function checkArabic(ar, fw, at) {
     if (!isObj(ar) || !only(ar, ["title", "authority", "deadlines", "sections"]))

package/dist/policy/delta.js

@@ -3,7 +3,13 @@
 // in doubt, it adds power. Mirrors policy/delta.py.
 /** How much a rule holds back: an enforced allow opens (0), an audit rule does nothing (1), a hold
  * (2) and a deny (3) close. */
-const rank = (r) => r.audit === true ? 1 : r.action === "allow" ? 0 : r.action === "require_approval" ? 2 : 3;
+const rank = (r) => r.audit === true || r.action === "require_proof"
+    ? 1
+    : r.action === "allow"
+        ? 0
+        : r.action === "require_approval"
+            ? 2
+            : 3;
 const matcher = (r) => JSON.stringify([r.tool, r.args_match ?? null, r.ignore_case === true]);
 export function policyDelta(current, proposed) {
     const before = new Map(current.rules.map((r) => [r.id, r]));

package/dist/policy/index.d.ts

@@ -6,12 +6,42 @@ export interface Rule {
     tool: string;
     args_match?: string;
     ignore_case?: boolean;
-    /** spec/approvals.md: `require_approval` holds the call for a second person. */
-    action: "deny" | "allow" | "require_approval";
+    /** spec/approvals.md: `require_approval` holds the call for a second person; spec/policy.md
+     * §9: `require_proof` lets it run, and its result must prove itself. */
+    action: "deny" | "allow" | "require_approval" | "require_proof";
+    /** spec/policy.md §9: with `require_proof`, the system's own id in the result (`refund_id`). */
+    field?: string;
     reason?: string;
     /** N3 (idea C5): record-only. Never decides a call; what it would do is recorded (POLICY_AUDIT). */
     audit?: boolean;
+    /** spec/policy.md §1: MCP tool annotations the call must have (from the server's `tools/list`). */
+    hints?: Partial<ToolHints>;
+    /** spec/policy.md §1: the session environments the rule holds in (a session's `environment`). */
+    environment?: string[];
 }
+/** spec/api.md: a session's environment, e.g. `production`, `staging`, `dev`. */
+export declare const ENVIRONMENT: RegExp;
+/** MCP tool annotations as booleans, with the spec's defaults for what a server leaves out. */
+export interface ToolHints {
+    read_only: boolean;
+    destructive: boolean;
+    idempotent: boolean;
+    open_world: boolean;
+}
+/**
+ * spec/policy.md §1: a tool's annotations (MCP `readOnlyHint`, `destructiveHint`, `idempotentHint`,
+ * `openWorldHint`) as hints. Missing ones take MCP's defaults: not read-only, destructive, not
+ * idempotent, open-world; destructive and idempotent only mean something for a tool that writes.
+ * An unknown tool gets the defaults too. Hints are the server's word, not proof.
+ */
+export declare function mcpToolHints(annotations?: unknown): ToolHints;
+/**
+ * spec/policy.md §1a: a tool definition's identity, `sha256:<hex>` of the canonical JSON of its
+ * name, title, description, schemas and annotations (the fields present). A changed description
+ * or schema changes it: what the agent was shown is no longer what was approved. `null` for
+ * something that isn't a named tool.
+ */
+export declare function toolDefinitionHash(tool: unknown): string | null;
 export interface Policy {
     version: 1;
     rules: Rule[];
@@ -20,7 +50,7 @@ export interface Policy {
 }
 export interface Decision {
     rule: string;
-    action: "deny" | "allow" | "require_approval";
+    action: "deny" | "allow" | "require_approval" | "require_proof";
     reason?: string;
 }
 interface Compiled {
@@ -31,11 +61,15 @@ interface Compiled {
 /** Parses a policy file; throws on anything invalid (the server refuses to start). */
 export declare function loadPolicy(bytes: Uint8Array): Policy;
 export declare function compilePolicy(policy: Policy): Compiled[];
-/** The first rule that matches this call, or null (allowed). `names`: the tool's names (an MCP tool has two). */
-export declare function decide(compiled: readonly Compiled[], names: readonly string[], args: unknown): Decision | null;
+/** spec/api.md: the session's environment, from its `session.open` (the first line). */
+export declare function environmentOf(lines: readonly string[]): string | undefined;
+/** The first rule that matches this call, or null (allowed). `names`: the tool's names (an MCP tool
+ * has two). `hints`: an MCP tool's annotations; a rule with `hints` never matches a call without.
+ * `environment`: the session's; a rule with `environment` never matches a session without one. */
+export declare function decide(compiled: readonly Compiled[], names: readonly string[], args: unknown, hints?: ToolHints, environment?: string): Decision | null;
 export type CompiledPolicy = ReturnType<typeof compilePolicy>;
 /** N3 (idea C5): every audit rule this call matches, and what it would do if it were enforced. */
-export declare function audited(compiled: readonly Compiled[], names: readonly string[], args: unknown): Array<{
+export declare function audited(compiled: readonly Compiled[], names: readonly string[], args: unknown, hints?: ToolHints, environment?: string): Array<{
     rule: string;
     would: Rule["action"];
 }>;