@zanii/blackbox 0.0.0-stage → 0.2.0

package/dist/directives/index.d.ts

@@ -0,0 +1,35 @@
+import { type Summary } from "../fleet/index.ts";
+export interface DirectiveItem {
+    tenant: string;
+    summary: Summary;
+    /** The models the session's calls used, sorted. */
+    models: string[];
+}
+export interface Directive {
+    id: string;
+    applies_to: {
+        model: string;
+    } | {
+        tool: string;
+    };
+    pattern: string[];
+    signature: string;
+    sessions: number;
+    type_sessions: number;
+    affected_ppm: number;
+    failure_ppm: number;
+    tenants: number;
+    first_seen: string;
+}
+export interface DirectiveOptions {
+    k?: number;
+    minSessions?: number;
+    minAffectedPpm?: number;
+}
+/** spec/directives.md §1. */
+export declare function directives(items: readonly DirectiveItem[], { k, minSessions, minAffectedPpm }?: DirectiveOptions): Directive[];
+/** spec/directives.md §2: the directives for the types one fleet flies, with its own sessions. */
+export declare function fleetDirectives(all: readonly Directive[], mine: readonly DirectiveItem[]): Array<Directive & {
+    your_sessions: string[];
+    your_type_sessions: number;
+}>;

package/dist/directives/index.js

@@ -0,0 +1,80 @@
+// Airworthiness directives (spec/directives.md): one failure pattern, on one model or tool, across
+// fleets. Pure; mirrors sdks/python/src/zanii_blackbox/directives.py; pinned by
+// spec/vectors/directives.json.
+import { createHash } from "node:crypto";
+import { signatureOf } from "../fleet/index.js";
+const FAILED = new Set(["failure", "abandoned"]);
+const typesOf = (item) => [
+    ...[...new Set(item.models)].sort().map((m) => `model:${m}`),
+    ...Object.keys(item.summary.tools)
+        .sort()
+        .map((t) => `tool:${t}`),
+];
+/** spec/directives.md §1. */
+export function directives(items, { k = 3, minSessions = 5, minAffectedPpm = 50_000 } = {}) {
+    const typeSessions = new Map();
+    const groups = new Map();
+    for (const item of items) {
+        const sig = signatureOf(item.summary);
+        for (const type of typesOf(item)) {
+            typeSessions.set(type, (typeSessions.get(type) ?? 0) + 1);
+            if (!sig)
+                continue;
+            const key = `${type}|${sig.pattern.join(",")}`;
+            const g = groups.get(key) ?? {
+                type,
+                pattern: sig.pattern,
+                signature: sig.signature,
+                tenants: new Set(),
+                sessions: 0,
+                failed: 0,
+                first: item.summary.started_at,
+            };
+            g.tenants.add(item.tenant);
+            g.sessions++;
+            if (FAILED.has(item.summary.outcome ?? ""))
+                g.failed++;
+            if (item.summary.started_at < g.first)
+                g.first = item.summary.started_at;
+            groups.set(key, g);
+        }
+    }
+    const out = [];
+    for (const [key, g] of groups) {
+        const total = typeSessions.get(g.type) ?? 0;
+        const affected = Math.floor((g.sessions * 1_000_000) / total);
+        if (g.tenants.size < k || g.sessions < minSessions || affected < minAffectedPpm)
+            continue;
+        const [kind, ...rest] = g.type.split(":");
+        const name = rest.join(":");
+        out.push({
+            id: `AD-${createHash("sha256").update(key).digest("hex").slice(0, 10)}`,
+            applies_to: kind === "model" ? { model: name } : { tool: name },
+            pattern: g.pattern,
+            signature: g.signature,
+            sessions: g.sessions,
+            type_sessions: total,
+            affected_ppm: affected,
+            failure_ppm: Math.floor((g.failed * 1_000_000) / g.sessions),
+            tenants: g.tenants.size,
+            first_seen: g.first,
+        });
+    }
+    return out.sort((a, b) => b.affected_ppm !== a.affected_ppm ? b.affected_ppm - a.affected_ppm : a.id < b.id ? -1 : 1);
+}
+/** spec/directives.md §2: the directives for the types one fleet flies, with its own sessions. */
+export function fleetDirectives(all, mine) {
+    const out = [];
+    for (const d of all) {
+        const type = "model" in d.applies_to ? `model:${d.applies_to.model}` : `tool:${d.applies_to.tool}`;
+        const flying = mine.filter((i) => typesOf(i).includes(type));
+        if (flying.length === 0)
+            continue;
+        const hit = flying
+            .filter((i) => signatureOf(i.summary)?.signature === d.signature)
+            .map((i) => i.summary.session_id)
+            .sort();
+        out.push({ ...d, your_sessions: hit, your_type_sessions: flying.length });
+    }
+    return out;
+}

package/dist/drills/index.d.ts

@@ -0,0 +1,43 @@
+export type Fault = {
+    kind: "provider_error";
+    at_call: number;
+    status: number;
+} | {
+    kind: "latency";
+    at_call: number;
+    ms: number;
+} | {
+    kind: "truncate";
+    at_call: number;
+} | {
+    kind: "tool_error";
+    at_tool_call: number;
+    message?: string;
+} | {
+    kind: "bad_data";
+    at_tool_call: number;
+    text: string;
+};
+/** spec/drills.md §1: the drill with defaults filled in, or an error message. */
+export declare function checkDrill(value: unknown): {
+    faults: Fault[];
+} | string;
+/** spec/drills.md §3. */
+export declare function drillReport(lines: readonly string[]): {
+    drill: false;
+    planned: never[];
+    injected: never[];
+    findings_after: never[];
+    recovered: boolean;
+    outcome: null;
+} | {
+    drill: true;
+    planned: Fault[];
+    injected: {
+        seq: number;
+        kind: string;
+    }[];
+    findings_after: string[];
+    recovered: boolean;
+    outcome: string | null;
+};

package/dist/drills/index.js

@@ -0,0 +1,101 @@
+// Drills (spec/drills.md): scheduled failures, checked, and what the agent did about them. Pure;
+// mirrors sdks/python/src/zanii_blackbox/drills.py; pinned by spec/vectors/drills.json.
+const MODEL_KINDS = new Set(["provider_error", "latency", "truncate"]);
+const TOOL_KINDS = new Set(["tool_error", "bad_data"]);
+const int = (v, min, max) => Number.isSafeInteger(v) && v >= min && v <= max;
+/** spec/drills.md §1: the drill with defaults filled in, or an error message. */
+export function checkDrill(value) {
+    const faults = value?.faults;
+    if (typeof value !== "object" || value === null || Array.isArray(value) || !Array.isArray(faults))
+        return "drill must be {faults: [...]}";
+    if (faults.length === 0 || faults.length > 20)
+        return "drill.faults must hold 1-20 faults";
+    const out = [];
+    for (const [i, raw] of faults.entries()) {
+        const f = raw;
+        const at = `drill.faults[${i}]`;
+        if (typeof f !== "object" || f === null)
+            return `${at} must be an object`;
+        if (MODEL_KINDS.has(f.kind)) {
+            if (!int(f.at_call, 1, 1_000_000))
+                return `${at}.at_call must be an integer of at least 1`;
+            if (f.kind === "provider_error") {
+                const status = f.status ?? 500;
+                if (!(status === 429 || int(status, 500, 599)))
+                    return `${at}.status must be 429 or 500-599`;
+                out.push({
+                    kind: "provider_error",
+                    at_call: f.at_call,
+                    status: status,
+                });
+            }
+            else if (f.kind === "latency") {
+                if (!int(f.ms, 1, 60_000))
+                    return `${at}.ms must be 1-60000`;
+                out.push({ kind: "latency", at_call: f.at_call, ms: f.ms });
+            }
+            else
+                out.push({ kind: "truncate", at_call: f.at_call });
+        }
+        else if (TOOL_KINDS.has(f.kind)) {
+            if (!int(f.at_tool_call, 1, 1_000_000))
+                return `${at}.at_tool_call must be an integer of at least 1`;
+            if (f.kind === "tool_error") {
+                if (f.message !== undefined && (typeof f.message !== "string" || f.message.length > 500))
+                    return `${at}.message must be at most 500 characters`;
+                out.push({
+                    kind: "tool_error",
+                    at_tool_call: f.at_tool_call,
+                    ...(f.message !== undefined ? { message: f.message } : {}),
+                });
+            }
+            else {
+                if (typeof f.text !== "string" || f.text.length > 2000)
+                    return `${at}.text must be a string of at most 2000 characters`;
+                out.push({ kind: "bad_data", at_tool_call: f.at_tool_call, text: f.text });
+            }
+        }
+        else
+            return `${at}.kind must be provider_error, latency, truncate, tool_error or bad_data`;
+    }
+    return { faults: out };
+}
+/** spec/drills.md §3. */
+export function drillReport(lines) {
+    const events = lines.map((l) => JSON.parse(l));
+    const drill = events[0]?.meta.drill?.faults;
+    if (events[0]?.kind !== "session.open" || !drill)
+        return {
+            drill: false,
+            planned: [],
+            injected: [],
+            findings_after: [],
+            recovered: false,
+            outcome: null,
+        };
+    const injected = events
+        .filter((e) => (e.kind === "llm.request" || e.kind === "tool.call") &&
+        typeof e.meta.injected === "object" &&
+        e.meta.injected !== null)
+        .map((e) => ({ seq: e.seq, kind: String(e.meta.injected.kind) }));
+    const first = injected[0]?.seq ?? Number.POSITIVE_INFINITY;
+    const lastInjected = Math.max(-1, ...events.filter((e) => e.meta.injected !== undefined).map((e) => e.seq));
+    const codes = [
+        ...new Set(events.filter((e) => e.kind === "finding" && e.seq > first).map((e) => String(e.meta.code))),
+    ].sort();
+    const recovered = events.some((e) => e.kind === "llm.response" &&
+        e.seq > lastInjected &&
+        e.meta.status === 200 &&
+        e.meta.injected === undefined &&
+        injected.length > 0);
+    const outcomes = events.filter((e) => e.kind === "outcome");
+    const outcome = outcomes.length ? String(outcomes.at(-1)?.meta.outcome) : null;
+    return {
+        drill: true,
+        planned: drill,
+        injected,
+        findings_after: codes,
+        recovered,
+        outcome,
+    };
+}

package/dist/duty/index.d.ts

@@ -0,0 +1,21 @@
+import { type LoadedPrices } from "../cost/index.ts";
+export interface DutyLimits {
+    max_minutes?: number;
+    max_llm_calls?: number;
+    max_context_pct?: number;
+}
+export interface DutyFinding {
+    code: "DUTY_LIMIT";
+    source: "duty";
+    severity: "warning";
+    ref: {
+        seq: number;
+        limit: "minutes" | "llm_calls" | "context_pct";
+        value: number;
+        max: number;
+    };
+}
+/** spec/duty.md §2: at most one DUTY_LIMIT per limit, where it was first crossed. */
+export declare function dutyFindings(lines: readonly string[], prices?: LoadedPrices, defaults?: DutyLimits): DutyFinding[];
+/** Checks a flight plan's `duty` (spec/duty.md §1): an error message, or undefined. */
+export declare function checkDuty(duty: unknown): string | undefined;

package/dist/duty/index.js

@@ -0,0 +1,68 @@
+// Duty-time limits (spec/duty.md): long sessions and full context windows, detected. Pure; mirrors
+// sdks/python/src/zanii_blackbox/duty.py; pinned by spec/vectors/duty.json.
+import { priceFor, promptTokens, sessionCost } from "../cost/index.js";
+const KEYS = ["max_minutes", "max_llm_calls", "max_context_pct"];
+/** spec/duty.md §1: the flight plan's limits win, limit by limit, over the server's defaults. */
+function limitsOf(open, defaults) {
+    const plan = open?.meta.flight_plan?.duty;
+    const out = { max_minutes: 0, max_llm_calls: 0, max_context_pct: 0 };
+    for (const k of KEYS) {
+        const v = plan?.[k] ?? defaults[k];
+        if (Number.isSafeInteger(v) && v > 0)
+            out[k] = v;
+    }
+    return out;
+}
+/** spec/duty.md §2: at most one DUTY_LIMIT per limit, where it was first crossed. */
+export function dutyFindings(lines, prices, defaults = {}) {
+    const events = lines.map((l) => JSON.parse(l));
+    const open = events[0]?.kind === "session.open" ? events[0] : undefined;
+    const limits = limitsOf(open, defaults);
+    const out = [];
+    const finding = (seq, limit, value, max) => out.push({
+        code: "DUTY_LIMIT",
+        source: "duty",
+        severity: "warning",
+        ref: { seq, limit, value, max },
+    });
+    if (limits.max_minutes && open) {
+        const start = Date.parse(open.ts);
+        const late = events.find((e) => Date.parse(e.ts) - start > limits.max_minutes * 60_000);
+        if (late)
+            finding(late.seq, "minutes", Math.floor((Date.parse(late.ts) - start) / 60_000), limits.max_minutes);
+    }
+    if (limits.max_llm_calls) {
+        const requests = events.filter((e) => e.kind === "llm.request");
+        const over = requests[limits.max_llm_calls];
+        if (over)
+            finding(over.seq, "llm_calls", limits.max_llm_calls + 1, limits.max_llm_calls);
+    }
+    if (limits.max_context_pct && prices) {
+        for (const c of sessionCost(lines, prices).calls) {
+            const window = c.model ? priceFor(c.model, prices.table)?.price.context_window : undefined;
+            if (!window)
+                continue;
+            const prompt = promptTokens(c.tokens);
+            const pct = Math.floor((prompt * 100) / window);
+            if (pct >= limits.max_context_pct) {
+                finding(c.seq, "context_pct", pct, limits.max_context_pct);
+                break;
+            }
+        }
+    }
+    return out;
+}
+/** Checks a flight plan's `duty` (spec/duty.md §1): an error message, or undefined. */
+export function checkDuty(duty) {
+    if (typeof duty !== "object" || duty === null || Array.isArray(duty))
+        return "flight_plan.duty must be an object";
+    for (const [k, v] of Object.entries(duty)) {
+        if (!KEYS.includes(k))
+            return `flight_plan.duty has an unknown field: ${k}`;
+        if (!Number.isSafeInteger(v) ||
+            v < 1 ||
+            (k === "max_context_pct" && v > 100))
+            return `flight_plan.duty.${k} must be a positive integer${k === "max_context_pct" ? " of at most 100" : ""}`;
+    }
+    return undefined;
+}

package/dist/fleet/index.d.ts

@@ -0,0 +1,141 @@
+import { type LoadedPrices } from "../cost/index.ts";
+import { type Bodies } from "../reconcile/record.ts";
+export interface Summary {
+    session_id: string;
+    label: string | null;
+    tenant: string | null;
+    objective: string | null;
+    started_at: string;
+    last_event_at: string;
+    closed: boolean;
+    outcome: string | null;
+    events: number;
+    model_calls: number;
+    tool_calls: number;
+    tools: Record<string, number>;
+    spend_micro_usd: number;
+    findings: {
+        advisory: number;
+        caution: number;
+        warning: number;
+    };
+    finding_codes: Record<string, number>;
+    latest_findings: Array<{
+        seq: number;
+        code: string;
+        fault: string | null;
+        severity: string;
+    }>;
+    claims: {
+        satisfied: number;
+        failed: number;
+        unverified: number;
+    };
+    blocked_by: string | null;
+    near_misses: number;
+}
+export declare function sessionSummary(lines: readonly string[], bodies: Bodies, prices: LoadedPrices): Summary;
+/** The incident report (spec/fleet.md §2) in Markdown, in English or Arabic. */
+export declare function incidentReport(lines: readonly string[], bodies: Bodies, prices: LoadedPrices, lang: "en" | "ar"): string;
+/** Per agent label (spec/fleet.md §2): outcomes, warnings, near misses, exceedances by code (FOQA),
+ * spend, cost per success. */
+export declare function logbook(summaries: readonly Summary[]): {
+    labels: {
+        label: string;
+        sessions: number;
+        outcomes: {
+            success: number;
+            failure: number;
+            abandoned: number;
+            unlabelled: number;
+        };
+        findings_warning: number;
+        near_misses: number;
+        exceedances: {
+            [k: string]: number;
+        };
+        spend_micro_usd: number;
+        cost_per_success_micro_usd: number | null;
+    }[];
+};
+export interface Collision {
+    resource: string;
+    sessions: Array<{
+        session_id: string;
+        tool: string;
+        seq: number;
+    }>;
+}
+/** Changes to the same resource from two or more sessions (A7), by resource; each session's first change. */
+export declare function collisions(records: ReadonlyArray<{
+    lines: readonly string[];
+    bodies: Bodies;
+}>): Collision[];
+export interface AirworthinessCriteria {
+    /** The newest closed sessions judged (default 20). */
+    window: number;
+    /** Fewer closed sessions than this: insufficient_data (default 5). */
+    min_sessions: number;
+    /** Successes over closed sessions, in parts per million (default 900,000 = 90%). */
+    min_success_ppm: number;
+    /** Warning findings allowed across the window (default 0). */
+    max_warnings: number;
+}
+export declare const DEFAULT_AIRWORTHINESS: AirworthinessCriteria;
+/** Is this agent (label) fit to fly? (spec/fleet.md §5). `summaries` newest first. */
+export declare function airworthiness(label: string, summaries: readonly Summary[], criteria?: AirworthinessCriteria): {
+    v: 1;
+    label: string;
+    criteria: AirworthinessCriteria;
+    sessions: string[];
+    successes: number;
+    success_ppm: number;
+    warnings: number;
+    near_misses: number;
+    verdict: string;
+    reasons: string[];
+};
+/** spec/signatures.md §1: the failure pattern and its id, or null. */
+export declare function signatureOf(summary: Summary): {
+    signature: string;
+    pattern: string[];
+} | null;
+/** spec/signatures.md §2: groups seen at k or more distinct tenants; counts only. */
+export declare function signatures(items: ReadonlyArray<{
+    tenant: string;
+    summary: Summary;
+}>, k?: number): {
+    signature: string;
+    pattern: string[];
+    tenants: number;
+    sessions: number;
+    failures: number;
+    failure_ppm: number;
+}[];
+/** The did:key of an Ed25519 identity, from its 32-byte seed. */
+export declare function didFromSeed(seed: Uint8Array): string;
+/** Signs a certificate with an Ed25519 identity (the 32-byte seed): adds `signed: {by, sig}`. */
+export declare function signCertificate<T extends Record<string, unknown>>(cert: T, did: string, seed: Uint8Array): T & {
+    signed: {
+        by: string;
+        sig: string;
+    };
+};
+/** Checks a signed certificate offline, from the signer's did:key alone. */
+export declare function verifyCertificate(cert: Record<string, unknown>, expectedDid?: string): {
+    ok: boolean;
+    by: string | null;
+    reason?: string;
+};
+export interface FindingGroup {
+    code: string;
+    fault: string | null;
+    severity: string;
+    count: number;
+    first_seq: number;
+    last_seq: number;
+    first_at: string;
+    last_at: string;
+}
+/** The record's findings grouped by code: how many, and when first and last, in first-seen order. */
+export declare function aggregateFindings(lines: readonly string[]): FindingGroup[];