@zanii/blackbox 0.0.0-stage → 0.1.0

package/dist/precog/index.d.ts

@@ -0,0 +1,96 @@
+import type { Finding } from "../analysis/index.ts";
+import type { LoadedPrices } from "../cost/index.ts";
+import type { Bodies } from "../reconcile/record.ts";
+export interface PrecogModel {
+    version: string;
+    note?: string;
+    calls: number;
+    features: string[];
+    mean: number[];
+    std: number[];
+    weights: number[];
+    bias: number;
+    alert_ppm: number;
+}
+export declare const FEATURES: string[];
+/** The record up to and including the `calls`-th llm.response (all of it if it has fewer). */
+export declare function prefixOf(lines: readonly string[], calls: number): {
+    prefix: string[];
+    seen: number;
+};
+/** spec/precog.md §1, over a prefix. PRECOG_RISK findings are left out. */
+export declare function features(prefix: readonly string[], bodies: Bodies, prices: LoadedPrices): number[];
+export declare function predict(model: PrecogModel, x: readonly number[]): {
+    risk_ppm: number;
+    level: string;
+};
+/** The live alert (spec/precog.md §3): one PRECOG_RISK once the record reaches `calls` model calls. */
+export declare function precogFindings(lines: readonly string[], bodies: Bodies, prices: LoadedPrices, model: PrecogModel): Finding[];
+/** Batch gradient descent on the log loss, deterministic (spec/precog.md §4). */
+export declare function train(examples: ReadonlyArray<{
+    x: readonly number[];
+    y: number;
+}>, prior: PrecogModel, options?: {
+    iterations?: number;
+    rate?: number;
+    l2?: number;
+}): PrecogModel;
+/** Recall at ≤ 5% false alarms (spec/precog.md §5). */
+export declare function calibration(scores: ReadonlyArray<{
+    risk_ppm: number;
+    y: number;
+}>): {
+    sessions: number;
+    positives: number;
+    negatives: number;
+    threshold_ppm: null;
+    recall_ppm: null;
+    false_alarm_ppm: null;
+} | {
+    sessions: number;
+    positives: number;
+    negatives: number;
+    threshold_ppm: number;
+    recall_ppm: number;
+    false_alarm_ppm: number;
+};
+/** spec/precog.md §6: which side of a reproducible split a session falls on, by its id's hash. */
+export declare function splitOf(sessionId: string, testPercent: number): "train" | "test";
+export declare const TARGET: {
+    recall_ppm: number;
+    false_alarm_ppm: number;
+};
+export declare const MIN_EACH = 30;
+/** spec/precog.md §6: the model's calibration on the test side, against the roadmap target. */
+export declare function precogReport(model: PrecogModel, examples: ReadonlyArray<{
+    session_id: string;
+    x: readonly number[];
+    y: number;
+}>, testPercent: number): {
+    v: 1;
+    version: string;
+    test_percent: number;
+    in_sample: boolean;
+    train_sessions: number;
+    calibration: {
+        sessions: number;
+        positives: number;
+        negatives: number;
+        threshold_ppm: null;
+        recall_ppm: null;
+        false_alarm_ppm: null;
+    } | {
+        sessions: number;
+        positives: number;
+        negatives: number;
+        threshold_ppm: number;
+        recall_ppm: number;
+        false_alarm_ppm: number;
+    };
+    target: {
+        recall_ppm: number;
+        false_alarm_ppm: number;
+    };
+    min_each: number;
+    verdict: string;
+};

package/dist/precog/index.js

@@ -0,0 +1,167 @@
+// Precog v1 (spec/precog.md): early-trajectory features, a logistic model, training, calibration.
+// Mirrors sdks/python/src/zanii_blackbox/precog.py.
+import { createHash } from "node:crypto";
+import { sessionSummary } from "../fleet/index.js";
+export const FEATURES = [
+    "model_calls",
+    "tool_calls",
+    "distinct_tools",
+    "log_spend",
+    "warnings",
+    "cautions",
+    "near_misses",
+    "blocked",
+];
+/** The record up to and including the `calls`-th llm.response (all of it if it has fewer). */
+export function prefixOf(lines, calls) {
+    let seen = 0;
+    for (const [i, line] of lines.entries())
+        if (JSON.parse(line).kind === "llm.response" && ++seen === calls)
+            return { prefix: lines.slice(0, i + 1), seen };
+    return { prefix: [...lines], seen };
+}
+/** spec/precog.md §1, over a prefix. PRECOG_RISK findings are left out. */
+export function features(prefix, bodies, prices) {
+    const own = prefix.filter((l) => {
+        const e = JSON.parse(l);
+        return !(e.kind === "finding" && e.meta.code === "PRECOG_RISK");
+    });
+    const s = sessionSummary(own, bodies, prices);
+    return [
+        s.model_calls,
+        s.tool_calls,
+        Object.keys(s.tools).length,
+        Math.log(1 + s.spend_micro_usd),
+        s.findings.warning,
+        s.findings.caution,
+        s.near_misses,
+        s.blocked_by ? 1 : 0,
+    ];
+}
+const sigmoid = (z) => 1 / (1 + Math.exp(-Math.min(500, Math.max(-500, z))));
+export function predict(model, x) {
+    let z = model.bias;
+    for (const [i, v] of x.entries())
+        z +=
+            model.weights[i] * ((v - model.mean[i]) / model.std[i]);
+    const risk = Math.floor(sigmoid(z) * 1_000_000 + 0.5);
+    const level = risk >= model.alert_ppm ? "high" : risk >= Math.floor(model.alert_ppm / 2) ? "elevated" : "low";
+    return { risk_ppm: risk, level };
+}
+/** The live alert (spec/precog.md §3): one PRECOG_RISK once the record reaches `calls` model calls. */
+export function precogFindings(lines, bodies, prices, model) {
+    const { prefix, seen } = prefixOf(lines, model.calls);
+    if (seen < model.calls)
+        return [];
+    const p = predict(model, features(prefix, bodies, prices));
+    return p.level === "high"
+        ? [
+            {
+                code: "PRECOG_RISK",
+                source: "precog",
+                severity: "caution",
+                ref: { calls: model.calls, risk_ppm: p.risk_ppm },
+            },
+        ]
+        : [];
+}
+/** Batch gradient descent on the log loss, deterministic (spec/precog.md §4). */
+export function train(examples, prior, options = {}) {
+    const { iterations = 500, rate = 0.1, l2 = 0.01 } = options;
+    const n = examples.length;
+    const d = FEATURES.length;
+    const mean = [];
+    const std = [];
+    for (let j = 0; j < d; j++) {
+        let s = 0;
+        for (const e of examples)
+            s += e.x[j];
+        const m = n ? s / n : 0;
+        let q = 0;
+        for (const e of examples)
+            q += (e.x[j] - m) ** 2;
+        const v = n ? Math.sqrt(q / n) : 0;
+        mean.push(m);
+        std.push(v === 0 ? 1 : v);
+    }
+    const xs = examples.map((e) => e.x.map((v, j) => (v - mean[j]) / std[j]));
+    const w = new Array(d).fill(0);
+    let b = 0;
+    for (let it = 0; it < iterations && n > 0; it++) {
+        const gw = new Array(d).fill(0);
+        let gb = 0;
+        for (const [k, e] of examples.entries()) {
+            const row = xs[k];
+            let z = b;
+            for (let j = 0; j < d; j++)
+                z += w[j] * row[j];
+            const err = sigmoid(z) - e.y;
+            for (let j = 0; j < d; j++)
+                gw[j] = gw[j] + err * row[j];
+            gb += err;
+        }
+        for (let j = 0; j < d; j++)
+            w[j] = w[j] - rate * (gw[j] / n + l2 * w[j]);
+        b -= rate * (gb / n);
+    }
+    return {
+        version: `trained-${n}`,
+        calls: prior.calls,
+        features: [...FEATURES],
+        mean,
+        std,
+        weights: w,
+        bias: b,
+        alert_ppm: prior.alert_ppm,
+    };
+}
+/** Recall at ≤ 5% false alarms (spec/precog.md §5). */
+export function calibration(scores) {
+    const neg = scores
+        .filter((s) => s.y === 0)
+        .map((s) => s.risk_ppm)
+        .sort((a, b) => a - b);
+    const pos = scores.filter((s) => s.y === 1).map((s) => s.risk_ppm);
+    const base = { sessions: scores.length, positives: pos.length, negatives: neg.length };
+    if (neg.length === 0 || pos.length === 0)
+        return { ...base, threshold_ppm: null, recall_ppm: null, false_alarm_ppm: null };
+    const threshold = neg[Math.max(0, Math.ceil((95 * neg.length) / 100) - 1)];
+    const ppm = (k, of) => Math.floor((k * 1_000_000) / of);
+    return {
+        ...base,
+        threshold_ppm: threshold,
+        recall_ppm: ppm(pos.filter((v) => v > threshold).length, pos.length),
+        false_alarm_ppm: ppm(neg.filter((v) => v > threshold).length, neg.length),
+    };
+}
+/** spec/precog.md §6: which side of a reproducible split a session falls on, by its id's hash. */
+export function splitOf(sessionId, testPercent) {
+    const h = createHash("sha256").update(sessionId, "utf8").digest();
+    return h.readUInt32BE(0) % 100 < testPercent ? "test" : "train";
+}
+export const TARGET = { recall_ppm: 700_000, false_alarm_ppm: 50_000 };
+export const MIN_EACH = 30;
+/** spec/precog.md §6: the model's calibration on the test side, against the roadmap target. */
+export function precogReport(model, examples, testPercent) {
+    const test = testPercent === 0
+        ? examples
+        : examples.filter((e) => splitOf(e.session_id, testPercent) === "test");
+    const cal = calibration(test.map((e) => ({ risk_ppm: predict(model, e.x).risk_ppm, y: e.y })));
+    const verdict = cal.positives < MIN_EACH || cal.negatives < MIN_EACH
+        ? "not enough data"
+        : cal.recall_ppm >= TARGET.recall_ppm &&
+            cal.false_alarm_ppm <= TARGET.false_alarm_ppm
+            ? "meets target"
+            : "misses target";
+    return {
+        v: 1,
+        version: model.version,
+        test_percent: testPercent,
+        in_sample: testPercent === 0,
+        train_sessions: examples.length - (testPercent === 0 ? 0 : test.length),
+        calibration: cal,
+        target: TARGET,
+        min_each: MIN_EACH,
+        verdict,
+    };
+}

package/dist/precog/intervention.d.ts

@@ -0,0 +1,22 @@
+export type Outcome = "success" | "failure";
+export interface InterventionEvidence {
+    pairs: number;
+    recovered: number;
+    not_recovered: number;
+    disrupted: number;
+    kept: number;
+    r_milli: number | null;
+    d_milli: number | null;
+    /** 95 % Wilson intervals. */
+    r_ci95_milli: [number, number] | null;
+    d_ci95_milli: [number, number] | null;
+    /** Act only above this failure chance; null without evidence on both sides. */
+    break_even_milli: number | null;
+}
+/** The evidence from (source outcome, fork outcome) pairs. */
+export declare function interventionEvidence(pairs: ReadonlyArray<{
+    before: Outcome;
+    after: Outcome;
+}>): InterventionEvidence;
+/** The expected gain, in thousandths of a run, of acting on a run with this failure chance. */
+export declare function interventionGain(e: InterventionEvidence, failureMilli: number): number | null;

package/dist/precog/intervention.js

@@ -0,0 +1,44 @@
+// Intervention evidence (spec/precog.md §9, Stage 2 R6): is restarting a run worth it? From replay
+// forks with known outcomes: recovery r (a failed run's fork succeeded) and disruption d (a good
+// run's fork failed). Acting on a run with failure chance p gains p·r − (1−p)·d, so it pays only
+// when p > d / (r + d) ("The Intervention Paradox", arXiv 2602.03338). Mirrors precog/intervention.py.
+// Rates are in thousandths (integers, so the record stays exact).
+const milli = (x) => Math.round(x * 1000);
+function wilson(k, n) {
+    if (n === 0)
+        return null;
+    const z = 1.959963984540054;
+    const p = k / n;
+    const den = 1 + (z * z) / n;
+    const mid = (p + (z * z) / (2 * n)) / den;
+    const half = (z * Math.sqrt((p * (1 - p)) / n + (z * z) / (4 * n * n))) / den;
+    return [milli(Math.max(0, mid - half)), milli(Math.min(1, mid + half))];
+}
+/** The evidence from (source outcome, fork outcome) pairs. */
+export function interventionEvidence(pairs) {
+    const failed = pairs.filter((p) => p.before === "failure");
+    const good = pairs.filter((p) => p.before === "success");
+    const recovered = failed.filter((p) => p.after === "success").length;
+    const disrupted = good.filter((p) => p.after === "failure").length;
+    const r = failed.length ? recovered / failed.length : null;
+    const d = good.length ? disrupted / good.length : null;
+    return {
+        pairs: pairs.length,
+        recovered,
+        not_recovered: failed.length - recovered,
+        disrupted,
+        kept: good.length - disrupted,
+        r_milli: r === null ? null : milli(r),
+        d_milli: d === null ? null : milli(d),
+        r_ci95_milli: wilson(recovered, failed.length),
+        d_ci95_milli: wilson(disrupted, good.length),
+        break_even_milli: r === null || d === null || r + d === 0 ? null : milli(d / (r + d)),
+    };
+}
+/** The expected gain, in thousandths of a run, of acting on a run with this failure chance. */
+export function interventionGain(e, failureMilli) {
+    if (e.r_milli === null || e.d_milli === null)
+        return null;
+    const p = failureMilli / 1000;
+    return milli(p * (e.r_milli / 1000) - (1 - p) * (e.d_milli / 1000));
+}

package/dist/precog/normal.d.ts

@@ -0,0 +1,31 @@
+export interface NormalModel {
+    version: 1;
+    runs: number;
+    /** "a\tb" → how often b followed a, in the successful runs. */
+    transitions: Record<string, number>;
+    /** a → how often anything followed it. */
+    out: Record<string, number>;
+    symbols: number;
+}
+export interface NormalScore {
+    steps: number;
+    /** Mean surprise of each step, −ln p, in thousandths (integers, so the record stays exact). */
+    surprise_milli: number;
+    unseen_milli: number;
+    loop_milli: number;
+}
+/** A run's steps as symbols: each model call, and each tool by name (MCP or SDK-reported). */
+export declare function activities(lines: readonly string[]): string[];
+/** The map of the given successful runs (each a session's lines). */
+export declare function buildNormal(runs: ReadonlyArray<readonly string[]>): NormalModel;
+/** How far a run strays from the normal. Add-half smoothing; one more symbol for the unseen. */
+export declare function normalScore(model: NormalModel, lines: readonly string[]): NormalScore;
+/** The alarm threshold from the scores of held-out successful runs: flag a score above it. With
+ * probability ≥ `confidence`, at most `alpha` of normal runs are flagged (an order statistic with a
+ * Clopper-Pearson-style binomial bound). The lowest such threshold, for the most recall; null when
+ * there are too few runs for the guarantee. */
+export declare function normalThreshold(scores: readonly number[], alpha?: number, confidence?: number): {
+    threshold: number;
+    above: number;
+    needed: number;
+} | null;

package/dist/precog/normal.js

@@ -0,0 +1,89 @@
+// Each customer's normal (spec/precog.md §8, Stage 2 R5): a map of the paths a customer's own
+// successful runs take, learned without failure labels, and a score for how far a run strays from
+// it. The alarm threshold comes with a guarantee: with the chosen confidence, at most `alpha` of
+// normal runs are flagged. Mirrors precog/normal.py.
+/** A run's steps as symbols: each model call, and each tool by name (MCP or SDK-reported). */
+export function activities(lines) {
+    const out = [];
+    for (const line of lines) {
+        const e = JSON.parse(line);
+        const m = e.meta;
+        if (e.kind === "llm.request")
+            out.push("model");
+        else if (e.kind === "tool.call" && m.method === "tools/call")
+            out.push(`tool:${String(m.server ?? "")}__${String(m.tool ?? "")}`);
+        else if (e.kind === "sdk.event" && m.type === "tool.call" && typeof m.name === "string")
+            out.push(`tool:${m.name}`);
+    }
+    return out;
+}
+/** The map of the given successful runs (each a session's lines). */
+export function buildNormal(runs) {
+    const transitions = {};
+    const out = {};
+    const symbols = new Set();
+    for (const lines of runs) {
+        const acts = activities(lines);
+        acts.forEach((b, i) => {
+            const a = i === 0 ? "start" : acts[i - 1];
+            const key = `${a}\t${b}`;
+            transitions[key] = (transitions[key] ?? 0) + 1;
+            out[a] = (out[a] ?? 0) + 1;
+            symbols.add(b);
+        });
+    }
+    return { version: 1, runs: runs.length, transitions, out, symbols: symbols.size };
+}
+/** How far a run strays from the normal. Add-half smoothing; one more symbol for the unseen. */
+export function normalScore(model, lines) {
+    const acts = activities(lines);
+    const k = model.symbols + 1;
+    let surprise = 0;
+    let unseen = 0;
+    acts.forEach((b, i) => {
+        const a = i === 0 ? "start" : acts[i - 1];
+        const n = model.transitions[`${a}\t${b}`] ?? 0;
+        if (n === 0)
+            unseen++;
+        surprise += -Math.log((n + 0.5) / ((model.out[a] ?? 0) + 0.5 * k));
+    });
+    const bigrams = acts.slice(1).map((b, i) => `${acts[i]}\t${b}`);
+    const loops = bigrams.filter((g, i) => bigrams.indexOf(g) < i).length;
+    const per = (x, n) => (n ? Math.round((1000 * x) / n) : 0);
+    return {
+        steps: acts.length,
+        surprise_milli: per(surprise, acts.length),
+        unseen_milli: per(unseen, acts.length),
+        loop_milli: per(loops, bigrams.length),
+    };
+}
+/** P(X ≥ j) for X ~ Binomial(n, p), from the pmf in log space (no underflow for large n). */
+function binomialTail(n, p, j) {
+    let logPmf = n * Math.log(1 - p); // ln P(X = 0)
+    let below = 0;
+    for (let x = 0; x < j; x++) {
+        below += Math.exp(logPmf);
+        logPmf += Math.log((n - x) / (x + 1)) + Math.log(p / (1 - p));
+    }
+    return Math.max(0, 1 - below);
+}
+/** The alarm threshold from the scores of held-out successful runs: flag a score above it. With
+ * probability ≥ `confidence`, at most `alpha` of normal runs are flagged (an order statistic with a
+ * Clopper-Pearson-style binomial bound). The lowest such threshold, for the most recall; null when
+ * there are too few runs for the guarantee. */
+export function normalThreshold(scores, alpha = 0.05, confidence = 0.95) {
+    const n = scores.length;
+    const sorted = [...scores].sort((a, b) => b - a); // highest first
+    const needed = Math.ceil(Math.log(1 - confidence) / Math.log(1 - alpha));
+    // j calibration runs may score above the threshold: the largest j the guarantee allows
+    let best = -1;
+    for (let j = 0; j < n; j++) {
+        if (binomialTail(n, alpha, j + 1) >= confidence)
+            best = j;
+        else
+            break;
+    }
+    if (best < 0)
+        return null;
+    return { threshold: sorted[best], above: best, needed };
+}

package/dist/preflight/index.d.ts

@@ -0,0 +1,11 @@
+export interface PreflightFinding {
+    code: "PREFLIGHT_DEGRADED";
+    source: "preflight";
+    severity: "advisory";
+    /** `degraded`: the optional items that failed, comma-separated (a ref holds scalars only). */
+    ref: {
+        seq: number;
+        degraded: string;
+    };
+}
+export declare function preflightFindings(lines: readonly string[]): PreflightFinding[];

package/dist/preflight/index.js

@@ -0,0 +1,19 @@
+// Pre-flight (spec/preflight.md §3): a session that left with equipment missing. Pure; mirrors
+// sdks/python/src/zanii_blackbox/preflight.py; pinned by spec/vectors/preflight.json.
+export function preflightFindings(lines) {
+    const first = lines[0];
+    if (first === undefined)
+        return [];
+    const e = JSON.parse(first);
+    const degraded = e.kind === "session.open" ? e.meta.preflight?.degraded : undefined;
+    if (!Array.isArray(degraded) || degraded.length === 0)
+        return [];
+    return [
+        {
+            code: "PREFLIGHT_DEGRADED",
+            source: "preflight",
+            severity: "advisory",
+            ref: { seq: e.seq, degraded: degraded.map(String).join(",") },
+        },
+    ];
+}

package/dist/ratings/index.d.ts

@@ -0,0 +1,21 @@
+import { type LoadedPrices } from "../cost/index.ts";
+import { type AirworthinessCriteria, airworthiness, type Summary } from "../fleet/index.ts";
+export type Rating = ReturnType<typeof airworthiness> & {
+    model: string;
+};
+export interface TypeUnratedFinding {
+    code: "TYPE_UNRATED";
+    source: "type_rating";
+    severity: "advisory" | "warning";
+    ref: {
+        seq: number;
+        model: string;
+    };
+}
+/** spec/type-ratings.md §1. `items` newest first. */
+export declare function typeRatings(items: ReadonlyArray<{
+    summary: Summary;
+    models: readonly string[];
+}>, criteria?: AirworthinessCriteria): Rating[];
+/** spec/type-ratings.md §2. `severity` is "warning" when the server requires type ratings. */
+export declare function typeUnrated(lines: readonly string[], prices: LoadedPrices, ratings: readonly Rating[], severity?: "advisory" | "warning"): TypeUnratedFinding[];

package/dist/ratings/index.js

@@ -0,0 +1,48 @@
+// Type ratings (spec/type-ratings.md): airworthiness per label and model, and flying unrated.
+// Pure; mirrors sdks/python/src/zanii_blackbox/ratings.py; pinned by spec/vectors/type-ratings.json.
+import { sessionCost } from "../cost/index.js";
+import { airworthiness, DEFAULT_AIRWORTHINESS, } from "../fleet/index.js";
+/** spec/type-ratings.md §1. `items` newest first. */
+export function typeRatings(items, criteria = DEFAULT_AIRWORTHINESS) {
+    const pairs = new Map();
+    for (const i of items)
+        if (i.summary.closed)
+            for (const model of new Set(i.models)) {
+                const label = i.summary.label ?? "(no label)";
+                pairs.set(JSON.stringify([label, model]), { label, model });
+            }
+    const cmp = (a, b) => (a < b ? -1 : a > b ? 1 : 0);
+    return [...pairs.values()]
+        .sort((a, b) => cmp(a.label, b.label) || cmp(a.model, b.model))
+        .map(({ label, model }) => ({
+        ...airworthiness(label, items.filter((i) => i.models.includes(model)).map((i) => i.summary), criteria),
+        model,
+    }));
+}
+/** spec/type-ratings.md §2. `severity` is "warning" when the server requires type ratings. */
+export function typeUnrated(lines, prices, ratings, severity = "advisory") {
+    const open = lines[0]
+        ? JSON.parse(lines[0])
+        : null;
+    if (open?.kind !== "session.open")
+        return [];
+    const label = typeof open.meta.label === "string" ? open.meta.label : "(no label)";
+    const mine = ratings.filter((r) => r.label === label);
+    if (!mine.some((r) => r.verdict === "airworthy"))
+        return [];
+    const out = [];
+    const seen = new Set();
+    for (const c of sessionCost(lines, prices).calls) {
+        if (!c.model || seen.has(c.model))
+            continue;
+        seen.add(c.model);
+        if (!mine.some((r) => r.model === c.model && r.verdict === "airworthy"))
+            out.push({
+                code: "TYPE_UNRATED",
+                source: "type_rating",
+                severity,
+                ref: { seq: c.seq, model: c.model },
+            });
+    }
+    return out;
+}

package/dist/reconcile/claude-code.d.ts

@@ -0,0 +1,19 @@
+import type { Call, Receipt } from "./record.ts";
+import { type Finding, type LocalLog } from "./shared.ts";
+export declare const CLAUDE_CODE_TYPES: ReadonlySet<string>;
+export interface Outcome {
+    findings: Finding[];
+    expected: Call[];
+    matched: {
+        calls: number;
+        tool_results: number;
+    };
+    notes: string[];
+    agentSessions: string[];
+    /** When receipts are required (§7). */
+    workspace?: {
+        receipts: number;
+        changed_by: string[];
+    };
+}
+export declare function reconcileClaudeCode(calls: Call[], log: LocalLog, agentSessions: string[], allReceipts?: Receipt[], requireReceipts?: boolean): Outcome;