@zanii/blackbox 0.21.2 → 0.23.0

package/dist/cost/index.js

@@ -1,6 +1,7 @@
 // The cost engine (spec/cost.md): integer micro-USD from a versioned price table, AED display.
 // Mirrors sdks/python/src/zanii_blackbox/cost.py.
 import { createHash } from "node:crypto";
+import { envelopesOf } from "../reconcile/record.js";
 const CLASSES = ["input", "output", "cache_read", "cache_write"];
 const EXTRA = [
     "cache_write_1h",
@@ -47,6 +48,9 @@ export function loadPrices(bytes) {
                     whole(model, `service_tiers.${name}.${c}`, t[c]);
         if (p.status !== undefined && p.status !== "deprecated" && p.status !== "beta")
             throw new Error(`price table: ${model}.status must be deprecated or beta`);
+        if (p.batch_percent !== undefined &&
+            !(Number.isSafeInteger(p.batch_percent) && p.batch_percent >= 1 && p.batch_percent <= 100))
+            throw new Error(`price table: ${model}.batch_percent must be an integer 1-100`);
     }
     if (table.uplifts !== undefined) {
         if (!Array.isArray(table.uplifts))
@@ -85,7 +89,8 @@ provider) {
     return plain !== model ? lookup(plain, table) : null;
 }
 const GEO = /^(?:us|eu|apac|au|ca|jp|global|us-gov)\./;
-const VENDOR = /^(?:anthropic|amazon|meta|mistral|cohere|ai21|deepseek|openai|qwen|writer|google)\./;
+// `google/`: Vertex's OpenAI-compatible route (spec/cost.md §1)
+const VENDOR = /^(?:(?:anthropic|amazon|meta|mistral|cohere|ai21|deepseek|openai|qwen|writer|google)\.|google\/)/;
 function lookup(model, table) {
     const exact = table.models[model];
     if (exact)
@@ -96,6 +101,12 @@ function lookup(model, table) {
             best = key;
     return best === null ? null : { key: best, price: table.models[best] };
 }
+const SONIC = [
+    "input_speechTokens",
+    "input_textTokens",
+    "output_speechTokens",
+    "output_textTokens",
+];
 /** Token classes from a recorded, flattened `meta.usage` (spec/cost.md §2). */
 export function tokensOf(usage) {
     const n = (k) => {
@@ -105,14 +116,16 @@ export function tokensOf(usage) {
     if ("promptTokenCount" in usage) {
         // Gemini (spec/providers.md §5): the prompt count includes the cached part; thoughts are output.
         const cached = n("cachedContentTokenCount");
+        // Gemini Live names its output `responseTokenCount` (spec/providers.md §7)
         const audioIn = n("promptTokensDetails_AUDIO");
-        const audioOut = n("candidatesTokensDetails_AUDIO");
+        const audioOut = n("candidatesTokensDetails_AUDIO") + n("responseTokensDetails_AUDIO");
         const imageIn = n("promptTokensDetails_IMAGE");
-        const imageOut = n("candidatesTokensDetails_IMAGE");
+        const imageOut = n("candidatesTokensDetails_IMAGE") + n("responseTokensDetails_IMAGE");
         return extras({
             input: Math.max(0, n("promptTokenCount") - cached - audioIn - imageIn) +
                 n("toolUsePromptTokenCount"),
-            output: Math.max(0, n("candidatesTokenCount") - audioOut - imageOut) + n("thoughtsTokenCount"),
+            output: Math.max(0, n("candidatesTokenCount") + n("responseTokenCount") - audioOut - imageOut) +
+                n("thoughtsTokenCount"),
             cache_read: cached,
             cache_write: 0,
         }, {
@@ -122,6 +135,14 @@ export function tokensOf(usage) {
             output_image: imageOut,
         });
     }
+    if (SONIC.some((k) => k in usage))
+        // Nova Sonic's usageEvent delta (spec/providers.md §8): speech is priced as audio
+        return extras({
+            input: n("input_textTokens"),
+            output: n("output_textTokens"),
+            cache_read: 0,
+            cache_write: 0,
+        }, { input_audio: n("input_speechTokens"), output_audio: n("output_speechTokens") });
     if ("inputTokens" in usage)
         // Bedrock Converse (spec/providers.md §5)
         return {
@@ -146,16 +167,22 @@ export function tokensOf(usage) {
     // Anthropic's cache names are its own: they win over the Responses shape, since Anthropic also
     // sends output_tokens_details (thinking_tokens) since Claude Sonnet 5
     const anthropicCache = "cache_read_input_tokens" in usage || "cache_creation_input_tokens" in usage;
-    // OpenAI's transcriptions write input_token_details (singular); its images, image_tokens
+    // OpenAI's transcriptions and Realtime write input_token_details (singular); its images,
+    // image_tokens. Realtime's audio and image counts include their cached part (spec/providers.md §7).
     if (!anthropicCache &&
         Object.keys(usage).some((k) => k.startsWith("input_tokens_details_") ||
             k.startsWith("output_tokens_details_") ||
-            k.startsWith("input_token_details_"))) {
-        const cached = n("input_tokens_details_cached_tokens");
+            k.startsWith("input_token_details_") ||
+            k.startsWith("output_token_details_"))) {
+        const cached = n("input_tokens_details_cached_tokens") + n("input_token_details_cached_tokens");
         const written = n("input_tokens_details_cache_write_tokens");
-        const audioIn = n("input_tokens_details_audio_tokens") + n("input_token_details_audio_tokens");
-        const audioOut = n("output_tokens_details_audio_tokens");
-        const imageIn = n("input_tokens_details_image_tokens");
+        const audioIn = n("input_tokens_details_audio_tokens") +
+            Math.max(0, n("input_token_details_audio_tokens") -
+                n("input_token_details_cached_tokens_details_audio_tokens"));
+        const audioOut = n("output_tokens_details_audio_tokens") + n("output_token_details_audio_tokens");
+        const imageIn = n("input_tokens_details_image_tokens") +
+            Math.max(0, n("input_token_details_image_tokens") -
+                n("input_token_details_cached_tokens_details_image_tokens"));
         const imageOut = n("output_tokens_details_image_tokens");
         return extras({
             input: Math.max(0, n("input_tokens") - cached - written - audioIn - imageIn),
@@ -249,7 +276,17 @@ export function priceCall(call, table) {
     const match = call.model === null ? null : priceFor(call.model, table, call.provider);
     if (match) {
         const { price, tier } = effectivePrice(match.price, tokens, call.serviceTier);
-        const base = callMicroUsd(tokens, price, requests);
+        const percent = match.price.batch_percent;
+        if (call.batch && percent === undefined)
+            return {
+                key: match.key,
+                tokens,
+                micro_usd: null,
+                ...(requests ? { requests } : {}),
+                batch: { requests: call.batch.requests },
+            };
+        const full = callMicroUsd(tokens, price, requests);
+        const base = call.batch ? Math.floor((full * percent + 50) / 100) : full;
         const uplift = upliftPercent(table, call.provider ?? null, call.model, call.usage);
         return {
             key: match.key,
@@ -258,8 +295,19 @@ export function priceCall(call, table) {
             ...(requests ? { requests } : {}),
             ...(tier ? { tier } : {}),
             ...(uplift ? { uplift_percent: uplift } : {}),
+            ...(call.batch
+                ? { batch: { requests: call.batch.requests, percent: percent } }
+                : {}),
         };
     }
+    if (call.batch)
+        return {
+            key: null,
+            tokens,
+            micro_usd: null,
+            ...(requests ? { requests } : {}),
+            batch: { requests: call.batch.requests },
+        };
     if (typeof call.reportedMicroUsd === "number")
         return {
             key: null,
@@ -294,6 +342,22 @@ export function formatAed(fils) {
     const abs = Math.abs(fils);
     return `${sign}AED ${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}`;
 }
+/** A results download's `meta.batch.models` (spec/providers.md §6), the well-formed entries; null
+ *  when the response isn't one. */
+export function batchModels(batch) {
+    if (typeof batch !== "object" || batch === null)
+        return null;
+    const models = batch.models;
+    if (!Array.isArray(models))
+        return null;
+    return models.filter((m) => typeof m === "object" &&
+        m !== null &&
+        typeof m.model === "string" &&
+        Number.isSafeInteger(m.requests) &&
+        typeof m.usage === "object" &&
+        m.usage !== null &&
+        !Array.isArray(m.usage));
+}
 /** The session's cost report from its recorded lines. */
 export function sessionCost(lines, prices) {
     const calls = [];
@@ -304,8 +368,7 @@ export function sessionCost(lines, prices) {
     const providers = new Map(); // request seq → the configured provider id
     let webSearch = 0;
     let pinnedSha; // session.open's meta.prices_sha256 (L3.1.5)
-    for (const line of lines) {
-        const e = JSON.parse(line);
+    for (const e of envelopesOf(lines)) {
         if (e.kind === "session.open")
             pinnedSha = e.meta.prices_sha256;
         if (e.kind === "llm.request") {
@@ -316,10 +379,44 @@ export function sessionCost(lines, prices) {
                 providers.set(e.seq, e.meta.provider);
             continue;
         }
-        if (e.kind !== "llm.response" && e.kind !== "llm.incomplete")
+        // spec/cost.md §2: a client's sampling reply that names its model and usage is a model call too
+        const sampling = e.kind === "tool.sampling" && e.meta.phase === "reply" && typeof e.meta.model === "string";
+        // spec/providers.md §7: a realtime session's usage reports are its model calls
+        const realtime = e.kind === "llm.realtime.server";
+        if (e.kind !== "llm.response" && e.kind !== "llm.incomplete" && !sampling && !realtime)
             continue;
-        if (replayed.has(e.meta.request_seq))
+        if (!sampling && replayed.has(e.meta.request_seq))
             continue;
+        const provider = sampling ? null : (providers.get(e.meta.request_seq) ?? null);
+        const via = sampling
+            ? { via: "mcp_sampling" }
+            : realtime
+                ? { via: "realtime" }
+                : {};
+        const add = (seq, model, priced) => {
+            const { key, ...call } = priced;
+            for (const c of CLASSES)
+                total[c] += call.tokens[c];
+            for (const c of EXTRA)
+                if (call.tokens[c] !== undefined)
+                    total[c] = (total[c] ?? 0) + call.tokens[c];
+            webSearch += call.requests?.web_search ?? 0;
+            // a model with a price but no batch rate is unpriced, not unknown (spec/cost.md §2)
+            if (call.micro_usd === null) {
+                if (key === null)
+                    unknown.add(model ?? "(no model)");
+            }
+            else
+                micro += call.micro_usd;
+            calls.push({ seq, model, ...call, ...via });
+        };
+        // spec/cost.md §2: a batch results download, one call per model, at the batch rate
+        const batch = sampling ? null : batchModels(e.meta.batch);
+        if (batch) {
+            for (const m of batch)
+                add(e.seq, m.model, priceCall({ model: m.model, usage: m.usage, provider, batch: { requests: m.requests } }, prices.table));
+            continue;
+        }
         const usage = e.meta.usage;
         const model = typeof e.meta.model === "string" ? e.meta.model : null;
         if (typeof usage !== "object" || usage === null) {
@@ -334,27 +431,15 @@ export function sessionCost(lines, prices) {
                 });
             continue;
         }
-        const priced = priceCall({
+        add(e.seq, model, priceCall({
             model,
             usage: usage,
-            provider: providers.get(e.meta.request_seq) ?? null,
+            provider,
             serviceTier: typeof e.meta.service_tier === "string" ? e.meta.service_tier : null,
             reportedMicroUsd: Number.isSafeInteger(e.meta.reported_cost_micro_usd)
                 ? e.meta.reported_cost_micro_usd
                 : null,
-        }, prices.table);
-        const { key: _key, ...call } = priced;
-        for (const c of CLASSES)
-            total[c] += call.tokens[c];
-        for (const c of EXTRA)
-            if (call.tokens[c] !== undefined)
-                total[c] = (total[c] ?? 0) + call.tokens[c];
-        webSearch += call.requests?.web_search ?? 0;
-        if (call.micro_usd === null)
-            unknown.add(model ?? "(no model)");
-        else
-            micro += call.micro_usd;
-        calls.push({ seq: e.seq, model, ...call });
+        }, prices.table));
     }
     return {
         v: 1,

package/dist/cv/index.js

@@ -17,7 +17,7 @@ export function buildCv(o) {
         v: 1,
         type: "agentcv",
         did: o.did,
-        owner: o.owner ?? null,
+        ...(o.owner != null ? { owner: o.owner } : {}), // left out when unset, as zanii 0.26 does
         summary: o.summary,
         work: o.work ?? [],
         authority: o.authority ?? [],

package/dist/duty/index.js

@@ -1,6 +1,7 @@
 // Duty-time limits (spec/duty.md): long sessions and full context windows, detected. Pure; mirrors
 // sdks/python/src/zanii_blackbox/duty.py; pinned by spec/vectors/duty.json.
 import { priceFor, promptTokens, sessionCost } from "../cost/index.js";
+import { envelopesOf } from "../reconcile/record.js";
 const KEYS = ["max_minutes", "max_llm_calls", "max_context_pct"];
 /** spec/duty.md §1: the flight plan's limits win, limit by limit, over the server's defaults. */
 function limitsOf(open, defaults) {
@@ -15,7 +16,7 @@ function limitsOf(open, defaults) {
 }
 /** spec/duty.md §2: at most one DUTY_LIMIT per limit, where it was first crossed. */
 export function dutyFindings(lines, prices, defaults = {}) {
-    const events = lines.map((l) => JSON.parse(l));
+    const events = envelopesOf(lines);
     const open = events[0]?.kind === "session.open" ? events[0] : undefined;
     const limits = limitsOf(open, defaults);
     const out = [];

package/dist/evals/index.d.ts

@@ -0,0 +1,233 @@
+import type { Summary } from "../fleet/index.ts";
+type Obj = {
+    [key: string]: unknown;
+};
+export declare const SCORERS: readonly ["exact", "contains", "regex", "json_schema", "numeric", "tool_called", "tool_not_called", "findings_absent", "judge"];
+export type ScorerKind = (typeof SCORERS)[number];
+export interface Scorer {
+    kind: ScorerKind;
+    pattern?: string;
+    schema?: Obj;
+    tolerance_milli?: number;
+    tool?: string;
+    codes?: string[];
+}
+export interface EvalCase {
+    id: string;
+    input: unknown;
+    expected?: string | number;
+    rubric?: string;
+    tags: string[];
+    scorer: Scorer;
+}
+export declare const CASE_ID: RegExp;
+/** §2: a scorer, checked: the normalised scorer, or what's wrong with it. */
+export declare function checkScorer(v: unknown, at?: string): Scorer | string;
+/**
+ * §1: a dataset, checked: `{name, cases}` with every case normalised (an id, tags, its scorer or
+ * the dataset's), or the first problem. Numbers must be integers: decimals are written as strings.
+ */
+export declare function checkDataset(v: unknown): {
+    name: string;
+    cases: EvalCase[];
+} | string;
+/** §1: what names a dataset's content exactly: `sha256:` over the canonical JSON of {name, cases}. */
+export declare function datasetHash(d: {
+    name: string;
+    cases: readonly EvalCase[];
+}): string;
+/** §1: a JSON Lines file of cases: one object per non-empty line, or the first line that isn't. */
+export declare function parseCasesJsonl(text: string): unknown[] | string;
+/** §1: one case per saved scenario (spec/replay.md §3): a fork of the incident, passing when none
+ * of its findings comes back. */
+export declare function casesFromScenarios(scenarios: readonly unknown[]): Obj[] | string;
+/** §1: one case per recorded session: a fork of it, expected to answer what it answered then. */
+export declare function casesFromSessions(items: ReadonlyArray<{
+    session_id: string;
+    answer: string;
+}>): Obj[];
+/** What one run of the agent left, read from its record: the final answer, the tools it called,
+ * its findings, and the judge's verdict (undefined: not asked; null: an outage). */
+export interface Observation {
+    answer: string | null;
+    tools: {
+        [tool: string]: number;
+    };
+    finding_codes: {
+        [code: string]: number;
+    };
+    judge?: {
+        steps: Array<{
+            rubric: string;
+            pass: boolean;
+            reason?: string;
+        }>;
+    } | null;
+}
+export interface CaseScore {
+    pass: boolean;
+    reason: string;
+    outage?: true;
+}
+/** A decimal as integer milli-units, the digits past the third decimal dropped. */
+export declare function toMilli(s: string): bigint;
+/** §2: one case scored against what its run left. An outage of the judge is never a pass. */
+export declare function scoreCase(c: EvalCase, o: Observation): CaseScore;
+export interface RunSummary {
+    total: number;
+    passed: number;
+    failed: number;
+    outages: number;
+    pass_ppm: number;
+    pass_low_ppm: number;
+    pass_high_ppm: number;
+    cost_micro_usd: number;
+    cost_per_case_micro_usd: number;
+    tags: {
+        [tag: string]: {
+            total: number;
+            passed: number;
+        };
+    };
+}
+/** §3: a run's totals: passes with the 95% range, outages, cost, and per tag. */
+export declare function runSummary(results: ReadonlyArray<{
+    pass: boolean;
+    outage?: boolean;
+    cost_micro_usd: number;
+    tags?: readonly string[];
+}>): RunSummary;
+type Verdict = "better" | "worse" | "unclear";
+interface RunDoc {
+    id: string;
+    dataset: {
+        id: string;
+        hash: string;
+    };
+    results: ReadonlyArray<{
+        case: string;
+        pass: boolean;
+    }>;
+    summary: RunSummary;
+}
+/** §3: run `b` against run `a`: better, worse or unclear, and the cases each fixed or broke. */
+export declare function compareRuns(a: RunDoc, b: RunDoc): {
+    a: string;
+    b: string;
+    same_dataset: boolean;
+    verdict: Verdict;
+    pass_ppm: {
+        a: number;
+        b: number;
+    };
+    pass_range_ppm: {
+        a: number[];
+        b: number[];
+    };
+    fixed: string[];
+    broken: string[];
+    cost_per_case_delta_micro_usd: number;
+};
+export declare const PROMPT_NAME: RegExp;
+/** §4: a prompt text's version hash: the gateway's `system_sha256` of a request whose system
+ * prompt is this text (an Anthropic `system` string, or a Chat Completions system message). */
+export declare function promptSha256(text: string): string;
+/** §5: the arm a session falls in: the first 8 hex of sha256(`<experiment>:<session>`) mod 1000,
+ * walked through the arms' per-mille shares in order. The same session always gets the same arm. */
+export declare function assignArm(experimentId: string, sessionId: string, arms: ReadonlyArray<{
+    name: string;
+    share_permille: number;
+}>): string;
+export interface Arm {
+    name: string;
+    version: number;
+    share_permille: number;
+}
+/** §5: an experiment, checked against its prompt's versions: `{name, prompt, arms}`, or why not. */
+export declare function checkExperiment(v: unknown, versions: readonly number[]): {
+    name: string;
+    prompt: string;
+    arms: Arm[];
+} | string;
+/** §4: how a group of runs did: success with the 95% range, spend per run, findings. */
+export declare function armStats(runs: readonly Summary[]): {
+    sessions: number;
+    success: number;
+    success_ppm: number;
+    success_low_ppm: number;
+    success_high_ppm: number;
+    spend_per_run_micro_usd: number;
+    findings: {
+        advisory: number;
+        caution: number;
+        warning: number;
+    };
+    finding_codes: {
+        [k: string]: number;
+    };
+};
+/**
+ * §4: each version of a prompt, from closed sessions' summaries: the runs that declared it (an
+ * experiment's arm or the SDK's `promptVersion`), and the runs that declared none whose first
+ * request's system prompt hash is the version's. Oldest version first, each against the one before.
+ */
+export declare function promptResults(summaries: readonly Summary[], prompt: {
+    name: string;
+    versions: ReadonlyArray<{
+        version: number;
+        sha256: string;
+    }>;
+}): {
+    name: string;
+    sessions: number;
+    versions: {
+        sessions: number;
+        success: number;
+        success_ppm: number;
+        success_low_ppm: number;
+        success_high_ppm: number;
+        spend_per_run_micro_usd: number;
+        findings: {
+            advisory: number;
+            caution: number;
+            warning: number;
+        };
+        finding_codes: {
+            [k: string]: number;
+        };
+        version: number;
+        sha256: string;
+        declared: number;
+        detected: number;
+        vs_previous: Verdict | null;
+    }[];
+};
+/** §5: each arm of an experiment, from the closed sessions it assigned, against the first arm. */
+export declare function experimentResults(summaries: readonly Summary[], experiment: {
+    id: string;
+    arms: readonly Arm[];
+}): {
+    id: string;
+    sessions: number;
+    arms: {
+        sessions: number;
+        success: number;
+        success_ppm: number;
+        success_low_ppm: number;
+        success_high_ppm: number;
+        spend_per_run_micro_usd: number;
+        findings: {
+            advisory: number;
+            caution: number;
+            warning: number;
+        };
+        finding_codes: {
+            [k: string]: number;
+        };
+        name: string;
+        version: number;
+        share_permille: number;
+        vs_control: Verdict | null;
+    }[];
+};
+export {};