@zanii/blackbox 0.0.0-stage → 0.2.0

package/dist/cost/index.js

@@ -0,0 +1,293 @@
+// The cost engine (spec/cost.md): integer micro-USD from a versioned price table, AED display.
+// Mirrors sdks/python/src/zanii_blackbox/cost.py.
+import { createHash } from "node:crypto";
+const CLASSES = ["input", "output", "cache_read", "cache_write"];
+const EXTRA = ["cache_write_1h", "input_audio", "output_audio"];
+const OPTIONAL_PRICES = [
+    "context_window",
+    "cache_write_1h",
+    "input_audio",
+    "output_audio",
+    "web_search_per_1k",
+];
+/** Parses and checks a price table; its sha256 is of the exact file bytes. */
+export function loadPrices(bytes) {
+    const table = JSON.parse(new TextDecoder().decode(bytes));
+    if (table.currency !== "USD" ||
+        table.unit !== "micro_usd_per_mtok" ||
+        typeof table.models !== "object")
+        throw new Error("not a v1 price table (USD, micro_usd_per_mtok)");
+    const whole = (model, field, v) => {
+        if (!Number.isSafeInteger(v) || v < 0)
+            throw new Error(`price table: ${model}.${field} must be an integer ≥ 0`);
+    };
+    for (const [model, p] of Object.entries(table.models)) {
+        for (const c of CLASSES)
+            whole(model, c, p[c]);
+        for (const c of OPTIONAL_PRICES)
+            if (p[c] !== undefined)
+                whole(model, c, p[c]);
+        for (const [i, t] of (p.context_tiers ?? []).entries()) {
+            whole(model, `context_tiers[${i}].above`, t.above);
+            for (const c of CLASSES)
+                if (t[c] !== undefined)
+                    whole(model, `context_tiers[${i}].${c}`, t[c]);
+        }
+        for (const [name, t] of Object.entries(p.service_tiers ?? {}))
+            for (const c of CLASSES)
+                if (t[c] !== undefined)
+                    whole(model, `service_tiers.${name}.${c}`, t[c]);
+        if (p.status !== undefined && p.status !== "deprecated" && p.status !== "beta")
+            throw new Error(`price table: ${model}.status must be deprecated or beta`);
+    }
+    return { table, sha256: `sha256:${createHash("sha256").update(bytes).digest("hex")}` };
+}
+/** The exact model, else the longest table key the model starts with; null when there's none. */
+export function priceFor(model, table, 
+/** The configured provider id: `<provider>/<model>` keys are tried first (spec/cost.md §1). */
+provider) {
+    if (provider) {
+        const scoped = lookup(`${provider}/${model}`, table);
+        if (scoped)
+            return scoped;
+    }
+    const found = lookup(model, table);
+    if (found)
+        return found;
+    // spec/cost.md §1: a cloud's model id (Bedrock's `us.anthropic.claude-…`) as the vendor's own
+    const plain = model.replace(GEO, "").replace(VENDOR, "");
+    return plain !== model ? lookup(plain, table) : null;
+}
+const GEO = /^(?:us|eu|apac|au|ca|jp|global|us-gov)\./;
+const VENDOR = /^(?:anthropic|amazon|meta|mistral|cohere|ai21|deepseek|openai|qwen|writer|google)\./;
+function lookup(model, table) {
+    const exact = table.models[model];
+    if (exact)
+        return { key: model, price: exact };
+    let best = null;
+    for (const key of Object.keys(table.models))
+        if (model.startsWith(key) && (best === null || key.length > best.length))
+            best = key;
+    return best === null ? null : { key: best, price: table.models[best] };
+}
+/** Token classes from a recorded, flattened `meta.usage` (spec/cost.md §2). */
+export function tokensOf(usage) {
+    const n = (k) => {
+        const v = usage[k];
+        return typeof v === "number" && Number.isSafeInteger(v) && v > 0 ? v : 0;
+    };
+    if ("promptTokenCount" in usage) {
+        // Gemini (spec/providers.md §5): the prompt count includes the cached part; thoughts are output.
+        const cached = n("cachedContentTokenCount");
+        const audioIn = n("promptTokensDetails_AUDIO");
+        const audioOut = n("candidatesTokensDetails_AUDIO");
+        return extras({
+            input: Math.max(0, n("promptTokenCount") - cached - audioIn) + n("toolUsePromptTokenCount"),
+            output: Math.max(0, n("candidatesTokenCount") - audioOut) + n("thoughtsTokenCount"),
+            cache_read: cached,
+            cache_write: 0,
+        }, { input_audio: audioIn, output_audio: audioOut });
+    }
+    if ("inputTokens" in usage)
+        // Bedrock Converse (spec/providers.md §5)
+        return {
+            input: n("inputTokens"),
+            output: n("outputTokens"),
+            cache_read: n("cacheReadInputTokens"),
+            cache_write: n("cacheWriteInputTokens"),
+        };
+    if ("prompt_tokens" in usage) {
+        // prompt_tokens includes both the cached part and the part written to the cache (B1)
+        const cached = n("prompt_tokens_details_cached_tokens");
+        const written = n("prompt_tokens_details_cache_write_tokens");
+        const audioIn = n("prompt_tokens_details_audio_tokens");
+        const audioOut = n("completion_tokens_details_audio_tokens");
+        return extras({
+            input: Math.max(0, n("prompt_tokens") - cached - written - audioIn),
+            output: Math.max(0, n("completion_tokens") - audioOut),
+            cache_read: cached,
+            cache_write: written,
+        }, { input_audio: audioIn, output_audio: audioOut });
+    }
+    if (Object.keys(usage).some((k) => k.startsWith("input_tokens_details_") || k.startsWith("output_tokens_details_"))) {
+        const cached = n("input_tokens_details_cached_tokens");
+        const written = n("input_tokens_details_cache_write_tokens");
+        const audioIn = n("input_tokens_details_audio_tokens");
+        const audioOut = n("output_tokens_details_audio_tokens");
+        return extras({
+            input: Math.max(0, n("input_tokens") - cached - written - audioIn),
+            output: Math.max(0, n("output_tokens") - audioOut),
+            cache_read: cached,
+            cache_write: written,
+        }, { input_audio: audioIn, output_audio: audioOut });
+    }
+    // Anthropic: cache_creation_input_tokens is both lifetimes; the 1-hour part is priced apart
+    const hour = n("cache_creation_ephemeral_1h_input_tokens");
+    return extras({
+        input: n("input_tokens"),
+        output: n("output_tokens"),
+        cache_read: n("cache_read_input_tokens"),
+        cache_write: Math.max(0, n("cache_creation_input_tokens") - hour),
+    }, { cache_write_1h: hour });
+}
+/** Adds the optional classes that are non-zero, so a record without them prices as before. */
+function extras(base, more) {
+    const out = { ...base };
+    for (const c of EXTRA)
+        if ((more[c] ?? 0) > 0)
+            out[c] = more[c];
+    return out;
+}
+/** Requests a call made to a provider's server tools (Anthropic web search). */
+export function requestsOf(usage) {
+    const v = usage.server_tool_use_web_search_requests;
+    return typeof v === "number" && Number.isSafeInteger(v) && v > 0 ? { web_search: v } : undefined;
+}
+/** A call's prompt, in tokens: what a context tier is decided on. */
+export function promptTokens(t) {
+    return t.input + t.cache_read + t.cache_write + (t.cache_write_1h ?? 0) + (t.input_audio ?? 0);
+}
+/** The prices a call pays: a context tier when its prompt is over one, then its service tier. */
+export function effectivePrice(price, tokens, serviceTier) {
+    let out = price;
+    const tier = {};
+    const size = promptTokens(tokens);
+    const context = [...(price.context_tiers ?? [])]
+        .sort((a, b) => b.above - a.above)
+        .find((t) => size > t.above);
+    if (context) {
+        out = { ...out, ...pick(context) };
+        tier.context_above = context.above;
+    }
+    const service = serviceTier ? price.service_tiers?.[serviceTier] : undefined;
+    if (service && serviceTier) {
+        out = { ...out, ...pick(service) };
+        tier.service = serviceTier;
+    }
+    return Object.keys(tier).length ? { price: out, tier } : { price: out };
+}
+function pick(t) {
+    const o = {};
+    for (const c of CLASSES)
+        if (t[c] !== undefined)
+            o[c] = t[c];
+    return o;
+}
+/** round_half_up((tokens × price + requests × price per 1k × 1000) / 1e6), integers only. */
+export function callMicroUsd(tokens, price, requests) {
+    let sum = 0;
+    for (const c of CLASSES)
+        sum += tokens[c] * price[c];
+    sum += (tokens.cache_write_1h ?? 0) * (price.cache_write_1h ?? price.cache_write);
+    sum += (tokens.input_audio ?? 0) * (price.input_audio ?? price.input);
+    sum += (tokens.output_audio ?? 0) * (price.output_audio ?? price.output);
+    sum += (requests?.web_search ?? 0) * (price.web_search_per_1k ?? 0) * 1000;
+    if (!Number.isSafeInteger(sum))
+        throw new RangeError("cost overflow: tokens × price beyond 2^53");
+    return Math.floor((sum + 500_000) / 1_000_000);
+}
+/** One call's cost as the cost report prices it: the table (provider-scoped, tiered), else the
+ *  provider's own reported cost, else null. The spend and duty checks use the same. */
+export function priceCall(call, table) {
+    const tokens = tokensOf(call.usage);
+    const requests = requestsOf(call.usage);
+    const match = call.model === null ? null : priceFor(call.model, table, call.provider);
+    if (match) {
+        const { price, tier } = effectivePrice(match.price, tokens, call.serviceTier);
+        return {
+            key: match.key,
+            tokens,
+            micro_usd: callMicroUsd(tokens, price, requests),
+            ...(requests ? { requests } : {}),
+            ...(tier ? { tier } : {}),
+        };
+    }
+    if (typeof call.reportedMicroUsd === "number")
+        return {
+            key: null,
+            tokens,
+            micro_usd: call.reportedMicroUsd,
+            ...(requests ? { requests } : {}),
+            source: "reported",
+        };
+    return { key: null, tokens, micro_usd: null, ...(requests ? { requests } : {}) };
+}
+/** fils at the CBUAE peg (3.6725 AED / USD), rounded half up. */
+export function toAedFils(microUsd) {
+    return Math.floor((microUsd * 36725 + 50_000_000) / 100_000_000);
+}
+export function formatAed(fils) {
+    const sign = fils < 0 ? "-" : "";
+    const abs = Math.abs(fils);
+    return `${sign}AED ${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}`;
+}
+/** The session's cost report from its recorded lines. */
+export function sessionCost(lines, prices) {
+    const calls = [];
+    const unknown = new Set();
+    const total = { input: 0, output: 0, cache_read: 0, cache_write: 0 };
+    let micro = 0;
+    const replayed = new Set(); // requests served from a cassette: no provider was paid (spec/cost.md)
+    const providers = new Map(); // request seq → the configured provider id
+    let webSearch = 0;
+    let pinnedSha; // session.open's meta.prices_sha256 (L3.1.5)
+    for (const line of lines) {
+        const e = JSON.parse(line);
+        if (e.kind === "session.open")
+            pinnedSha = e.meta.prices_sha256;
+        if (e.kind === "llm.request") {
+            const r = e.meta.replay;
+            if (Number.isSafeInteger(r?.take))
+                replayed.add(e.seq);
+            if (typeof e.meta.provider === "string")
+                providers.set(e.seq, e.meta.provider);
+            continue;
+        }
+        if (e.kind !== "llm.response" && e.kind !== "llm.incomplete")
+            continue;
+        if (replayed.has(e.meta.request_seq))
+            continue;
+        const usage = e.meta.usage;
+        if (typeof usage !== "object" || usage === null)
+            continue;
+        const model = typeof e.meta.model === "string" ? e.meta.model : null;
+        const priced = priceCall({
+            model,
+            usage: usage,
+            provider: providers.get(e.meta.request_seq) ?? null,
+            serviceTier: typeof e.meta.service_tier === "string" ? e.meta.service_tier : null,
+            reportedMicroUsd: Number.isSafeInteger(e.meta.reported_cost_micro_usd)
+                ? e.meta.reported_cost_micro_usd
+                : null,
+        }, prices.table);
+        const { key: _key, ...call } = priced;
+        for (const c of CLASSES)
+            total[c] += call.tokens[c];
+        for (const c of EXTRA)
+            if (call.tokens[c] !== undefined)
+                total[c] = (total[c] ?? 0) + call.tokens[c];
+        webSearch += call.requests?.web_search ?? 0;
+        if (call.micro_usd === null)
+            unknown.add(model ?? "(no model)");
+        else
+            micro += call.micro_usd;
+        calls.push({ seq: e.seq, model, ...call });
+    }
+    return {
+        v: 1,
+        price_table: {
+            version: prices.table.version,
+            sha256: prices.sha256,
+            ...(typeof pinnedSha === "string" ? { pinned: pinnedSha === prices.sha256 } : {}),
+        },
+        total_micro_usd: micro,
+        total_aed_fils: toAedFils(micro),
+        calls_priced: calls.filter((c) => c.micro_usd !== null).length,
+        calls_unpriced: calls.filter((c) => c.micro_usd === null).length,
+        unknown_models: [...unknown].sort(),
+        tokens: total,
+        calls,
+        ...(replayed.size ? { calls_replayed: replayed.size } : {}),
+        ...(webSearch ? { requests: { web_search: webSearch } } : {}),
+    };
+}

package/dist/data/index.d.ts

@@ -0,0 +1,191 @@
+export interface Identifier {
+    id: string;
+    title: string;
+    pattern: string;
+    normalize: "digits" | "lower" | "upper" | "exact";
+    keep_last?: number;
+    check?: "luhn";
+    personal: boolean;
+}
+/** spec/data/identifiers-v1.json: always on. */
+export declare const CORE_IDENTIFIERS: readonly Identifier[];
+/** spec/data/core-v1.json: evidence defaults for common writes (§5). */
+export declare const CORE_EVIDENCE: ReadonlyArray<{
+    tool: string;
+    field: string;
+}>;
+/**
+ * §6: the canary values as one identifier, matched literally. Longer values first, so a value that
+ * contains another wins. ASCII punctuation is escaped the same way both languages read it.
+ */
+export declare function canaryIdentifier(values: readonly string[]): Identifier;
+export declare function normalizeValue(value: string, how: Identifier["normalize"]): string;
+export declare function luhn(digits: string): boolean;
+export interface Found {
+    id: string;
+    value: string;
+}
+/** §1: each distinct (id, normalized value) in text order, at most 200. Earlier identifiers win a span. */
+export declare function dataScan(text: string, identifiers: readonly Identifier[]): Found[];
+/** Every match counted (not only distinct ones), for `data.classes`. */
+export declare function dataClasses(text: string, identifiers: readonly Identifier[]): Record<string, number>;
+/** §2: the tenant's fingerprint of one identifier value. */
+export declare function dataFingerprint(key: Uint8Array | string, tenant: string, id: string, value: string): string;
+/** §2: the `data` meta of a body, or undefined when it holds no identifier. */
+export declare function dataMeta(text: string, identifiers: readonly Identifier[], key: Uint8Array | string, tenant: string): {
+    classes: Record<string, number>;
+    fps: Array<{
+        id: string;
+        fp: string;
+    }>;
+} | undefined;
+export interface Lineage {
+    subjects: Array<{
+        fp: string;
+        id: string;
+        first: {
+            seq: number;
+            kind: string;
+            from: string;
+        };
+        sent: Array<{
+            seq: number;
+            to: string;
+            region?: string;
+        }>;
+    }>;
+    destinations: Array<{
+        to: string;
+        region?: string;
+        classes: Record<string, number>;
+        subjects: number;
+    }>;
+}
+/** §3: where each subject's data came from and went in one session. */
+export declare function lineage(lines: readonly string[]): Lineage;
+export interface Topology {
+    nodes: Array<{
+        id: string;
+        kind: string;
+        region?: string;
+        sessions: number;
+    }>;
+    edges: Array<{
+        from: string;
+        to: string;
+        region?: string;
+        classes: Record<string, number>;
+        subjects: number;
+        sessions: number;
+        flags: Array<"outside_region" | "region_unknown">;
+    }>;
+}
+/**
+ * §3a: many sessions' lineage as one map: where data is read from (nodes on the left) and sent to,
+ * with how many subjects and sessions each flow carried. With `dataRegion`, a flow to a model in
+ * another region is flagged `outside_region`, one to a model of no known region `region_unknown`.
+ */
+export declare function dataTopology(sessions: ReadonlyArray<{
+    session_id: string;
+    lines: readonly string[];
+}>, dataRegion?: string | null): Topology;
+export interface DataRules {
+    identifiers: readonly Identifier[];
+    residency: boolean;
+    restricted: ReadonlyArray<{
+        identifiers: readonly string[];
+        allow: readonly string[];
+    }>;
+    evidence: ReadonlyArray<{
+        tool: string;
+        field: string;
+    }>;
+}
+export interface DataFinding {
+    code: "DATA_LEFT_REGION" | "DATA_REGION_UNKNOWN" | "DATA_TO_UNAPPROVED_DESTINATION" | "DATA_CANARY_TRIPPED" | "CONSENT_WITHDRAWN_USE" | "EGRESS_OPEN" | "TOOL_WITNESS_INVALID" | "UNEVIDENCED_SIDE_EFFECT";
+    severity: "advisory" | "caution" | "warning";
+    ref: Record<string, string | number>;
+}
+/** §4, §6, §7, §9: findings over a session's events. */
+export declare function dataFindings(lines: readonly string[], rules: Pick<DataRules, "identifiers" | "residency" | "restricted">, dataRegion: string | null): DataFinding[];
+export declare const hasEvidence: (value: unknown, field: string) => boolean;
+/** §5: successful results of state-changing tools with no evidence of their own. */
+export declare function unevidenced(lines: readonly string[], body: (hash: string) => Uint8Array | undefined, rules: DataRules["evidence"]): DataFinding[];
+/** §9: a tool signs what it returns, so the result is the tool's word, not the agent's. */
+export declare function toolWitness(keypair: {
+    did?: string;
+    privateKey: Uint8Array;
+}): {
+    did: string;
+    sign(result: unknown): {
+        result: unknown;
+        witness: {
+            did: string;
+            sig: string;
+        };
+    };
+};
+/** §9: does `witness` sign `result`? */
+export declare function verifyToolWitness(result: unknown, witness: {
+    did?: unknown;
+    sig?: unknown;
+}): boolean;
+/** §9: the witness an MCP result or an SDK tool.result body carries, checked; undefined if none. */
+export declare function witnessOf(bodyJson: unknown): {
+    did: string;
+    ok: boolean;
+} | undefined;
+export interface Connector {
+    v: 1;
+    id: string;
+    format: "csv" | "jsonl";
+    fields: {
+        at: string;
+        action: string;
+        object?: string;
+        evidence?: string;
+        amount?: string;
+        actor?: string;
+    };
+    actions: Record<string, string>;
+    /** Which tool arguments hold the object and the amount the call acted on. */
+    arguments?: {
+        object?: string;
+        amount?: string;
+    };
+    window_seconds?: number;
+}
+export interface AuditRow {
+    row: number;
+    at: number;
+    action: string;
+    object?: string;
+    evidence?: string;
+    amount?: string;
+    actor?: string;
+}
+/** §10: why a connector is invalid, or undefined. */
+export declare function checkAuditConnector(value: unknown): string | undefined;
+/** RFC 4180: a header row, then rows; quoted fields may hold commas, quotes ("") and newlines. */
+export declare function parseCsv(text: string): string[][];
+/** §10: the rows of an audit-log export, in file order; rows without a time or action are skipped. */
+export declare function parseAuditLog(text: string, connector: Connector): AuditRow[];
+export interface SystemReport {
+    connector: string;
+    calls: number;
+    rows: number;
+    matched: Array<{
+        seq: number;
+        row: number;
+        by: "evidence" | "action";
+    }>;
+    findings: Array<{
+        code: "AGENT_ACTION_NOT_IN_SYSTEM" | "SYSTEM_CHANGE_NOT_IN_RECORD" | "SYSTEM_RECORD_MISMATCH";
+        ref: {
+            seq?: number;
+            row?: number;
+        };
+    }>;
+}
+/** §10: the session's successful mapped tool calls against the system's own log. */
+export declare function reconcileSystem(lines: readonly string[], body: (hash: string) => Uint8Array | undefined, rows: readonly AuditRow[], connector: Connector, evidence: DataRules["evidence"]): SystemReport;