@zanii/blackbox 0.21.2 → 0.23.0

package/dist/otlp/index.js

@@ -3,6 +3,7 @@
 import { createHash } from "node:crypto";
 import { taxonomyOf } from "../analysis/taxonomy.js";
 import { tokensOf } from "../cost/index.js";
+import { canonical } from "../reconcile/shared.js";
 const hex = (s, n) => createHash("sha256").update(s).digest("hex").slice(0, n);
 const nanos = (ts) => `${Date.parse(ts)}000000`;
 const str = (key, v) => typeof v === "string" ? [{ key, value: { stringValue: v } }] : [];
@@ -525,3 +526,470 @@ export function fromOtlp(body) {
     }
     return { events };
 }
+// ---------------------------------------------------------------- metrics and logs (spec/otlp.md §5)
+const MAX_POINTS = 5000;
+const NANOS = /^[0-9]{1,20}$/;
+const DECIMAL = /^-?[0-9]+(\.[0-9]+)?([eE][+-]?[0-9]+)?$/;
+const arr = (v) => (Array.isArray(v) ? v : []);
+const obj = (v) => v && typeof v === "object" && !Array.isArray(v) ? v : {};
+const idOf = (v) => createHash("sha256").update(canonical(v)).digest("hex").slice(0, 32);
+const nanoStr = (v) => {
+    const s = typeof v === "string" || typeof v === "number" ? String(v) : "";
+    return NANOS.test(s) ? s : "";
+};
+/** An OTLP integer (a JSON number or a decimal string): a number when safe, else the string. */
+const intOf = (v) => {
+    const s = typeof v === "number" && Number.isInteger(v) ? String(v) : typeof v === "string" ? v : "";
+    if (!/^-?[0-9]{1,20}$/.test(s))
+        return null;
+    const n = Number(s);
+    return Number.isSafeInteger(n) ? n : s;
+};
+/** spec/otlp.md §5: a double as integer micro-units, half up; null when it isn't a finite number
+ * or its micro-units aren't a safe integer. */
+export function microOf(v) {
+    const n = typeof v === "number" ? v : typeof v === "string" && DECIMAL.test(v) ? Number(v) : NaN;
+    if (!Number.isFinite(n))
+        return null;
+    const m = Math.floor(n * 1e6 + 0.5);
+    return Number.isSafeInteger(m) ? m : null;
+}
+/** No floats in the record (house rule): a number that isn't a safe integer becomes its decimal
+ * text, as JavaScript writes it, at any depth. */
+const integral = (v) => typeof v === "number"
+    ? Number.isSafeInteger(v)
+        ? v
+        : String(v)
+    : Array.isArray(v)
+        ? v.map(integral)
+        : v && typeof v === "object"
+            ? Object.fromEntries(Object.entries(v).map(([k, x]) => [k, integral(x)]))
+            : v;
+/** A data point or log record about GenAI or this gateway: a `gen_ai.*` or `blackbox.*` key. */
+const keyed = (a) => Object.keys(a).some((k) => k.startsWith("gen_ai.") || k.startsWith("blackbox."));
+const scopeOf = (s) => {
+    const scope = obj(s);
+    return {
+        name: typeof scope.name === "string" ? scope.name : "",
+        ...(typeof scope.version === "string" ? { version: scope.version } : {}),
+    };
+};
+/** Over 60,000 bytes: the attributes of at most 1,000 bytes are kept (as for spans, §4). */
+const capped = (data, drop = []) => {
+    if (size(data) <= MAX_DATA)
+        return data;
+    for (const k of drop)
+        delete data[k];
+    data.attributes = Object.fromEntries(Object.entries(data.attributes).filter(([, v]) => size(v) <= 1000));
+    data.truncated = true;
+    return data;
+};
+const put = (o, key, v) => {
+    if (v !== null && v !== undefined)
+        o[key] = v;
+};
+const METRIC_KINDS = [
+    ["gauge", "gauge"],
+    ["sum", "sum"],
+    ["histogram", "histogram"],
+    ["exponentialHistogram", "exponential_histogram"],
+    ["summary", "summary"],
+];
+/**
+ * spec/otlp.md §5: OTLP/JSON metrics as session events, one per data point of a GenAI metric
+ * (`gen_ai.*`) or a point with a `gen_ai.*` or `blackbox.*` attribute (the resource's count);
+ * others are skipped. Doubles become integer micro-units; the event_id is a hash of the point's
+ * identity, so a resent point adds nothing.
+ */
+export function fromOtlpMetrics(body) {
+    const rm = body?.resourceMetrics;
+    if (!body || typeof body !== "object" || !Array.isArray(rm))
+        return { error: "not OTLP/JSON metrics (no resourceMetrics)" };
+    const events = [];
+    let points = 0;
+    for (const r of arr(rm)) {
+        const res = attributes(obj(r.resource).attributes);
+        const service = typeof res["service.name"] === "string" ? res["service.name"] : undefined;
+        const sid = res["blackbox.session_id"];
+        for (const sm of arr(r.scopeMetrics)) {
+            const scope = scopeOf(sm.scope);
+            for (const m of arr(sm.metrics)) {
+                if (typeof m.name !== "string" || !m.name)
+                    return { error: "a metric without a name" };
+                const name = m.name;
+                const found = METRIC_KINDS.find(([k]) => m[k] && typeof m[k] === "object");
+                if (!found)
+                    continue;
+                const [field, kind] = found;
+                const agg = obj(m[field]);
+                for (const p of arr(agg.dataPoints)) {
+                    if (++points > MAX_POINTS)
+                        return { error: `more than ${MAX_POINTS} data points` };
+                    const own = integral(attributes(p.attributes));
+                    if (!name.startsWith("gen_ai.") && !keyed(own) && !keyed(res))
+                        continue;
+                    const attrs = {
+                        ...(typeof sid === "string" ? { "blackbox.session_id": sid } : {}),
+                        ...own,
+                    };
+                    const data = { source: "otlp", signal: "metrics" };
+                    put(data, "service", service);
+                    data.scope = scope;
+                    data.metric = name;
+                    if (typeof m.unit === "string" && m.unit)
+                        data.unit = m.unit;
+                    data.kind = kind;
+                    if (typeof agg.aggregationTemporality === "number")
+                        data.temporality = agg.aggregationTemporality;
+                    if (typeof agg.isMonotonic === "boolean")
+                        data.monotonic = agg.isMonotonic;
+                    const start = nanoStr(p.startTimeUnixNano);
+                    if (start && start !== "0")
+                        data.start_unix_nano = start;
+                    data.time_unix_nano = nanoStr(p.timeUnixNano);
+                    data.attributes = attrs;
+                    if (kind === "gauge" || kind === "sum") {
+                        put(data, "value", p.asInt !== undefined ? intOf(p.asInt) : null);
+                        put(data, "value_micro", p.asInt === undefined ? microOf(p.asDouble) : null);
+                    }
+                    else {
+                        put(data, "count", intOf(p.count));
+                        put(data, "sum_micro", microOf(p.sum));
+                        put(data, "min_micro", microOf(p.min));
+                        put(data, "max_micro", microOf(p.max));
+                        if (kind === "histogram") {
+                            const counts = arr(p.bucketCounts).map((c) => intOf(c));
+                            const bounds = arr(p.explicitBounds).map((b) => microOf(b));
+                            if (counts.length)
+                                data.bucket_counts = counts;
+                            if (bounds.length)
+                                data.bounds_micro = bounds;
+                        }
+                    }
+                    const id = idOf([service ?? null, scope.name, name, start, data.time_unix_nano, attrs]);
+                    events.push({
+                        event_id: `otel:m:${id}`,
+                        type: "otel.metric",
+                        name: name.slice(0, 256),
+                        ts: isoOfNanos(p.timeUnixNano),
+                        data: capped(data),
+                    });
+                }
+            }
+        }
+    }
+    return { events };
+}
+/**
+ * spec/otlp.md §5: OTLP/JSON logs as session events, one per GenAI record (an event name
+ * `gen_ai.*`, or a `gen_ai.*` or `blackbox.*` attribute, the resource's included); others are
+ * skipped. The body is recorded as `body_sha256`, and itself only with `bodies: true`.
+ */
+export function fromOtlpLogs(body, options = {}) {
+    const rl = body?.resourceLogs;
+    if (!body || typeof body !== "object" || !Array.isArray(rl))
+        return { error: "not OTLP/JSON logs (no resourceLogs)" };
+    const events = [];
+    let records = 0;
+    for (const r of arr(rl)) {
+        const res = attributes(obj(r.resource).attributes);
+        const service = typeof res["service.name"] === "string" ? res["service.name"] : undefined;
+        const sid = res["blackbox.session_id"];
+        for (const sl of arr(r.scopeLogs)) {
+            const scope = scopeOf(sl.scope);
+            for (const lr of arr(sl.logRecords)) {
+                if (++records > MAX_POINTS)
+                    return { error: `more than ${MAX_POINTS} log records` };
+                const own = integral(attributes(lr.attributes));
+                const eventName = typeof lr.eventName === "string" && lr.eventName
+                    ? lr.eventName
+                    : typeof own["event.name"] === "string" && own["event.name"]
+                        ? own["event.name"]
+                        : undefined;
+                if (!eventName?.startsWith("gen_ai.") && !keyed(own) && !keyed(res))
+                    continue;
+                const attrs = {
+                    ...(typeof sid === "string" ? { "blackbox.session_id": sid } : {}),
+                    ...own,
+                };
+                const data = { source: "otlp", signal: "logs" };
+                put(data, "service", service);
+                data.scope = scope;
+                const time = nanoStr(lr.timeUnixNano);
+                const observed = nanoStr(lr.observedTimeUnixNano);
+                data.time_unix_nano = time;
+                if (observed && observed !== "0")
+                    data.observed_unix_nano = observed;
+                if (Number.isSafeInteger(lr.severityNumber) && lr.severityNumber !== 0)
+                    data.severity_number = lr.severityNumber;
+                if (typeof lr.severityText === "string" && lr.severityText)
+                    data.severity_text = lr.severityText;
+                put(data, "event_name", eventName);
+                if (typeof lr.traceId === "string" && /^[0-9a-f]{32}$/i.test(lr.traceId))
+                    data.trace_id = lr.traceId.toLowerCase();
+                if (typeof lr.spanId === "string" && /^[0-9a-f]{16}$/i.test(lr.spanId))
+                    data.span_id = lr.spanId.toLowerCase();
+                data.attributes = attrs;
+                const value = lr.body === undefined ? undefined : integral(anyValue(lr.body));
+                if (value !== undefined)
+                    data.body_sha256 = `sha256:${createHash("sha256").update(canonical(value)).digest("hex")}`;
+                const id = idOf([service ?? null, scope.name, data]);
+                if (value !== undefined && options.bodies)
+                    data.body = value;
+                events.push({
+                    event_id: `otel:l:${id}`,
+                    type: "otel.log",
+                    name: String(eventName ?? data.severity_text ?? "log").slice(0, 256),
+                    ts: isoOfNanos(time && time !== "0" ? time : observed),
+                    data: capped(data, ["body"]),
+                });
+            }
+        }
+    }
+    return { events };
+}
+// ---------------------------------------------------------------- exporting metrics and logs (§6)
+/** The GenAI conventions' advised bucket boundaries: tokens, and seconds (as milliseconds). */
+const TOKEN_BOUNDS = [
+    1, 4, 16, 64, 256, 1024, 4096, 16384, 65536, 262144, 1048576, 4194304, 16777216, 67108864,
+];
+const SECOND_BOUNDS = [
+    0.01, 0.02, 0.04, 0.08, 0.16, 0.32, 0.64, 1.28, 2.56, 5.12, 10.24, 20.48, 40.96, 81.92,
+];
+const MS_BOUNDS = [10, 20, 40, 80, 160, 320, 640, 1280, 2560, 5120, 10240, 20480, 40960, 81920];
+const collect = (m, attrs, v) => {
+    const key = JSON.stringify(attrs);
+    const p = m.get(key) ?? { attrs, values: [] };
+    p.values.push(v);
+    m.set(key, p);
+};
+const histogram = (m, bounds, scale, times) => [...m.values()].map((p) => {
+    const counts = new Array(bounds.length + 1).fill(0);
+    for (const v of p.values) {
+        const i = bounds.findIndex((b) => v <= b);
+        counts[i < 0 ? bounds.length : i] = counts[i < 0 ? bounds.length : i] + 1;
+    }
+    return {
+        attributes: p.attrs,
+        ...times,
+        count: String(p.values.length),
+        sum: p.values.reduce((a, b) => a + b, 0) / scale,
+        bucketCounts: counts.map(String),
+        explicitBounds: scale === 1 ? bounds : SECOND_BOUNDS,
+        min: Math.min(...p.values) / scale,
+        max: Math.max(...p.values) / scale,
+    };
+});
+/** What failed about a model call (the `error.type` of §2), or undefined. */
+const callError = (end) => {
+    if (!end)
+        return "incomplete";
+    if (end.kind === "llm.incomplete")
+        return String(end.meta.reason ?? "incomplete");
+    const status = end.meta.status;
+    return typeof status === "number" && status >= 400 ? String(status) : undefined;
+};
+const resourceOf = (session, service = "zanii-blackbox") => ({
+    attributes: [...str("service.name", service), ...str("blackbox.session_id", session)],
+});
+/**
+ * spec/otlp.md §6: a session's metrics as OTLP/JSON: token usage and call duration (the GenAI
+ * client histograms), the calls witnessed, and the priced cost in micro-USD per model from its cost
+ * report (spec/cost.md), cumulative from the session's open to its last event.
+ */
+export function toOtlpMetrics(lines, cost) {
+    const events = lines.map((l) => JSON.parse(l));
+    const first = events[0];
+    if (!first)
+        return { resourceMetrics: [] };
+    const session = first.session_id;
+    const times = {
+        startTimeUnixNano: nanos(first.ts),
+        timeUnixNano: nanos(events[events.length - 1].ts),
+    };
+    const ends = new Map();
+    for (const e of events)
+        if ((e.kind === "llm.response" || e.kind === "llm.incomplete") &&
+            typeof e.meta.request_seq === "number")
+            ends.set(e.meta.request_seq, e);
+    const tokens = new Map();
+    const durations = new Map();
+    let chats = 0;
+    for (const e of events.filter((x) => x.kind === "llm.request")) {
+        chats++;
+        const end = ends.get(e.seq);
+        const provider = typeof e.meta.provider === "string" ? e.meta.provider : undefined;
+        const model = end?.meta.model ?? e.meta.model;
+        const base = [
+            ...str("gen_ai.operation.name", "chat"),
+            ...str("gen_ai.provider.name", provider ? (PROVIDER_NAMES[provider] ?? provider) : undefined),
+            ...str("gen_ai.response.model", model),
+        ];
+        if (!end)
+            continue;
+        collect(durations, [...base, ...str("error.type", callError(end))], Math.max(0, Date.parse(end.ts) - Date.parse(e.ts)));
+        const usage = end.meta.usage && typeof end.meta.usage === "object"
+            ? tokensOf(end.meta.usage)
+            : null;
+        if (!usage)
+            continue;
+        collect(tokens, [...base, ...str("gen_ai.token.type", "input")], usage.input + usage.cache_read + usage.cache_write);
+        collect(tokens, [...base, ...str("gen_ai.token.type", "output")], usage.output);
+    }
+    const tools = events.filter((e) => e.kind === "tool.call").length;
+    const micro = new Map();
+    for (const c of cost?.calls ?? [])
+        if (typeof c.micro_usd === "number")
+            micro.set(c.model ?? "", (micro.get(c.model ?? "") ?? 0) + c.micro_usd);
+    const sum = (dataPoints) => ({
+        aggregationTemporality: 2,
+        isMonotonic: true,
+        dataPoints,
+    });
+    const metrics = [
+        {
+            name: "gen_ai.client.token.usage",
+            description: "Tokens used by the session's model calls",
+            unit: "{token}",
+            histogram: {
+                aggregationTemporality: 2,
+                dataPoints: histogram(tokens, TOKEN_BOUNDS, 1, times),
+            },
+        },
+        {
+            name: "gen_ai.client.operation.duration",
+            description: "The session's model calls, from the request to the answer",
+            unit: "s",
+            histogram: {
+                aggregationTemporality: 2,
+                dataPoints: histogram(durations, MS_BOUNDS, 1000, times),
+            },
+        },
+        {
+            name: "blackbox.session.calls",
+            description: "Calls the gateway witnessed",
+            unit: "{call}",
+            sum: sum([
+                ["chat", chats],
+                ["tool", tools],
+            ]
+                .filter(([, n]) => n > 0)
+                .map(([type, n]) => ({
+                attributes: str("blackbox.call.type", type),
+                ...times,
+                asInt: String(n),
+            }))),
+        },
+        {
+            name: "blackbox.session.cost",
+            description: "The session's priced cost in micro-USD (spec/cost.md)",
+            unit: "{micro_USD}",
+            sum: sum([...micro].map(([model, n]) => ({
+                attributes: str("gen_ai.response.model", model || undefined),
+                ...times,
+                asInt: String(n),
+            }))),
+        },
+    ].filter((m) => (m.histogram ?? m.sum)?.dataPoints.length);
+    return {
+        resourceMetrics: [
+            {
+                resource: resourceOf(session),
+                scopeMetrics: [{ scope: { name: "zanii-blackbox", version: "1" }, metrics }],
+            },
+        ],
+    };
+}
+/** A finding's severity as an OpenTelemetry severity number. */
+const SEVERITY = {
+    advisory: 9,
+    caution: 13,
+    warning: 13,
+    medium: 13,
+    high: 17,
+    critical: 21,
+};
+/**
+ * spec/otlp.md §6: a session's findings as OTLP/JSON logs, one record each (event name
+ * `blackbox.finding`, the code as its body), in the session's trace (the root span's ids).
+ */
+export function toOtlpLogs(lines) {
+    const events = lines.map((l) => JSON.parse(l));
+    const first = events[0];
+    if (!first)
+        return { resourceLogs: [] };
+    const session = first.session_id;
+    const logRecords = events
+        .filter((e) => e.kind === "finding")
+        .map((e) => {
+        const code = String(e.meta.code);
+        const severity = typeof e.meta.severity === "string" ? e.meta.severity : undefined;
+        return {
+            timeUnixNano: nanos(e.ts),
+            ...(severity && SEVERITY[severity] ? { severityNumber: SEVERITY[severity] } : {}),
+            ...(severity ? { severityText: severity } : {}),
+            body: { stringValue: code },
+            attributes: [
+                ...str("blackbox.finding.code", code),
+                ...str("blackbox.severity", severity),
+                ...str("blackbox.fault", e.meta.fault),
+                ...list("blackbox.owasp_asi", taxonomyOf(code).owasp.map((o) => o.id)),
+                ...list("blackbox.mitre_atlas", taxonomyOf(code).atlas.map((a) => a.id)),
+                ...int("blackbox.seq", e.seq),
+                ...str("gen_ai.conversation.id", session),
+            ],
+            traceId: hex(session, 32),
+            spanId: hex(`${session}:0`, 16),
+            eventName: "blackbox.finding",
+        };
+    });
+    return {
+        resourceLogs: [
+            {
+                resource: resourceOf(session),
+                scopeLogs: [{ scope: { name: "zanii-blackbox", version: "1" }, logRecords }],
+            },
+        ],
+    };
+}
+/**
+ * spec/otlp.md §7: the gateway's own spans for a session's forwarded requests, in the session's
+ * trace: each a SERVER span `forward {provider}`, child of the record's model-call span (§1, §2).
+ */
+export function gatewayTrace(sessionId, calls) {
+    const spans = calls.map((c) => {
+        const error = c.reason !== undefined
+            ? c.reason
+            : typeof c.status === "number" && c.status >= 400
+                ? String(c.status)
+                : undefined;
+        return {
+            traceId: hex(sessionId, 32),
+            spanId: hex(`${sessionId}:gateway:${c.request_seq}`, 16),
+            parentSpanId: hex(`${sessionId}:${c.request_seq}`, 16),
+            name: `forward ${c.provider}`,
+            kind: 2,
+            startTimeUnixNano: c.start_unix_nano,
+            endTimeUnixNano: c.end_unix_nano,
+            attributes: [
+                ...str("gen_ai.provider.name", PROVIDER_NAMES[c.provider] ?? c.provider),
+                ...str("gen_ai.response.model", c.model),
+                ...str("blackbox.provider", c.provider),
+                ...int("http.response.status_code", c.status),
+                ...str("error.type", error),
+                ...int("blackbox.recorded_seq", c.request_seq),
+                ...int("blackbox.response_seq", c.response_seq),
+                ...str("gen_ai.conversation.id", sessionId),
+            ],
+            ...(error !== undefined ? { status: { code: 2, message: error } } : {}),
+        };
+    });
+    return {
+        resourceSpans: [
+            {
+                resource: resourceOf(sessionId, "zanii-blackbox-gateway"),
+                scopeSpans: [{ scope: { name: "zanii-blackbox-gateway", version: "1" }, spans }],
+            },
+        ],
+    };
+}

package/dist/otlp/protobuf.d.ts

@@ -2,3 +2,10 @@
 export declare function otlpFromProtobuf(buf: Uint8Array): {
     resourceSpans: object[];
 };
+export type Signal = "traces" | "metrics" | "logs";
+/** spec/otlp.md §6: an Export{Trace,Metrics,Logs}ServiceRequest's bytes as OTLP/JSON, every field
+ * read (64-bit integers as strings, ids as hex). Throws on bad bytes. */
+export declare function otlpDecode(buf: Uint8Array, signal: Signal): Record<string, unknown>;
+/** spec/otlp.md §6: OTLP/JSON as the Export{Trace,Metrics,Logs}ServiceRequest's protobuf bytes,
+ * the fields in number order, so both SDKs give the same bytes (spec/vectors/otlp-protobuf-out.json). */
+export declare function otlpEncode(doc: object, signal: Signal): Uint8Array;