@autter/otlp-ingester 1.0.0 → 1.2.0

package/dist/llm.js

@@ -0,0 +1,131 @@
+import { estimateLlmCostUsd } from "./llm-pricing.js";
+/**
+ * LLM-call extraction from OTLP spans. Three attribute families are
+ * recognised, so most stacks work with zero Autter-specific code:
+ *
+ * 1. OpenTelemetry GenAI semantic conventions (`gen_ai.*`) — emitted by
+ *    OpenLLMetry, OpenLIT, the official OTel instrumentations, and
+ *    @autter/runtime-node's own `withLlmCall`/`trackLlmCall` helpers.
+ * 2. Vercel AI SDK telemetry (`ai.*` spans from `experimental_telemetry`).
+ *    Only the provider-level `.doGenerate`/`.doStream`/`.doEmbed` spans
+ *    become calls — the umbrella `ai.generateText` span aggregates its
+ *    children's usage and would double-count.
+ * 3. Autter extension attributes (`autter.llm.cost_usd`, `autter.user_id`,
+ *    `autter.session_id`) for exact costs and user attribution.
+ *
+ * Cost precedence: reported (`autter.llm.cost_usd`/`gen_ai.usage.cost`) →
+ * estimated from the built-in pricing table → 0 with costSource "none".
+ */
+const VERCEL_PROVIDER_CALL = /\.do(Generate|Stream|Embed)$/;
+function numAttr(attrs, ...keys) {
+    for (const key of keys) {
+        const raw = attrs.get(key);
+        if (raw === undefined)
+            continue;
+        const value = Number(raw);
+        if (Number.isFinite(value))
+            return value;
+    }
+    return null;
+}
+function strAttr(attrs, ...keys) {
+    for (const key of keys) {
+        const raw = attrs.get(key);
+        if (raw !== undefined && raw !== "")
+            return raw;
+    }
+    return null;
+}
+/** "ai.generateText.doGenerate" → "generateText"; "ai.embed.doEmbed" → "embed". */
+function vercelOperation(name) {
+    const parts = name.split(".");
+    return parts.length >= 2 ? (parts[1] ?? "") : "";
+}
+/**
+ * Returns the LLM-call row for a span, or null when the span is not an LLM
+ * provider call. Never throws — attribute soup from arbitrary SDKs must not
+ * take down the ingest path.
+ */
+export function extractLlmCall(attrs, facts) {
+    const isVercelSpan = facts.name.startsWith("ai.");
+    // Umbrella Vercel spans (ai.generateText, ai.streamText, ai.toolCall, …)
+    // are skipped: usage lives on (and aggregates) their .do* children.
+    if (isVercelSpan && !VERCEL_PROVIDER_CALL.test(facts.name))
+        return null;
+    const model = strAttr(attrs, "gen_ai.response.model", "gen_ai.request.model", "ai.response.model", "ai.model.id");
+    // gen_ai.provider.name is the current semconv key, gen_ai.system the
+    // pre-1.37 one. Vercel's ai.model.provider looks like "openai.chat" —
+    // keep the provider segment, the mode is already in `operation`.
+    const provider = strAttr(attrs, "gen_ai.provider.name", "gen_ai.system") ??
+        strAttr(attrs, "ai.model.provider")?.toLowerCase().split(".")[0] ??
+        null;
+    // A call must identify at least a model or a provider system — this is
+    // what keeps ordinary spans (HTTP, DB, tool calls) out of the table.
+    if (!model && !provider)
+        return null;
+    const inputTokens = numAttr(attrs, "gen_ai.usage.input_tokens", "gen_ai.usage.prompt_tokens", "ai.usage.promptTokens", "ai.usage.inputTokens", 
+    // Vercel .doEmbed spans report a single total under ai.usage.tokens.
+    "ai.usage.tokens") ?? 0;
+    const outputTokens = numAttr(attrs, "gen_ai.usage.output_tokens", "gen_ai.usage.completion_tokens", "ai.usage.completionTokens", "ai.usage.outputTokens") ?? 0;
+    const reportedCost = numAttr(attrs, "autter.llm.cost_usd", "gen_ai.usage.cost", "ai.usage.cost");
+    let costUsd = 0;
+    let costSource = "none";
+    if (reportedCost !== null && reportedCost >= 0) {
+        costUsd = reportedCost;
+        costSource = "reported";
+    }
+    else {
+        const estimated = estimateLlmCostUsd(model ?? "", inputTokens, outputTokens);
+        if (estimated !== null) {
+            costUsd = estimated;
+            costSource = "estimated";
+        }
+    }
+    const operation = strAttr(attrs, "gen_ai.operation.name") ??
+        (isVercelSpan ? vercelOperation(facts.name) : "");
+    return {
+        service: facts.service,
+        environment: facts.environment,
+        release: facts.release,
+        traceId: facts.traceId,
+        spanId: facts.spanId,
+        provider: (provider ?? "").slice(0, 100),
+        model: (model ?? "").slice(0, 200),
+        operation: operation.slice(0, 100),
+        inputTokens: Math.max(0, Math.round(inputTokens)),
+        outputTokens: Math.max(0, Math.round(outputTokens)),
+        costUsd,
+        costSource,
+        durationMs: facts.durationMs,
+        status: facts.isError ? "error" : "ok",
+        errorType: facts.isError
+            ? (strAttr(attrs, "error.type") ?? facts.exceptionType ?? "Error").slice(0, 200)
+            : "",
+        userId: (strAttr(attrs, "autter.user_id", "ai.telemetry.metadata.userId", "ai.telemetry.metadata.user_id", "enduser.id", "user.id") ?? "").slice(0, 200),
+        sessionId: (strAttr(attrs, "autter.session_id", "ai.telemetry.metadata.sessionId", "session.id") ?? "").slice(0, 200),
+        attributes: llmAttributeSubset(attrs),
+        startedAt: facts.startedAt,
+    };
+}
+/**
+ * Drill-down context stored as JSON alongside the typed columns: only the
+ * LLM-relevant attribute families, size-capped so a prompt accidentally
+ * stuffed into an attribute can't bloat the row.
+ */
+function llmAttributeSubset(attrs) {
+    const subset = {};
+    let size = 0;
+    for (const [key, value] of attrs) {
+        if (!key.startsWith("gen_ai.") &&
+            !key.startsWith("ai.") &&
+            !key.startsWith("autter.llm.")) {
+            continue;
+        }
+        const trimmed = value.length > 500 ? `${value.slice(0, 500)}…` : value;
+        size += key.length + trimmed.length;
+        if (size > 4000)
+            break;
+        subset[key] = trimmed;
+    }
+    return Object.keys(subset).length > 0 ? subset : null;
+}

package/dist/migrations.js

@@ -76,6 +76,43 @@ export const MIGRATIONS = [
 				MODIFY COLUMN attributes String DEFAULT '{}' CODEC(ZSTD(1))`,
         ],
     },
+    // LLM observability: one row per provider call (model, tokens, cost,
+    // latency, user) extracted from GenAI/Vercel-AI/Autter span attributes.
+    // TTL is fixed at the 90-day default here; deployments that override
+    // LLM_CALL_TTL_DAYS get it applied on fresh databases via the baseline.
+    {
+        id: "0004-llm-calls",
+        statements: [
+            `CREATE TABLE IF NOT EXISTS {db}.runtime_llm_calls (
+				org_id          String,
+				repository_id   String,
+				service         LowCardinality(String),
+				environment     LowCardinality(String),
+				release         String DEFAULT '',
+				trace_id        String DEFAULT '',
+				span_id         String DEFAULT '',
+				provider        LowCardinality(String) DEFAULT '',
+				model           LowCardinality(String) DEFAULT '',
+				operation       LowCardinality(String) DEFAULT '',
+				input_tokens    UInt64 DEFAULT 0,
+				output_tokens   UInt64 DEFAULT 0,
+				cost_usd        Float64 DEFAULT 0,
+				cost_source     LowCardinality(String) DEFAULT 'none',
+				duration_ms     Float64 DEFAULT 0,
+				status          LowCardinality(String) DEFAULT 'ok',
+				error_type      String DEFAULT '',
+				user_id         String DEFAULT '',
+				session_id      String DEFAULT '',
+				attributes      String DEFAULT '{}' CODEC(ZSTD(1)),
+				started_at      DateTime64(3, 'UTC'),
+				ingested_at     DateTime64(3, 'UTC') DEFAULT now64(3)
+			)
+			ENGINE = MergeTree
+			PARTITION BY toDate(started_at)
+			ORDER BY (org_id, repository_id, started_at)
+			TTL toDateTime(started_at) + INTERVAL 90 DAY`,
+        ],
+    },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.js

@@ -43,6 +43,27 @@ const TYPE_TO_ERROR_TYPE = {
     unhandled_rejection: "UnhandledRejection",
     message: "Message",
 };
+// Content-level gate for the free-form `context` bag. The schema whitelist
+// above is structural; this masks obvious PII/secrets inside whatever a
+// (possibly outdated) SDK still sends: values under sensitive-looking keys
+// and email-shaped strings — mirroring redactAttributes() in
+// @autter/runtime-node and redactContext() in @autter/runtime-browser.
+const SENSITIVE_KEY_RE = /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i;
+const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
+const REDACTED = "[redacted]";
+function scrubContext(context) {
+    const out = {};
+    for (const [key, value] of Object.entries(context)) {
+        if (value === undefined || value === null)
+            continue;
+        out[key] = SENSITIVE_KEY_RE.test(key)
+            ? REDACTED
+            : typeof value === "string"
+                ? value.replace(EMAIL_VALUE_RE, REDACTED)
+                : value;
+    }
+    return out;
+}
 /** Default severity per event type when the SDK doesn't say. */
 const TYPE_TO_SEVERITY = {
     exception: "error",
@@ -52,12 +73,14 @@ const TYPE_TO_SEVERITY = {
 export function normalizeBrowserPayload(payload) {
     const occurrences = [];
     const rollups = new Map();
-    function bumpRollup(route, occurredAt, field) {
+    function bumpRollup(route, occurredAt, counts) {
         const bucketAt = new Date(Math.floor(occurredAt.getTime() / 60_000) * 60_000);
         const key = `${route} ${bucketAt.getTime()}`;
         const existing = rollups.get(key);
         if (existing) {
-            existing[field] += 1;
+            existing.requestCount += counts.requestCount ?? 0;
+            existing.errorCount += counts.errorCount ?? 0;
+            existing.sessionCount += counts.sessionCount ?? 0;
             return;
         }
         rollups.set(key, {
@@ -66,16 +89,16 @@ export function normalizeBrowserPayload(payload) {
             release: payload.release ?? null,
             route,
             bucketAt,
-            requestCount: field === "requestCount" ? 1 : 0,
-            errorCount: 0,
+            requestCount: counts.requestCount ?? 0,
+            errorCount: counts.errorCount ?? 0,
             durationSumMs: 0,
-            sessionCount: field === "sessionCount" ? 1 : 0,
+            sessionCount: counts.sessionCount ?? 0,
         });
     }
     for (const event of payload.events) {
         const occurredAt = new Date(event.timestamp);
         if (event.type === "session_start") {
-            bumpRollup("", occurredAt, "sessionCount");
+            bumpRollup("", occurredAt, { sessionCount: 1 });
             continue;
         }
         // Coarse usage counters: track_event("checkout_opened") becomes a
@@ -83,12 +106,21 @@ export function normalizeBrowserPayload(payload) {
         if (event.type === "track_event") {
             const name = (event.name ?? event.message ?? "").slice(0, 200);
             if (name)
-                bumpRollup(`event:${name}`, occurredAt, "requestCount");
+                bumpRollup(`event:${name}`, occurredAt, { requestCount: 1 });
             continue;
         }
+        const severity = asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error");
+        // The browser has no request stream, so error/message events are the
+        // only usage signal for many pages — count each as an event (and, for
+        // fatal/error severity, an error event). Without this, a page that
+        // only reports errors groups issues whose dashboards all read 0.
+        bumpRollup(event.route ? (event.route.split("?")[0] ?? "") : "", occurredAt, {
+            requestCount: 1,
+            errorCount: severity === "fatal" || severity === "error" ? 1 : 0,
+        });
         occurrences.push({
             source: "browser",
-            severity: asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error"),
+            severity,
             service: payload.service,
             environment: payload.environment,
             release: payload.release ?? null,
@@ -104,7 +136,7 @@ export function normalizeBrowserPayload(payload) {
                 ...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
                 ...(event.line !== undefined ? { line: event.line } : {}),
                 ...(event.column !== undefined ? { column: event.column } : {}),
-                ...(event.context ? { context: event.context } : {}),
+                ...(event.context ? { context: scrubContext(event.context) } : {}),
             },
             occurredAt,
         });

package/dist/normalize-otlp.d.ts

@@ -1,4 +1,4 @@
-import { type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
+import { type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
 /**
  * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields
  * we read (the full OTLP schema is large and versioned; unknown fields pass
@@ -65,6 +65,7 @@ export interface OtlpMetricsRequest {
                 unit?: string;
                 histogram?: {
                     dataPoints?: OtlpDataPoint[];
+                    aggregationTemporality?: string | number;
                 };
                 sum?: {
                     dataPoints?: OtlpDataPoint[];
@@ -77,6 +78,7 @@ export interface NormalizedTraces {
     occurrences: RuntimeOccurrenceInput[];
     spans: RuntimeSpanRow[];
     metricPoints: RuntimeMetricPoint[];
+    llmCalls: RuntimeLlmCall[];
     spanCount: number;
 }
 export declare function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces;

package/dist/normalize-otlp.js

@@ -1,3 +1,5 @@
+import { normalizeRoute } from "./fingerprint.js";
+import { extractLlmCall } from "./llm.js";
 import { asSeverity, } from "./types.js";
 function attrMap(attributes) {
     const map = new Map();
@@ -54,6 +56,7 @@ function resourceInfo(resource) {
             attrs.get("deployment.environment") ??
             "production",
         release: attrs.get("service.version") ?? null,
+        metricsWired: attrs.get("autter.metrics_wired") === "true",
     };
 }
 function routeOf(attrs) {
@@ -93,6 +96,7 @@ const MAX_SPANS_PER_REQUEST = 5000;
 export function normalizeTraces(request) {
     const occurrences = [];
     const spans = [];
+    const llmCalls = [];
     const rollups = new Map();
     let spanCount = 0;
     for (const resourceSpan of request.resourceSpans ?? []) {
@@ -128,6 +132,7 @@ export function normalizeTraces(request) {
                 });
                 // Error occurrences: one per exception event; if the span is
                 // errored without exception events, one from the span status.
+                const occurrencesBefore = occurrences.length;
                 const exceptionEvents = (span.events ?? []).filter((event) => event.name === "exception");
                 for (const event of exceptionEvents) {
                     const eventAttrs = attrMap(event.attributes);
@@ -171,14 +176,39 @@ export function normalizeTraces(request) {
                         occurredAt: startedAt,
                     });
                 }
+                // LLM provider calls (GenAI semconv, Vercel AI SDK telemetry,
+                // or Autter's own helpers) become usage/cost rows. The first
+                // exception event's type rides along so failed calls keep the
+                // provider error class even without an `error.type` attribute.
+                const llmCall = extractLlmCall(attrs, {
+                    name: span.name ?? "",
+                    traceId: span.traceId ?? "",
+                    spanId: span.spanId ?? "",
+                    isError,
+                    exceptionType: exceptionEvents[0]
+                        ? (attrMap(exceptionEvents[0].attributes).get("exception.type") ??
+                            null)
+                        : null,
+                    durationMs,
+                    startedAt,
+                    service: resource.service,
+                    environment: resource.environment,
+                    release: resource.release,
+                });
+                if (llmCall)
+                    llmCalls.push(llmCall);
                 // Server spans fold into 1-minute usage rollups so traffic is
-                // tracked even when the metrics pipeline isn't wired.
-                if (kind === "server") {
+                // tracked even when the metrics pipeline isn't wired. Spans
+                // exported by error-linked tail retention are skipped: the SDK
+                // ships those at ~100% alongside a metrics pipeline that already
+                // counts every request, so folding them in would double-count
+                // erroring routes.
+                if (kind === "server" && attrs.get("autter.tail_retained") !== "true") {
                     addToRollup(rollups, {
                         service: resource.service,
                         environment: resource.environment,
                         release: resource.release,
-                        route: route ?? "",
+                        route: normalizeRoute(route),
                         bucketAt: minuteBucket(startedAt),
                         requestCount: 1,
                         errorCount: isError ? 1 : 0,
@@ -186,10 +216,37 @@ export function normalizeTraces(request) {
                         sessionCount: 0,
                     });
                 }
+                else {
+                    // Occurrences on non-server spans (captureException's internal
+                    // error spans, workers, consumers) have no request rollup to
+                    // ride — count each as one event / one error event, or a
+                    // service whose errors arrive outside HTTP handlers groups
+                    // issues while every event counter stays 0. Server spans are
+                    // excluded: their request rollup above already represents them.
+                    for (const occ of occurrences.slice(occurrencesBefore)) {
+                        addToRollup(rollups, {
+                            service: occ.service,
+                            environment: occ.environment,
+                            release: occ.release,
+                            route: occ.route ?? "",
+                            bucketAt: minuteBucket(occ.occurredAt),
+                            requestCount: 1,
+                            errorCount: occ.severity === "fatal" || occ.severity === "error" ? 1 : 0,
+                            durationSumMs: 0,
+                            sessionCount: 0,
+                        });
+                    }
+                }
             }
         }
     }
-    return { occurrences, spans, metricPoints: [...rollups.values()], spanCount };
+    return {
+        occurrences,
+        spans,
+        metricPoints: [...rollups.values()],
+        llmCalls,
+        spanCount,
+    };
 }
 function minuteBucket(date) {
     return new Date(Math.floor(date.getTime() / 60_000) * 60_000);
@@ -222,6 +279,16 @@ const HTTP_DURATION_INSTRUMENTS = {
     "http.server.duration": 1,
     "http.server.request.duration": 1000,
 };
+/**
+ * Cumulative histograms report lifetime totals on every export; adding them
+ * into a SummingMergeTree would re-count all past requests each interval.
+ * Only deltas are summable — cumulative senders are skipped and covered by
+ * the span-fed rollup fallback instead. (Enum arrives as a number or an
+ * `AGGREGATION_TEMPORALITY_*` string depending on the serialiser.)
+ */
+function isCumulativeTemporality(t) {
+    return t === 2 || t === "AGGREGATION_TEMPORALITY_CUMULATIVE";
+}
 export function normalizeMetrics(request) {
     const rollups = new Map();
     for (const resourceMetric of request.resourceMetrics ?? []) {
@@ -233,6 +300,9 @@ export function normalizeMetrics(request) {
                     : undefined;
                 if (multiplier === undefined)
                     continue;
+                if (isCumulativeTemporality(metric.histogram?.aggregationTemporality)) {
+                    continue;
+                }
                 for (const dataPoint of metric.histogram?.dataPoints ?? []) {
                     const attrs = attrMap(dataPoint.attributes);
                     const statusCode = statusCodeOf(attrs);
@@ -243,7 +313,11 @@ export function normalizeMetrics(request) {
                         service: resource.service,
                         environment: resource.environment,
                         release: resource.release,
-                        route: routeOf(attrs) ?? "",
+                        // Same normalization as the span-fed rollups: emitters
+                        // are supposed to put route templates in `http.route`,
+                        // but some put raw paths there — keep the key space
+                        // bounded and consistent across both feeds.
+                        route: normalizeRoute(routeOf(attrs)),
                         bucketAt: minuteBucket(nanosToDate(dataPoint.timeUnixNano)),
                         requestCount: count,
                         errorCount: statusCode !== null && statusCode >= 500 ? count : 0,

package/dist/otlp-proto.js

@@ -64,7 +64,7 @@ message NumberDataPoint {
   sfixed64 as_int = 6;
   repeated KeyValue attributes = 7;
 }
-message Histogram { repeated HistogramDataPoint data_points = 1; }
+message Histogram { repeated HistogramDataPoint data_points = 1; int32 aggregation_temporality = 2; }
 message HistogramDataPoint {
   fixed64 time_unix_nano = 3;
   fixed64 count = 4;

package/dist/server.d.ts

@@ -1,8 +1,11 @@
 import { type Express } from "express";
 import { ClickHouseStore } from "./clickhouse.js";
 import type { IngesterConfig } from "./config.js";
+import { SinkForwarder } from "./sink.js";
 export interface IngesterApp {
     app: Express;
     store: ClickHouseStore;
+    /** Present when AUTTER_SINK_URL is configured. */
+    sink: SinkForwarder | null;
 }
 export declare function createIngesterApp(config: IngesterConfig): IngesterApp;

package/dist/server.js

@@ -1,13 +1,17 @@
-import { randomUUID } from "node:crypto";
 import express from "express";
 import { KeyResolver, RateLimiter } from "./auth.js";
 import { ClickHouseStore } from "./clickhouse.js";
-import { deriveFields, fingerprintOccurrence } from "./fingerprint.js";
+import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerprint.js";
 import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
 import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
+import { SinkForwarder } from "./sink.js";
 export function createIngesterApp(config) {
     const store = new ClickHouseStore(config);
+    // Fingerprinted occurrences feed the consumer's issue grouping, metric
+    // points feed the request/error-rate rollups, LLM calls feed spend
+    // watching. Delivery is at-least-once with bounded retries — see sink.ts.
+    const sink = config.sinkUrl ? new SinkForwarder(config) : null;
     const keys = new KeyResolver(config);
     const serverRateLimiter = new RateLimiter(config.rateLimitPerMinute);
     const clientRateLimiter = new RateLimiter(config.clientRateLimitPerMinute);
@@ -46,16 +50,19 @@ export function createIngesterApp(config) {
         next();
     });
     app.get("/healthz", async (_req, res) => {
+        const sinkStats = sink ? { sink: sink.stats() } : {};
         if (!store.configured) {
-            res.status(200).json({ ok: true, clickhouse: "unconfigured" });
+            res.status(200).json({ ok: true, clickhouse: "unconfigured", ...sinkStats });
             return;
         }
         try {
             const ok = await store.ping();
-            res.status(ok ? 200 : 503).json({ ok, clickhouse: ok ? "up" : "down" });
+            res
+                .status(ok ? 200 : 503)
+                .json({ ok, clickhouse: ok ? "up" : "down", ...sinkStats });
         }
         catch {
-            res.status(503).json({ ok: false, clickhouse: "down" });
+            res.status(503).json({ ok: false, clickhouse: "down", ...sinkStats });
         }
     });
     /** Auth + scope + rate limit; returns null (response sent) on failure. */
@@ -103,38 +110,20 @@ export function createIngesterApp(config) {
         }
         return ctx;
     }
-    function fingerprintAll(inputs) {
-        return inputs.map((input) => ({
-            ...input,
-            occurrenceId: randomUUID(),
-            fingerprint: fingerprintOccurrence(input),
-            ...deriveFields(input),
-        }));
-    }
-    /** Best-effort forward of fingerprinted occurrences for issue grouping. */
-    function forwardToSink(ctx, occurrences) {
-        if (!config.sinkUrl || occurrences.length === 0)
-            return;
-        void fetch(config.sinkUrl, {
-            method: "POST",
-            headers: {
-                "content-type": "application/json",
-                ...(config.sinkToken
-                    ? { authorization: `Bearer ${config.sinkToken}` }
-                    : {}),
-            },
-            body: JSON.stringify({
-                version: 1,
-                orgId: ctx.orgId,
-                repositoryId: ctx.repositoryId,
-                occurrences: occurrences.map((o) => ({
-                    ...o,
-                    occurredAt: o.occurredAt.toISOString(),
-                })),
-            }),
-            signal: AbortSignal.timeout(10_000),
-        }).catch((err) => {
-            console.warn("sink forward failed (non-fatal):", err?.message ?? err);
+    /** Ids are content-derived (occurrenceIdFor), NOT random: an exporter
+     * that retries a batch — after a 503 from a partially-failed ClickHouse
+     * write, or when only our 2xx got lost — must produce the same ids, so
+     * the sink consumer's per-occurrence dedupe holds across transport
+     * retries and duplicated ClickHouse rows stay identifiable. */
+    function fingerprintAll(ctx, inputs) {
+        return inputs.map((input, index) => {
+            const fingerprint = fingerprintOccurrence(input);
+            return {
+                ...input,
+                occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index),
+                fingerprint,
+                ...deriveFields(input),
+            };
         });
     }
     function storageError(res, err) {
@@ -167,20 +156,29 @@ export function createIngesterApp(config) {
         else {
             request = req.body;
         }
-        const { occurrences, spans, metricPoints } = normalizeTraces(request);
-        const fingerprinted = fingerprintAll(occurrences);
+        const { occurrences, spans, metricPoints, llmCalls } = normalizeTraces(request);
+        const fingerprinted = fingerprintAll(ctx, occurrences);
+        // ClickHouse has no cross-table transaction, so these four inserts can
+        // partially commit. Recovery boundary: any failure → 503 → the exporter
+        // retries the whole batch. Deterministic occurrence ids make the retry
+        // idempotent downstream (consumer dedupes per id; duplicate ClickHouse
+        // rows share an id, and the consumer's reconciler counts distinct ids),
+        // and nothing reaches the sink queue unless every insert succeeded —
+        // signals persisted by a partial write are picked up by the consumer's
+        // ClickHouse reconciliation instead.
         try {
             await Promise.all([
                 store.insertOccurrences(ctx, fingerprinted),
                 store.insertSpans(ctx, spans),
                 store.insertMetricPoints(ctx, metricPoints),
+                store.insertLlmCalls(ctx, llmCalls),
             ]);
         }
         catch (err) {
             storageError(res, err);
             return;
         }
-        forwardToSink(ctx, fingerprinted);
+        sink?.enqueue(ctx, fingerprinted, metricPoints, llmCalls);
         otlpSuccess(req, res);
     });
     app.post("/v1/metrics", async (req, res) => {
@@ -208,6 +206,7 @@ export function createIngesterApp(config) {
             storageError(res, err);
             return;
         }
+        sink?.enqueue(ctx, [], metricPoints);
         otlpSuccess(req, res);
     });
     app.post("/v1/browser", async (req, res) => {
@@ -234,7 +233,7 @@ export function createIngesterApp(config) {
             return;
         }
         const { occurrences, metricPoints } = normalizeBrowserPayload(parsed.data);
-        const fingerprinted = fingerprintAll(occurrences);
+        const fingerprinted = fingerprintAll(ctx, occurrences);
         try {
             await Promise.all([
                 store.insertOccurrences(ctx, fingerprinted),
@@ -245,7 +244,7 @@ export function createIngesterApp(config) {
             storageError(res, err);
             return;
         }
-        forwardToSink(ctx, fingerprinted);
+        sink?.enqueue(ctx, fingerprinted, metricPoints);
         res.status(202).json({ accepted: fingerprinted.length });
     });
     // Body-parser errors (oversized/malformed JSON) → clean 4xx, not a stack.
@@ -263,5 +262,5 @@ export function createIngesterApp(config) {
         console.error("unhandled error:", err);
         res.status(500).json({ error: "internal error" });
     });
-    return { app, store };
+    return { app, store, sink };
 }