@autter/otlp-ingester 1.0.0 → 1.2.0

package/README.md

@@ -8,7 +8,7 @@ one per-repo signal model, fingerprints errors, and writes ClickHouse.
 
 | Route | Payload | Purpose |
 | --- | --- | --- |
-| `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups |
+| `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups; GenAI spans → `runtime_llm_calls` |
 | `POST /v1/metrics` | OTLP/JSON `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups |
 | `POST /v1/browser` | Browser payload `version: 1` | Errors/rejections → occurrences; session pings → rollups |
 | `GET /healthz` | — | Liveness + ClickHouse reachability |
@@ -54,12 +54,33 @@ The validator webhook may return the same extra fields:
 | `AUTTER_INGEST_KEYS` | — | JSON: `[{"key":"...","orgId":"...","repositoryId":"..."}]` |
 | `AUTTER_KEY_VALIDATOR_URL` | — | Webhook: `POST {key}` → `{orgId, repositoryId}` (60 s cache) |
 | `AUTTER_KEY_VALIDATOR_TOKEN` | — | Bearer token sent to the validator |
-| `AUTTER_SINK_URL` | — | Webhook receiving fingerprinted occurrences for issue grouping |
+| `AUTTER_SINK_URL` | — | Issue-grouping webhook; at-least-once (`docs/ARCHITECTURE.md`) |
 | `AUTTER_SINK_TOKEN` | — | Bearer token sent to the sink |
+| `SINK_MAX_ATTEMPTS` | `12` | Delivery attempts per batch (1–60 s backoff) |
+| `SINK_MAX_BUFFERED_BATCHES` | `1000` | Retry buffer cap; oldest drops are logged |
+| `SINK_MAX_BUFFERED_MB` | `64` | Sink retry buffer cap (memory) |
 | `MAX_BODY_BYTES` | `1048576` | Request body cap |
 | `RATE_LIMIT_PER_MINUTE` | `300` | Per-key fixed window (server keys) |
 | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) |
 | `OCCURRENCE_TTL_DAYS` / `SPAN_TTL_DAYS` / `METRICS_TTL_DAYS` | `14` / `7` / `90` | ClickHouse TTLs (applied at table creation) |
+| `LLM_CALL_TTL_DAYS` | `90` | Retention for `runtime_llm_calls` rows |
+
+## LLM / GenAI calls
+
+Spans following the [OTel GenAI semconv](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
+(`gen_ai.*` attributes — emitted by `withLlmCall` in `@autter/runtime-node`,
+the Vercel AI SDK with telemetry enabled, and the GenAI instrumentations for
+Python/Go/etc.) are recognised automatically on `/v1/traces` and additionally
+stored per call in `runtime_llm_calls`: provider, model, operation, input/
+output tokens, duration, ok/error status (with the provider exception type
+in `error_type` for failed calls), and a USD cost. The cost is taken
+from the `autter.llm.cost_usd` span attribute when reported; otherwise it's
+estimated from the built-in price table in `src/llm-pricing.ts`
+(`cost_source` records which: `reported` / `estimated` / `unpriced`). An
+opaque calling-user id is read from `autter.user_id` (or the AI SDK's
+`metadata.userId`). The Vercel AI SDK's outer `ai.generateText`/`ai.streamText`
+spans are not counted — their inner `.doGenerate`/`.doEmbed` call spans carry
+the usage, and counting both would double the tokens.
 
 ## Schema migrations
 

package/dist/clickhouse.d.ts

@@ -1,6 +1,6 @@
 import type { IngesterConfig } from "./config.js";
 import { type Migration } from "./migrations.js";
-import type { IngestContext, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
+import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
 export declare class ClickHouseStore {
     private readonly config;
     private client;
@@ -22,6 +22,7 @@ export declare class ClickHouseStore {
     ping(): Promise<boolean>;
     insertOccurrences(ctx: IngestContext, occurrences: RuntimeOccurrence[]): Promise<void>;
     insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
+    insertLlmCalls(ctx: IngestContext, calls: RuntimeLlmCall[]): Promise<void>;
     insertMetricPoints(ctx: IngestContext, points: RuntimeMetricPoint[]): Promise<void>;
     close(): Promise<void>;
 }

package/dist/clickhouse.js

@@ -41,7 +41,7 @@ export class ClickHouseStore {
     }
     schemaStatements() {
         const db = this.config.clickhouseDatabase;
-        const { occurrenceTtlDays, spanTtlDays, metricsTtlDays } = this.config;
+        const { occurrenceTtlDays, spanTtlDays, metricsTtlDays, llmCallTtlDays } = this.config;
         return [
             `CREATE DATABASE IF NOT EXISTS ${db}`,
             `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
@@ -113,6 +113,34 @@ export class ClickHouseStore {
 			PARTITION BY toYYYYMM(bucket_at)
 			ORDER BY (org_id, repository_id, service, environment, release, route, bucket_at)
 			TTL bucket_at + INTERVAL ${metricsTtlDays} DAY`,
+            `CREATE TABLE IF NOT EXISTS ${db}.runtime_llm_calls (
+				org_id          String,
+				repository_id   String,
+				service         LowCardinality(String),
+				environment     LowCardinality(String),
+				release         String DEFAULT '',
+				trace_id        String DEFAULT '',
+				span_id         String DEFAULT '',
+				provider        LowCardinality(String) DEFAULT '',
+				model           LowCardinality(String) DEFAULT '',
+				operation       LowCardinality(String) DEFAULT '',
+				input_tokens    UInt64 DEFAULT 0,
+				output_tokens   UInt64 DEFAULT 0,
+				cost_usd        Float64 DEFAULT 0,
+				cost_source     LowCardinality(String) DEFAULT 'none',
+				duration_ms     Float64 DEFAULT 0,
+				status          LowCardinality(String) DEFAULT 'ok',
+				error_type      String DEFAULT '',
+				user_id         String DEFAULT '',
+				session_id      String DEFAULT '',
+				attributes      String DEFAULT '{}' CODEC(ZSTD(1)),
+				started_at      DateTime64(3, 'UTC'),
+				ingested_at     DateTime64(3, 'UTC') DEFAULT now64(3)
+			)
+			ENGINE = MergeTree
+			PARTITION BY toDate(started_at)
+			ORDER BY (org_id, repository_id, started_at)
+			TTL toDateTime(started_at) + INTERVAL ${llmCallTtlDays} DAY`,
         ];
     }
     /**
@@ -228,6 +256,39 @@ export class ClickHouseStore {
             })),
         });
     }
+    async insertLlmCalls(ctx, calls) {
+        if (calls.length === 0 || !this.configured)
+            return;
+        await this.ensureSchema();
+        await this.getClient().insert({
+            table: this.table("runtime_llm_calls"),
+            format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: calls.map((c) => ({
+                org_id: ctx.orgId,
+                repository_id: ctx.repositoryId,
+                service: c.service,
+                environment: c.environment,
+                release: c.release ?? "",
+                trace_id: c.traceId,
+                span_id: c.spanId,
+                provider: c.provider,
+                model: c.model,
+                operation: c.operation,
+                input_tokens: Math.max(0, Math.round(c.inputTokens)),
+                output_tokens: Math.max(0, Math.round(c.outputTokens)),
+                cost_usd: c.costUsd,
+                cost_source: c.costSource,
+                duration_ms: c.durationMs,
+                status: c.status,
+                error_type: c.errorType,
+                user_id: c.userId,
+                session_id: c.sessionId,
+                attributes: JSON.stringify(c.attributes ?? {}),
+                started_at: c.startedAt.toISOString(),
+            })),
+        });
+    }
     async insertMetricPoints(ctx, points) {
         if (points.length === 0 || !this.configured)
             return;

package/dist/config.d.ts

@@ -22,6 +22,11 @@ export interface IngesterConfig {
     /** Optional webhook receiving fingerprinted occurrences for issue grouping. */
     sinkUrl: string | null;
     sinkToken: string | null;
+    /** Sink delivery attempts per batch before giving up (backoff-capped ~8 min). */
+    sinkMaxAttempts: number;
+    /** Bounds for the in-memory sink retry buffer; oldest batches drop first. */
+    sinkMaxBufferedBatches: number;
+    sinkMaxBufferedMb: number;
     maxBodyBytes: number;
     /** Per-key requests per minute (server keys). */
     rateLimitPerMinute: number;
@@ -31,5 +36,6 @@ export interface IngesterConfig {
     occurrenceTtlDays: number;
     spanTtlDays: number;
     metricsTtlDays: number;
+    llmCallTtlDays: number;
 }
 export declare function loadConfig(): IngesterConfig;

package/dist/config.js

@@ -34,12 +34,20 @@ export function loadConfig() {
         keyValidatorToken: process.env.AUTTER_KEY_VALIDATOR_TOKEN || null,
         sinkUrl: process.env.AUTTER_SINK_URL || null,
         sinkToken: process.env.AUTTER_SINK_TOKEN || null,
+        // 12 attempts with 1s..60s exponential backoff spans ~8 minutes — long
+        // enough to ride out a routine consumer deploy without unbounded memory.
+        sinkMaxAttempts: intEnv("SINK_MAX_ATTEMPTS", 12),
+        sinkMaxBufferedBatches: intEnv("SINK_MAX_BUFFERED_BATCHES", 1000),
+        sinkMaxBufferedMb: intEnv("SINK_MAX_BUFFERED_MB", 64),
         maxBodyBytes: intEnv("MAX_BODY_BYTES", 1024 * 1024),
         rateLimitPerMinute: intEnv("RATE_LIMIT_PER_MINUTE", 300),
         clientRateLimitPerMinute: intEnv("CLIENT_RATE_LIMIT_PER_MINUTE", 120),
         occurrenceTtlDays: intEnv("OCCURRENCE_TTL_DAYS", 14),
         spanTtlDays: intEnv("SPAN_TTL_DAYS", 7),
         metricsTtlDays: intEnv("METRICS_TTL_DAYS", 90),
+        // LLM calls keep the metrics horizon, not the span one — cost trends
+        // need months, and per-call volume is small next to HTTP spans.
+        llmCallTtlDays: intEnv("LLM_CALL_TTL_DAYS", 90),
     };
     if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) {
         throw new Error(`Invalid CLICKHOUSE_DATABASE name: ${config.clickhouseDatabase}`);

package/dist/fingerprint.d.ts

@@ -4,6 +4,28 @@ export declare function normalizeMessage(message: string): string;
 export declare function normalizeRoute(route: string | null): string;
 export declare function normalizeStackFrames(stack: string | null, topN?: number): string[];
 export declare function fingerprintOccurrence(input: RuntimeOccurrenceInput): string;
+/**
+ * Deterministic per-occurrence identity (as opposed to the fingerprint,
+ * which is the per-ISSUE identity shared by every occurrence of a defect).
+ *
+ * The id must be a pure function of the signal, not a fresh UUID per
+ * request: OTLP exporters retry whole batches (after a 503 from a partial
+ * ClickHouse write, or when only the 2xx was lost), and both the ClickHouse
+ * rows and the sink consumer's dedupe ledger key on this id — random ids
+ * would turn every transport retry into a duplicate error downstream.
+ *
+ * Identical signals within one batch stay distinct through their batch
+ * position, which is stable across retries because exporters re-send the
+ * same serialized batch. Residual coalescing risk: two occurrences from
+ * DIFFERENT requests that share the same millisecond, message, and batch
+ * position while carrying neither a traceId nor a sessionId — accepted, as
+ * server signals virtually always carry a traceId and browser signals a
+ * sessionId.
+ */
+export declare function occurrenceIdFor(scope: {
+    orgId: string;
+    repositoryId: string;
+}, input: RuntimeOccurrenceInput, fingerprint: string, batchIndex: number): string;
 /**
  * Derived, aggregation-ready fields, computed from the SAME normalisers the
  * fingerprint hashes — so a stored fingerprint can always be explained by

package/dist/fingerprint.js

@@ -13,12 +13,22 @@ const LONG_HEX_RE = /\b[0-9a-f]{8,}\b/gi;
 // template too, or per-value messages fragment into separate fingerprints.
 const NUMBER_RE = /\b\d+(\.\d+)?/g;
 const QUOTED_RE = /(["'`])(?:\\.|(?!\1).)*\1/g;
+// `\b` never fires between `_` and a digit (both are word chars), so
+// underscore-glued ids — "prj_1013", "user_42", "order_9f3ac2d144" — escape
+// NUMBER_RE/LONG_HEX_RE entirely and fragment one defect into an issue per
+// id. Template the value after the underscore explicitly. (Letter-glued
+// digits like "sha256"/"utf8" stay literal on purpose — those are usually
+// meaningful tokens, not per-entity ids.)
+const UNDERSCORE_HEX_RE = /_[0-9a-f]{8,}\b/gi;
+const UNDERSCORE_NUMBER_RE = /_\d+(\.\d+)?\b/g;
 export function normalizeMessage(message) {
     return message
         .slice(0, 500)
         .replace(QUOTED_RE, "<str>")
         .replace(UUID_RE, "<uuid>")
         .replace(LONG_HEX_RE, "<hex>")
+        .replace(UNDERSCORE_HEX_RE, "_<hex>")
+        .replace(UNDERSCORE_NUMBER_RE, "_<n>")
         .replace(NUMBER_RE, "<n>")
         .replace(/\s+/g, " ")
         .trim();
@@ -44,11 +54,125 @@ export function normalizeRoute(route) {
         .join("/");
 }
 const FRAME_LOCATION_RE = /:\d+(:\d+)?\)?$/;
-export function normalizeStackFrames(stack, topN = 5) {
-    if (!stack)
-        return [];
-    return stack
-        .split("\n")
+// A Go location line: "\t<file>.go:<line> +0x<off>" (offset optional).
+const GO_LOCATION_RE = /^\s*(.+\.go):\d+(?:\s+\+0x[0-9a-f]+)?\s*$/;
+const GO_GOROUTINE_RE = /\bgoroutine \d+ \[/;
+// A .NET frame with a source location: "at <method>(...) in <file>:line <n>".
+const DOTNET_FRAME_RE = /^\s*at\s+.+\)\s+in\s+.+:line\s+\d+\s*$/i;
+const DOTNET_CS_RE = /\.cs:line\s+\d+/i;
+// A JVM frame: "at <fqmethod>(<File>.java:<line>)" / "(Native Method)".
+const JVM_FRAME_RE = /^\s*at\s+[\w$.]+(?:\/[\w$.]+)?\(.*\.(?:java|kt|scala|groovy):\d+\)\s*$/;
+const JVM_NATIVE_RE = /\((?:Native Method|Unknown Source)\)\s*$/;
+// A Rust backtrace frame ("   3: my::mod::func") or its "at <file>.rs:<n>" line.
+const RUST_FRAME_RE = /^\s*\d+:\s+(?:0x[0-9a-f]+\s+-\s+)?\S+::\S/;
+const RUST_AT_RS_RE = /^\s*at\s+\S+\.rs:\d+/;
+/**
+ * Classify a whole stack by language. Signatures are chosen to be unique to
+ * each runtime so a JS or Python stack always falls through to "script".
+ */
+function detectStackLanguage(lines) {
+    let hasDotnet = false;
+    let hasJvm = false;
+    let hasRust = false;
+    for (const line of lines) {
+        if (GO_LOCATION_RE.test(line) || GO_GOROUTINE_RE.test(line))
+            return "go";
+        if (DOTNET_FRAME_RE.test(line) || DOTNET_CS_RE.test(line))
+            hasDotnet = true;
+        if (JVM_FRAME_RE.test(line) || JVM_NATIVE_RE.test(line))
+            hasJvm = true;
+        if (RUST_FRAME_RE.test(line) ||
+            RUST_AT_RS_RE.test(line) ||
+            line.trim() === "stack backtrace:")
+            hasRust = true;
+    }
+    // .NET and JVM frames both start with "at"; decide by the location marker
+    // each detector matched (`.cs`/`:line` vs `.java`/Native Method).
+    if (hasDotnet)
+        return "dotnet";
+    if (hasJvm)
+        return "jvm";
+    if (hasRust)
+        return "rust";
+    return "script";
+}
+/** Drop the trailing call-argument group, e.g. "f(0x1, 0x2)" → "f". */
+function stripTrailingArgs(fn) {
+    return fn.replace(/\([^()]*\)\s*$/, "").trim();
+}
+function cleanGoFunc(fn) {
+    return stripTrailingArgs(fn.replace(/^created by\s+/, "").replace(/\s+in goroutine \d+\s*$/, ""));
+}
+/** Go: a function line followed by a "\t<file>.go:<line> +0x<off>" location. */
+function parseGoFrames(lines, topN) {
+    const frames = [];
+    let prevFunc = "";
+    for (const line of lines) {
+        const loc = GO_LOCATION_RE.exec(line);
+        if (loc && prevFunc) {
+            frames.push(`${cleanGoFunc(prevFunc)} (${loc[1].trim()})`);
+            if (frames.length >= topN)
+                break;
+            prevFunc = "";
+            continue;
+        }
+        const trimmed = line.trim();
+        if (trimmed && !GO_GOROUTINE_RE.test(trimmed))
+            prevFunc = trimmed;
+    }
+    return frames;
+}
+/** Rust: "   N: module::func" optionally followed by "  at <file>:<line>:<col>". */
+function parseRustFrames(lines, topN) {
+    const frames = [];
+    for (let i = 0; i < lines.length && frames.length < topN; i++) {
+        const m = /^\s*\d+:\s+(?:0x[0-9a-f]+\s+-\s+)?(.+?)\s*$/.exec(lines[i]);
+        if (!m)
+            continue;
+        const fn = m[1].replace(/::h[0-9a-f]{6,}$/, "").trim();
+        if (!fn)
+            continue;
+        const at = /^\s*at\s+(\S+?):\d+(?::\d+)?\s*$/.exec(lines[i + 1] ?? "");
+        if (at)
+            i++;
+        frames.push(at ? `${fn} (${at[1]})` : fn);
+    }
+    return frames;
+}
+/** JVM (Java/Kotlin/Scala): "\tat <fqmethod>(<File>:<line>)". */
+function parseJvmFrames(lines, topN) {
+    const frames = [];
+    for (const line of lines) {
+        const m = /^\s*at\s+(.+?)\((.*)\)\s*$/.exec(line);
+        if (!m)
+            continue;
+        frames.push(`${m[1].trim()}(${m[2].trim().replace(/:\d+$/, "")})`);
+        if (frames.length >= topN)
+            break;
+    }
+    return frames;
+}
+/** .NET: "   at <method>(<params>) in <file>:line <n>" (location optional). */
+function parseDotnetFrames(lines, topN) {
+    const frames = [];
+    for (const line of lines) {
+        const m = /^\s*at\s+(.+?)(?:\s+in\s+(.+?):line\s+\d+)?\s*$/.exec(line);
+        if (!m)
+            continue;
+        const method = stripTrailingArgs(m[1]);
+        const file = m[2]?.trim() ?? "";
+        frames.push(file ? `${method} (${file})` : method);
+        if (frames.length >= topN)
+            break;
+    }
+    return frames;
+}
+/**
+ * Historical JS/TS/Firefox/Python normalisation — output is intentionally
+ * unchanged so pre-existing issues in those runtimes keep their fingerprints.
+ */
+function parseScriptFrames(lines, topN) {
+    return lines
         .map((line) => line.trim())
         .filter((line) => /^at\s|@|^\s*File\s/.test(line) || /\.[jt]sx?/.test(line))
         .slice(0, topN)
@@ -58,6 +182,57 @@ export function normalizeStackFrames(stack, topN = 5) {
         .replace(/\s+/g, " ")
         .trim());
 }
+/**
+ * Safe fallback for a non-empty stack we could not parse into frames (an
+ * unsupported runtime, or a malformed one). Rather than discard everything —
+ * which collapses every same-message error into one issue — derive a stable
+ * signature from any structurally frame-like lines, with volatile tokens
+ * (addresses, offsets, line/column numbers) templated out so the SAME defect
+ * still groups across occurrences. When there is no frame-like structure at
+ * all we return nothing, exactly as before, and grouping falls back to the
+ * message + service + error type.
+ */
+function fallbackFrames(lines, topN) {
+    const framey = lines
+        .map((line) => line.trim())
+        .filter((line) => /(?:[/\\]|\.\w+)\S*[:(]\d+/.test(line) ||
+        /\b0x[0-9a-f]+/i.test(line) ||
+        /^(?:at|from)\b/.test(line) ||
+        /^\d+:\s/.test(line));
+    if (framey.length === 0)
+        return [];
+    return framey
+        .map((line) => line
+        .replace(/0x[0-9a-f]+/gi, "0x")
+        .replace(/:\d+(:\d+)?\b/g, "")
+        .replace(/\s+/g, " ")
+        .trim())
+        .filter(Boolean)
+        .slice(0, topN);
+}
+export function normalizeStackFrames(stack, topN = 5) {
+    if (!stack)
+        return [];
+    const lines = stack.split("\n");
+    let frames;
+    switch (detectStackLanguage(lines)) {
+        case "go":
+            frames = parseGoFrames(lines, topN);
+            break;
+        case "rust":
+            frames = parseRustFrames(lines, topN);
+            break;
+        case "jvm":
+            frames = parseJvmFrames(lines, topN);
+            break;
+        case "dotnet":
+            frames = parseDotnetFrames(lines, topN);
+            break;
+        default:
+            frames = parseScriptFrames(lines, topN);
+    }
+    return frames.length > 0 ? frames : fallbackFrames(lines, topN);
+}
 export function fingerprintOccurrence(input) {
     const parts = [
         input.source,
@@ -69,6 +244,42 @@ export function fingerprintOccurrence(input) {
     ];
     return createHash("sha256").update(parts.join(" ")).digest("hex").slice(0, 32);
 }
+/**
+ * Deterministic per-occurrence identity (as opposed to the fingerprint,
+ * which is the per-ISSUE identity shared by every occurrence of a defect).
+ *
+ * The id must be a pure function of the signal, not a fresh UUID per
+ * request: OTLP exporters retry whole batches (after a 503 from a partial
+ * ClickHouse write, or when only the 2xx was lost), and both the ClickHouse
+ * rows and the sink consumer's dedupe ledger key on this id — random ids
+ * would turn every transport retry into a duplicate error downstream.
+ *
+ * Identical signals within one batch stay distinct through their batch
+ * position, which is stable across retries because exporters re-send the
+ * same serialized batch. Residual coalescing risk: two occurrences from
+ * DIFFERENT requests that share the same millisecond, message, and batch
+ * position while carrying neither a traceId nor a sessionId — accepted, as
+ * server signals virtually always carry a traceId and browser signals a
+ * sessionId.
+ */
+export function occurrenceIdFor(scope, input, fingerprint, batchIndex) {
+    const parts = [
+        "v1",
+        scope.orgId,
+        scope.repositoryId,
+        fingerprint,
+        String(input.occurredAt.getTime()),
+        input.traceId ?? "",
+        input.sessionId ?? "",
+        input.message.slice(0, 1000),
+        String(batchIndex),
+    ];
+    // NUL-joined so a free-text field can never bleed into its neighbour.
+    return createHash("sha256")
+        .update(parts.join("\u0000"))
+        .digest("hex")
+        .slice(0, 32);
+}
 export function deriveFields(input) {
     const topFrames = normalizeStackFrames(input.stack);
     return {

package/dist/index.d.ts

@@ -1,3 +1,4 @@
 export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
+export { SinkForwarder, type SinkStats, type SinkTuning } from "./sink.js";
 export * from "./types.js";

package/dist/index.js

@@ -1,7 +1,7 @@
 import { loadConfig } from "./config.js";
 import { createIngesterApp } from "./server.js";
 const config = loadConfig();
-const { app, store } = createIngesterApp(config);
+const { app, store, sink } = createIngesterApp(config);
 const server = app.listen(config.port, () => {
     console.log(`autter otlp-ingester listening on :${config.port} ` +
         `(clickhouse: ${config.clickhouseUrl ? "configured" : "NOT configured"})`);
@@ -14,6 +14,15 @@ if (store.configured) {
 }
 async function shutdown(signal) {
     console.log(`${signal} received, shutting down`);
+    if (sink) {
+        const pending = sink.pendingCount();
+        sink.stop();
+        if (pending > 0) {
+            // The retry buffer is memory-only; everything in it is already in
+            // ClickHouse, so the consumer's reconciliation replays it.
+            console.warn(`${pending} sink batch(es) undelivered at shutdown — recoverable via ClickHouse replay`);
+        }
+    }
     server.close(() => {
         void store.close().finally(() => process.exit(0));
     });
@@ -23,4 +32,5 @@ process.on("SIGTERM", () => void shutdown("SIGTERM"));
 process.on("SIGINT", () => void shutdown("SIGINT"));
 export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
+export { SinkForwarder } from "./sink.js";
 export * from "./types.js";

package/dist/llm-pricing.d.ts

@@ -0,0 +1,20 @@
+/**
+ * Built-in USD price estimates for common LLM models, per 1M tokens.
+ *
+ * Estimation happens once at ingest so cost queries never re-price history
+ * (a price change should not rewrite last month's spend). The table is a
+ * best-effort default: SDKs can always report the exact figure via the
+ * `autter.llm.cost_usd` span attribute, which takes precedence.
+ *
+ * Matching is longest-prefix-wins on a normalised model id (lowercased,
+ * provider/router prefixes stripped), so `claude-sonnet-4-20250514` matches
+ * the `claude-sonnet-4` entry and gateway ids like `openai/gpt-5-mini`
+ * match `gpt-5-mini`. Unknown models estimate to 0 with costSource "none" —
+ * tokens are still tracked, and dashboards can flag unpriced volume.
+ */
+export declare function normalizeModelId(model: string): string;
+/**
+ * Estimate the USD cost of one call, or null when the model is unknown.
+ * Rates are per 1M tokens; token counts of 0 legitimately price to 0.
+ */
+export declare function estimateLlmCostUsd(model: string, inputTokens: number, outputTokens: number): number | null;

package/dist/llm-pricing.js

@@ -0,0 +1,120 @@
+/**
+ * Built-in USD price estimates for common LLM models, per 1M tokens.
+ *
+ * Estimation happens once at ingest so cost queries never re-price history
+ * (a price change should not rewrite last month's spend). The table is a
+ * best-effort default: SDKs can always report the exact figure via the
+ * `autter.llm.cost_usd` span attribute, which takes precedence.
+ *
+ * Matching is longest-prefix-wins on a normalised model id (lowercased,
+ * provider/router prefixes stripped), so `claude-sonnet-4-20250514` matches
+ * the `claude-sonnet-4` entry and gateway ids like `openai/gpt-5-mini`
+ * match `gpt-5-mini`. Unknown models estimate to 0 with costSource "none" —
+ * tokens are still tracked, and dashboards can flag unpriced volume.
+ */
+// Keep more-specific prefixes ABOVE their generic fallback (the list is
+// scanned for the longest match, but equal-length ties resolve by order).
+const MODEL_PRICING = [
+    // OpenAI
+    { prefix: "gpt-5-nano", inputPerMTok: 0.05, outputPerMTok: 0.4 },
+    { prefix: "gpt-5-mini", inputPerMTok: 0.25, outputPerMTok: 2 },
+    { prefix: "gpt-5", inputPerMTok: 1.25, outputPerMTok: 10 },
+    { prefix: "gpt-4.1-nano", inputPerMTok: 0.1, outputPerMTok: 0.4 },
+    { prefix: "gpt-4.1-mini", inputPerMTok: 0.4, outputPerMTok: 1.6 },
+    { prefix: "gpt-4.1", inputPerMTok: 2, outputPerMTok: 8 },
+    { prefix: "gpt-4o-mini", inputPerMTok: 0.15, outputPerMTok: 0.6 },
+    { prefix: "gpt-4o", inputPerMTok: 2.5, outputPerMTok: 10 },
+    { prefix: "o3-mini", inputPerMTok: 1.1, outputPerMTok: 4.4 },
+    { prefix: "o3", inputPerMTok: 2, outputPerMTok: 8 },
+    { prefix: "o4-mini", inputPerMTok: 1.1, outputPerMTok: 4.4 },
+    { prefix: "text-embedding-3-small", inputPerMTok: 0.02, outputPerMTok: 0 },
+    { prefix: "text-embedding-3-large", inputPerMTok: 0.13, outputPerMTok: 0 },
+    // Anthropic
+    { prefix: "claude-opus-4-5", inputPerMTok: 5, outputPerMTok: 25 },
+    { prefix: "claude-opus-4-8", inputPerMTok: 5, outputPerMTok: 25 },
+    { prefix: "claude-opus-4", inputPerMTok: 15, outputPerMTok: 75 },
+    { prefix: "claude-sonnet-4", inputPerMTok: 3, outputPerMTok: 15 },
+    { prefix: "claude-sonnet", inputPerMTok: 3, outputPerMTok: 15 },
+    { prefix: "claude-haiku-4", inputPerMTok: 1, outputPerMTok: 5 },
+    { prefix: "claude-3-5-haiku", inputPerMTok: 0.8, outputPerMTok: 4 },
+    { prefix: "claude-haiku", inputPerMTok: 1, outputPerMTok: 5 },
+    // Google
+    { prefix: "gemini-2.5-pro", inputPerMTok: 1.25, outputPerMTok: 10 },
+    { prefix: "gemini-2.5-flash-lite", inputPerMTok: 0.1, outputPerMTok: 0.4 },
+    { prefix: "gemini-2.5-flash", inputPerMTok: 0.3, outputPerMTok: 2.5 },
+    { prefix: "gemini-2.0-flash", inputPerMTok: 0.1, outputPerMTok: 0.4 },
+    // DeepSeek
+    { prefix: "deepseek-chat", inputPerMTok: 0.27, outputPerMTok: 1.1 },
+    { prefix: "deepseek-reasoner", inputPerMTok: 0.55, outputPerMTok: 2.19 },
+    // Mistral
+    { prefix: "mistral-large", inputPerMTok: 2, outputPerMTok: 6 },
+    { prefix: "mistral-small", inputPerMTok: 0.1, outputPerMTok: 0.3 },
+    // Meta (typical hosted-inference rates)
+    { prefix: "llama-3.1-405b", inputPerMTok: 3, outputPerMTok: 3 },
+    { prefix: "llama-3.1-70b", inputPerMTok: 0.6, outputPerMTok: 0.6 },
+    { prefix: "llama-3.1-8b", inputPerMTok: 0.1, outputPerMTok: 0.1 },
+    { prefix: "llama-3.3-70b", inputPerMTok: 0.6, outputPerMTok: 0.6 },
+    // xAI
+    { prefix: "grok-4", inputPerMTok: 3, outputPerMTok: 15 },
+    { prefix: "grok-3-mini", inputPerMTok: 0.3, outputPerMTok: 0.5 },
+    { prefix: "grok-3", inputPerMTok: 3, outputPerMTok: 15 },
+];
+// Router/provider prefixes seen in the wild ahead of the bare model id:
+// gateway ids ("openai/gpt-5"), Azure deployments ("azure/gpt-4o"), Bedrock
+// ("us.anthropic.claude-sonnet-4-...-v1:0"), Vertex ("models/gemini-2.5-pro").
+const STRIP_PREFIXES = [
+    "openai/",
+    "azure/",
+    "anthropic/",
+    "google/",
+    "vertex_ai/",
+    "vertex/",
+    "bedrock/",
+    "groq/",
+    "xai/",
+    "mistral/",
+    "meta/",
+    "meta-llama/",
+    "deepseek/",
+    "models/",
+    "us.",
+    "eu.",
+    "apac.",
+    "anthropic.",
+    "amazon.",
+];
+export function normalizeModelId(model) {
+    let id = model.trim().toLowerCase();
+    let stripped = true;
+    while (stripped) {
+        stripped = false;
+        for (const prefix of STRIP_PREFIXES) {
+            if (id.length > prefix.length && id.startsWith(prefix)) {
+                id = id.slice(prefix.length);
+                stripped = true;
+            }
+        }
+    }
+    return id;
+}
+/**
+ * Estimate the USD cost of one call, or null when the model is unknown.
+ * Rates are per 1M tokens; token counts of 0 legitimately price to 0.
+ */
+export function estimateLlmCostUsd(model, inputTokens, outputTokens) {
+    if (!model)
+        return null;
+    const id = normalizeModelId(model);
+    let best = null;
+    for (const entry of MODEL_PRICING) {
+        if (!id.startsWith(entry.prefix))
+            continue;
+        if (!best || entry.prefix.length > best.prefix.length)
+            best = entry;
+    }
+    if (!best)
+        return null;
+    const cost = (inputTokens / 1_000_000) * best.inputPerMTok +
+        (outputTokens / 1_000_000) * best.outputPerMTok;
+    return Math.round(cost * 1e6) / 1e6;
+}

package/dist/llm.d.ts

@@ -0,0 +1,20 @@
+import type { RuntimeLlmCall } from "./types.js";
+export interface LlmSpanFacts {
+    name: string;
+    traceId: string;
+    spanId: string;
+    isError: boolean;
+    /** exception.type from the span's exception event, if it had one. */
+    exceptionType: string | null;
+    durationMs: number;
+    startedAt: Date;
+    service: string;
+    environment: string;
+    release: string | null;
+}
+/**
+ * Returns the LLM-call row for a span, or null when the span is not an LLM
+ * provider call. Never throws — attribute soup from arbitrary SDKs must not
+ * take down the ingest path.
+ */
+export declare function extractLlmCall(attrs: Map<string, string>, facts: LlmSpanFacts): RuntimeLlmCall | null;