@autter/otlp-ingester 0.1.0 → 1.1.0

package/dist/normalize-browser.js

@@ -43,6 +43,27 @@ const TYPE_TO_ERROR_TYPE = {
     unhandled_rejection: "UnhandledRejection",
     message: "Message",
 };
+// Content-level gate for the free-form `context` bag. The schema whitelist
+// above is structural; this masks obvious PII/secrets inside whatever a
+// (possibly outdated) SDK still sends: values under sensitive-looking keys
+// and email-shaped strings — mirroring redactAttributes() in
+// @autter/runtime-node and redactContext() in @autter/runtime-browser.
+const SENSITIVE_KEY_RE = /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i;
+const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
+const REDACTED = "[redacted]";
+function scrubContext(context) {
+    const out = {};
+    for (const [key, value] of Object.entries(context)) {
+        if (value === undefined || value === null)
+            continue;
+        out[key] = SENSITIVE_KEY_RE.test(key)
+            ? REDACTED
+            : typeof value === "string"
+                ? value.replace(EMAIL_VALUE_RE, REDACTED)
+                : value;
+    }
+    return out;
+}
 /** Default severity per event type when the SDK doesn't say. */
 const TYPE_TO_SEVERITY = {
     exception: "error",
@@ -52,12 +73,14 @@ const TYPE_TO_SEVERITY = {
 export function normalizeBrowserPayload(payload) {
     const occurrences = [];
     const rollups = new Map();
-    function bumpRollup(route, occurredAt, field) {
+    function bumpRollup(route, occurredAt, counts) {
         const bucketAt = new Date(Math.floor(occurredAt.getTime() / 60_000) * 60_000);
         const key = `${route} ${bucketAt.getTime()}`;
         const existing = rollups.get(key);
         if (existing) {
-            existing[field] += 1;
+            existing.requestCount += counts.requestCount ?? 0;
+            existing.errorCount += counts.errorCount ?? 0;
+            existing.sessionCount += counts.sessionCount ?? 0;
             return;
         }
         rollups.set(key, {
@@ -66,16 +89,16 @@ export function normalizeBrowserPayload(payload) {
             release: payload.release ?? null,
             route,
             bucketAt,
-            requestCount: field === "requestCount" ? 1 : 0,
-            errorCount: 0,
+            requestCount: counts.requestCount ?? 0,
+            errorCount: counts.errorCount ?? 0,
             durationSumMs: 0,
-            sessionCount: field === "sessionCount" ? 1 : 0,
+            sessionCount: counts.sessionCount ?? 0,
         });
     }
     for (const event of payload.events) {
         const occurredAt = new Date(event.timestamp);
         if (event.type === "session_start") {
-            bumpRollup("", occurredAt, "sessionCount");
+            bumpRollup("", occurredAt, { sessionCount: 1 });
             continue;
         }
         // Coarse usage counters: track_event("checkout_opened") becomes a
@@ -83,12 +106,21 @@ export function normalizeBrowserPayload(payload) {
         if (event.type === "track_event") {
             const name = (event.name ?? event.message ?? "").slice(0, 200);
             if (name)
-                bumpRollup(`event:${name}`, occurredAt, "requestCount");
+                bumpRollup(`event:${name}`, occurredAt, { requestCount: 1 });
             continue;
         }
+        const severity = asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error");
+        // The browser has no request stream, so error/message events are the
+        // only usage signal for many pages — count each as an event (and, for
+        // fatal/error severity, an error event). Without this, a page that
+        // only reports errors groups issues whose dashboards all read 0.
+        bumpRollup(event.route ? (event.route.split("?")[0] ?? "") : "", occurredAt, {
+            requestCount: 1,
+            errorCount: severity === "fatal" || severity === "error" ? 1 : 0,
+        });
         occurrences.push({
             source: "browser",
-            severity: asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error"),
+            severity,
             service: payload.service,
             environment: payload.environment,
             release: payload.release ?? null,
@@ -104,7 +136,7 @@ export function normalizeBrowserPayload(payload) {
                 ...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
                 ...(event.line !== undefined ? { line: event.line } : {}),
                 ...(event.column !== undefined ? { column: event.column } : {}),
-                ...(event.context ? { context: event.context } : {}),
+                ...(event.context ? { context: scrubContext(event.context) } : {}),
             },
             occurredAt,
         });

package/dist/normalize-otlp.d.ts

@@ -1,4 +1,4 @@
-import { type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
+import { type RuntimeLlmCall, type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
 /**
  * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields
  * we read (the full OTLP schema is large and versioned; unknown fields pass
@@ -65,6 +65,7 @@ export interface OtlpMetricsRequest {
                 unit?: string;
                 histogram?: {
                     dataPoints?: OtlpDataPoint[];
+                    aggregationTemporality?: string | number;
                 };
                 sum?: {
                     dataPoints?: OtlpDataPoint[];
@@ -77,6 +78,7 @@ export interface NormalizedTraces {
     occurrences: RuntimeOccurrenceInput[];
     spans: RuntimeSpanRow[];
     metricPoints: RuntimeMetricPoint[];
+    llmCalls: RuntimeLlmCall[];
     spanCount: number;
 }
 export declare function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces;

package/dist/normalize-otlp.js

@@ -1,3 +1,5 @@
+import { normalizeRoute } from "./fingerprint.js";
+import { extractLlmCall } from "./llm.js";
 import { asSeverity, } from "./types.js";
 function attrMap(attributes) {
     const map = new Map();
@@ -54,6 +56,7 @@ function resourceInfo(resource) {
             attrs.get("deployment.environment") ??
             "production",
         release: attrs.get("service.version") ?? null,
+        metricsWired: attrs.get("autter.metrics_wired") === "true",
     };
 }
 function routeOf(attrs) {
@@ -93,6 +96,7 @@ const MAX_SPANS_PER_REQUEST = 5000;
 export function normalizeTraces(request) {
     const occurrences = [];
     const spans = [];
+    const llmCalls = [];
     const rollups = new Map();
     let spanCount = 0;
     for (const resourceSpan of request.resourceSpans ?? []) {
@@ -128,6 +132,7 @@ export function normalizeTraces(request) {
                 });
                 // Error occurrences: one per exception event; if the span is
                 // errored without exception events, one from the span status.
+                const occurrencesBefore = occurrences.length;
                 const exceptionEvents = (span.events ?? []).filter((event) => event.name === "exception");
                 for (const event of exceptionEvents) {
                     const eventAttrs = attrMap(event.attributes);
@@ -171,14 +176,39 @@ export function normalizeTraces(request) {
                         occurredAt: startedAt,
                     });
                 }
+                // LLM provider calls (GenAI semconv, Vercel AI SDK telemetry,
+                // or Autter's own helpers) become usage/cost rows. The first
+                // exception event's type rides along so failed calls keep the
+                // provider error class even without an `error.type` attribute.
+                const llmCall = extractLlmCall(attrs, {
+                    name: span.name ?? "",
+                    traceId: span.traceId ?? "",
+                    spanId: span.spanId ?? "",
+                    isError,
+                    exceptionType: exceptionEvents[0]
+                        ? (attrMap(exceptionEvents[0].attributes).get("exception.type") ??
+                            null)
+                        : null,
+                    durationMs,
+                    startedAt,
+                    service: resource.service,
+                    environment: resource.environment,
+                    release: resource.release,
+                });
+                if (llmCall)
+                    llmCalls.push(llmCall);
                 // Server spans fold into 1-minute usage rollups so traffic is
-                // tracked even when the metrics pipeline isn't wired.
-                if (kind === "server") {
+                // tracked even when the metrics pipeline isn't wired. Spans
+                // exported by error-linked tail retention are skipped: the SDK
+                // ships those at ~100% alongside a metrics pipeline that already
+                // counts every request, so folding them in would double-count
+                // erroring routes.
+                if (kind === "server" && attrs.get("autter.tail_retained") !== "true") {
                     addToRollup(rollups, {
                         service: resource.service,
                         environment: resource.environment,
                         release: resource.release,
-                        route: route ?? "",
+                        route: normalizeRoute(route),
                         bucketAt: minuteBucket(startedAt),
                         requestCount: 1,
                         errorCount: isError ? 1 : 0,
@@ -186,10 +216,37 @@ export function normalizeTraces(request) {
                         sessionCount: 0,
                     });
                 }
+                else {
+                    // Occurrences on non-server spans (captureException's internal
+                    // error spans, workers, consumers) have no request rollup to
+                    // ride — count each as one event / one error event, or a
+                    // service whose errors arrive outside HTTP handlers groups
+                    // issues while every event counter stays 0. Server spans are
+                    // excluded: their request rollup above already represents them.
+                    for (const occ of occurrences.slice(occurrencesBefore)) {
+                        addToRollup(rollups, {
+                            service: occ.service,
+                            environment: occ.environment,
+                            release: occ.release,
+                            route: occ.route ?? "",
+                            bucketAt: minuteBucket(occ.occurredAt),
+                            requestCount: 1,
+                            errorCount: occ.severity === "fatal" || occ.severity === "error" ? 1 : 0,
+                            durationSumMs: 0,
+                            sessionCount: 0,
+                        });
+                    }
+                }
             }
         }
     }
-    return { occurrences, spans, metricPoints: [...rollups.values()], spanCount };
+    return {
+        occurrences,
+        spans,
+        metricPoints: [...rollups.values()],
+        llmCalls,
+        spanCount,
+    };
 }
 function minuteBucket(date) {
     return new Date(Math.floor(date.getTime() / 60_000) * 60_000);
@@ -222,6 +279,16 @@ const HTTP_DURATION_INSTRUMENTS = {
     "http.server.duration": 1,
     "http.server.request.duration": 1000,
 };
+/**
+ * Cumulative histograms report lifetime totals on every export; adding them
+ * into a SummingMergeTree would re-count all past requests each interval.
+ * Only deltas are summable — cumulative senders are skipped and covered by
+ * the span-fed rollup fallback instead. (Enum arrives as a number or an
+ * `AGGREGATION_TEMPORALITY_*` string depending on the serialiser.)
+ */
+function isCumulativeTemporality(t) {
+    return t === 2 || t === "AGGREGATION_TEMPORALITY_CUMULATIVE";
+}
 export function normalizeMetrics(request) {
     const rollups = new Map();
     for (const resourceMetric of request.resourceMetrics ?? []) {
@@ -233,6 +300,9 @@ export function normalizeMetrics(request) {
                     : undefined;
                 if (multiplier === undefined)
                     continue;
+                if (isCumulativeTemporality(metric.histogram?.aggregationTemporality)) {
+                    continue;
+                }
                 for (const dataPoint of metric.histogram?.dataPoints ?? []) {
                     const attrs = attrMap(dataPoint.attributes);
                     const statusCode = statusCodeOf(attrs);
@@ -243,7 +313,11 @@ export function normalizeMetrics(request) {
                         service: resource.service,
                         environment: resource.environment,
                         release: resource.release,
-                        route: routeOf(attrs) ?? "",
+                        // Same normalization as the span-fed rollups: emitters
+                        // are supposed to put route templates in `http.route`,
+                        // but some put raw paths there — keep the key space
+                        // bounded and consistent across both feeds.
+                        route: normalizeRoute(routeOf(attrs)),
                         bucketAt: minuteBucket(nanosToDate(dataPoint.timeUnixNano)),
                         requestCount: count,
                         errorCount: statusCode !== null && statusCode >= 500 ? count : 0,

package/dist/otlp-proto.js

@@ -64,7 +64,7 @@ message NumberDataPoint {
   sfixed64 as_int = 6;
   repeated KeyValue attributes = 7;
 }
-message Histogram { repeated HistogramDataPoint data_points = 1; }
+message Histogram { repeated HistogramDataPoint data_points = 1; int32 aggregation_temporality = 2; }
 message HistogramDataPoint {
   fixed64 time_unix_nano = 3;
   fixed64 count = 4;

package/dist/server.d.ts

@@ -1,8 +1,11 @@
 import { type Express } from "express";
 import { ClickHouseStore } from "./clickhouse.js";
 import type { IngesterConfig } from "./config.js";
+import { SinkForwarder } from "./sink.js";
 export interface IngesterApp {
     app: Express;
     store: ClickHouseStore;
+    /** Present when AUTTER_SINK_URL is configured. */
+    sink: SinkForwarder | null;
 }
 export declare function createIngesterApp(config: IngesterConfig): IngesterApp;

package/dist/server.js

@@ -1,13 +1,17 @@
-import { randomUUID } from "node:crypto";
 import express from "express";
 import { KeyResolver, RateLimiter } from "./auth.js";
 import { ClickHouseStore } from "./clickhouse.js";
-import { deriveFields, fingerprintOccurrence } from "./fingerprint.js";
+import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerprint.js";
 import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
 import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
+import { SinkForwarder } from "./sink.js";
 export function createIngesterApp(config) {
     const store = new ClickHouseStore(config);
+    // Fingerprinted occurrences feed the consumer's issue grouping, metric
+    // points feed the request/error-rate rollups, LLM calls feed spend
+    // watching. Delivery is at-least-once with bounded retries — see sink.ts.
+    const sink = config.sinkUrl ? new SinkForwarder(config) : null;
     const keys = new KeyResolver(config);
     const serverRateLimiter = new RateLimiter(config.rateLimitPerMinute);
     const clientRateLimiter = new RateLimiter(config.clientRateLimitPerMinute);
@@ -46,16 +50,19 @@ export function createIngesterApp(config) {
         next();
     });
     app.get("/healthz", async (_req, res) => {
+        const sinkStats = sink ? { sink: sink.stats() } : {};
         if (!store.configured) {
-            res.status(200).json({ ok: true, clickhouse: "unconfigured" });
+            res.status(200).json({ ok: true, clickhouse: "unconfigured", ...sinkStats });
             return;
         }
         try {
             const ok = await store.ping();
-            res.status(ok ? 200 : 503).json({ ok, clickhouse: ok ? "up" : "down" });
+            res
+                .status(ok ? 200 : 503)
+                .json({ ok, clickhouse: ok ? "up" : "down", ...sinkStats });
         }
         catch {
-            res.status(503).json({ ok: false, clickhouse: "down" });
+            res.status(503).json({ ok: false, clickhouse: "down", ...sinkStats });
         }
     });
     /** Auth + scope + rate limit; returns null (response sent) on failure. */
@@ -103,38 +110,20 @@ export function createIngesterApp(config) {
         }
         return ctx;
     }
-    function fingerprintAll(inputs) {
-        return inputs.map((input) => ({
-            ...input,
-            occurrenceId: randomUUID(),
-            fingerprint: fingerprintOccurrence(input),
-            ...deriveFields(input),
-        }));
-    }
-    /** Best-effort forward of fingerprinted occurrences for issue grouping. */
-    function forwardToSink(ctx, occurrences) {
-        if (!config.sinkUrl || occurrences.length === 0)
-            return;
-        void fetch(config.sinkUrl, {
-            method: "POST",
-            headers: {
-                "content-type": "application/json",
-                ...(config.sinkToken
-                    ? { authorization: `Bearer ${config.sinkToken}` }
-                    : {}),
-            },
-            body: JSON.stringify({
-                version: 1,
-                orgId: ctx.orgId,
-                repositoryId: ctx.repositoryId,
-                occurrences: occurrences.map((o) => ({
-                    ...o,
-                    occurredAt: o.occurredAt.toISOString(),
-                })),
-            }),
-            signal: AbortSignal.timeout(10_000),
-        }).catch((err) => {
-            console.warn("sink forward failed (non-fatal):", err?.message ?? err);
+    /** Ids are content-derived (occurrenceIdFor), NOT random: an exporter
+     * that retries a batch — after a 503 from a partially-failed ClickHouse
+     * write, or when only our 2xx got lost — must produce the same ids, so
+     * the sink consumer's per-occurrence dedupe holds across transport
+     * retries and duplicated ClickHouse rows stay identifiable. */
+    function fingerprintAll(ctx, inputs) {
+        return inputs.map((input, index) => {
+            const fingerprint = fingerprintOccurrence(input);
+            return {
+                ...input,
+                occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index),
+                fingerprint,
+                ...deriveFields(input),
+            };
         });
     }
     function storageError(res, err) {
@@ -167,20 +156,29 @@ export function createIngesterApp(config) {
         else {
             request = req.body;
         }
-        const { occurrences, spans, metricPoints } = normalizeTraces(request);
-        const fingerprinted = fingerprintAll(occurrences);
+        const { occurrences, spans, metricPoints, llmCalls } = normalizeTraces(request);
+        const fingerprinted = fingerprintAll(ctx, occurrences);
+        // ClickHouse has no cross-table transaction, so these four inserts can
+        // partially commit. Recovery boundary: any failure → 503 → the exporter
+        // retries the whole batch. Deterministic occurrence ids make the retry
+        // idempotent downstream (consumer dedupes per id; duplicate ClickHouse
+        // rows share an id, and the consumer's reconciler counts distinct ids),
+        // and nothing reaches the sink queue unless every insert succeeded —
+        // signals persisted by a partial write are picked up by the consumer's
+        // ClickHouse reconciliation instead.
         try {
             await Promise.all([
                 store.insertOccurrences(ctx, fingerprinted),
                 store.insertSpans(ctx, spans),
                 store.insertMetricPoints(ctx, metricPoints),
+                store.insertLlmCalls(ctx, llmCalls),
             ]);
         }
         catch (err) {
             storageError(res, err);
             return;
         }
-        forwardToSink(ctx, fingerprinted);
+        sink?.enqueue(ctx, fingerprinted, metricPoints, llmCalls);
         otlpSuccess(req, res);
     });
     app.post("/v1/metrics", async (req, res) => {
@@ -208,6 +206,7 @@ export function createIngesterApp(config) {
             storageError(res, err);
             return;
         }
+        sink?.enqueue(ctx, [], metricPoints);
         otlpSuccess(req, res);
     });
     app.post("/v1/browser", async (req, res) => {
@@ -234,7 +233,7 @@ export function createIngesterApp(config) {
             return;
         }
         const { occurrences, metricPoints } = normalizeBrowserPayload(parsed.data);
-        const fingerprinted = fingerprintAll(occurrences);
+        const fingerprinted = fingerprintAll(ctx, occurrences);
         try {
             await Promise.all([
                 store.insertOccurrences(ctx, fingerprinted),
@@ -245,7 +244,7 @@ export function createIngesterApp(config) {
             storageError(res, err);
             return;
         }
-        forwardToSink(ctx, fingerprinted);
+        sink?.enqueue(ctx, fingerprinted, metricPoints);
         res.status(202).json({ accepted: fingerprinted.length });
     });
     // Body-parser errors (oversized/malformed JSON) → clean 4xx, not a stack.
@@ -263,5 +262,5 @@ export function createIngesterApp(config) {
         console.error("unhandled error:", err);
         res.status(500).json({ error: "internal error" });
     });
-    return { app, store };
+    return { app, store, sink };
 }

package/dist/sink.d.ts

@@ -0,0 +1,103 @@
+import type { IngesterConfig } from "./config.js";
+import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence } from "./types.js";
+/**
+ * At-least-once delivery to the sink webhook.
+ *
+ * The sink feeds the consumer's issue grouping and incident detection, so a
+ * lost batch means silently missing error occurrences — a single
+ * fire-and-forget POST is not enough (a routine consumer deploy is longer
+ * than one request timeout). Batches therefore queue in memory and retry
+ * with exponential backoff until delivered, permanently rejected, or the
+ * bounded buffer overflows.
+ *
+ * Durability boundary: the queue is in-memory only. Everything forwarded
+ * here was already written to ClickHouse (ingest 503s otherwise), so after
+ * a process crash or an overflow/permanent drop the consumer recovers by
+ * replaying the logged time range from ClickHouse — see
+ * docs/ARCHITECTURE.md "Sink webhook". Every batch carries a unique
+ * `batchId` so consumers can deduplicate retried deliveries.
+ */
+/** Delivery timing/concurrency knobs; overridable for tests. */
+export interface SinkTuning {
+    /** Base delay before the second attempt; doubles per attempt to the cap. */
+    retryBaseMs: number;
+    retryCapMs: number;
+    requestTimeoutMs: number;
+    /** Deliveries run concurrently while healthy, serially while failing. */
+    healthyConcurrency: number;
+}
+/**
+ * Operational counters for /healthz. Failure detail is reduced to a fixed
+ * category (`timeout`, `connection_error`, `http_<status>`, `error`) — the
+ * health endpoint is unauthenticated, so raw transport/exception text stays
+ * in server-side logs only.
+ */
+export interface SinkStats {
+    queued: number;
+    queuedBytes: number;
+    inFlight: number;
+    delivered: number;
+    retried: number;
+    droppedOverflow: number;
+    droppedPermanent: number;
+    consecutiveFailures: number;
+    lastFailureAt: string | null;
+    lastFailureReason: string | null;
+    oldestQueuedSince: string | null;
+}
+export declare class SinkForwarder {
+    private readonly config;
+    private readonly fetchImpl;
+    /** Always sorted by `seq`: queue[0] is the oldest batch. */
+    private readonly queue;
+    private queuedBytes;
+    /** Per-org share of queuedBytes — overflow evicts from the heaviest org. */
+    private readonly queuedBytesByOrg;
+    private inFlight;
+    private timer;
+    private stopped;
+    private seqCounter;
+    private delivered;
+    private retried;
+    private droppedOverflow;
+    private droppedPermanent;
+    private consecutiveFailures;
+    /**
+     * Bumped on every failure. A success only clears consecutiveFailures if
+     * no failure happened after that request STARTED — a concurrent success
+     * that overlapped a failure proves nothing about current sink health and
+     * must not reopen full concurrency mid-outage.
+     */
+    private failureEpoch;
+    private lastFailureAt;
+    private lastFailureReason;
+    private readonly tuning;
+    constructor(config: IngesterConfig, fetchImpl?: typeof fetch, tuning?: Partial<SinkTuning>);
+    /** Queue a batch for delivery. No-op when there is nothing to send. */
+    enqueue(ctx: IngestContext, occurrences: RuntimeOccurrence[], metricPoints?: RuntimeMetricPoint[], llmCalls?: RuntimeLlmCall[]): void;
+    stats(): SinkStats;
+    pendingCount(): number;
+    /** Stop scheduling new deliveries (shutdown). Queued batches are logged
+     * by the caller — they are recoverable from ClickHouse, not from here. */
+    stop(): void;
+    /** Serialize one delivery payload; `seq` pins its age for ordering. */
+    private buildBatch;
+    private maxBufferedBytes;
+    private track;
+    private untrack;
+    /** Re-insert a retrying batch at its age position (queue is seq-sorted),
+     * so overflow eviction still drops the genuinely oldest signals first. */
+    private insertBySeq;
+    /**
+     * Oldest-first eviction, charged to the heaviest tenant: the freshest
+     * signals matter most to grouping, and one org flooding the shared
+     * buffer must not evict everyone else's batches.
+     */
+    private enforceBounds;
+    /** The oldest batch of the org holding the most buffered bytes. */
+    private evictOne;
+    private currentLimit;
+    private pump;
+    private scheduleWake;
+    private send;
+}