@autter/otlp-ingester 0.1.0

package/dist/normalize-otlp.js

@@ -0,0 +1,258 @@
+import { asSeverity, } from "./types.js";
+function attrMap(attributes) {
+    const map = new Map();
+    for (const attr of attributes ?? []) {
+        if (!attr.key || !attr.value)
+            continue;
+        const v = attr.value;
+        const value = v.stringValue ??
+            (v.intValue !== undefined ? String(v.intValue) : undefined) ??
+            (v.doubleValue !== undefined ? String(v.doubleValue) : undefined) ??
+            (v.boolValue !== undefined ? String(v.boolValue) : undefined);
+        if (value !== undefined)
+            map.set(attr.key, value);
+    }
+    return map;
+}
+function nanosToDate(nanos) {
+    if (nanos === undefined)
+        return new Date();
+    const ms = Number(BigInt(String(nanos)) / 1000000n);
+    return new Date(ms);
+}
+function spanDurationMs(span) {
+    if (span.startTimeUnixNano === undefined || span.endTimeUnixNano === undefined) {
+        return 0;
+    }
+    const nanos = BigInt(String(span.endTimeUnixNano)) - BigInt(String(span.startTimeUnixNano));
+    return Math.max(0, Number(nanos) / 1_000_000);
+}
+function isErrorStatus(code) {
+    return code === 2 || code === "STATUS_CODE_ERROR";
+}
+const SPAN_KINDS = {
+    "0": "unspecified",
+    "1": "internal",
+    "2": "server",
+    "3": "client",
+    "4": "producer",
+    "5": "consumer",
+};
+function spanKind(kind) {
+    if (kind === undefined)
+        return "internal";
+    if (typeof kind === "string" && kind.startsWith("SPAN_KIND_")) {
+        return kind.slice("SPAN_KIND_".length).toLowerCase();
+    }
+    return SPAN_KINDS[String(kind)] ?? "internal";
+}
+function resourceInfo(resource) {
+    const attrs = attrMap(resource?.attributes);
+    return {
+        service: attrs.get("service.name") ?? "unknown",
+        environment: attrs.get("deployment.environment.name") ??
+            attrs.get("deployment.environment") ??
+            "production",
+        release: attrs.get("service.version") ?? null,
+    };
+}
+function routeOf(attrs) {
+    const route = attrs.get("http.route") ??
+        attrs.get("url.path") ??
+        attrs.get("http.target") ??
+        null;
+    return route ? (route.split("?")[0] ?? null) : null;
+}
+function statusCodeOf(attrs) {
+    const raw = attrs.get("http.response.status_code") ?? attrs.get("http.status_code");
+    if (!raw)
+        return null;
+    const code = Number.parseInt(raw, 10);
+    return Number.isFinite(code) ? code : null;
+}
+function methodOf(attrs) {
+    const raw = attrs.get("http.request.method") ?? attrs.get("http.method");
+    return raw ? raw.toUpperCase().slice(0, 16) : null;
+}
+/**
+ * Severity of an exception event. SDKs mark it with `autter.severity`
+ * ("fatal" | "error" | "warning" | "info"); `autter.unhandled: true` (the
+ * crash marker set by @autter/runtime-node) implies fatal. Default: error.
+ */
+function severityOf(eventAttrs, spanAttrs) {
+    const explicit = eventAttrs.get("autter.severity") ?? spanAttrs.get("autter.severity");
+    const severity = asSeverity(explicit, "error");
+    if (severity === "error" &&
+        (eventAttrs.get("autter.unhandled") === "true" ||
+            spanAttrs.get("autter.unhandled") === "true")) {
+        return "fatal";
+    }
+    return severity;
+}
+const MAX_SPANS_PER_REQUEST = 5000;
+export function normalizeTraces(request) {
+    const occurrences = [];
+    const spans = [];
+    const rollups = new Map();
+    let spanCount = 0;
+    for (const resourceSpan of request.resourceSpans ?? []) {
+        const resource = resourceInfo(resourceSpan.resource);
+        for (const scopeSpan of resourceSpan.scopeSpans ?? []) {
+            for (const span of scopeSpan.spans ?? []) {
+                if (spanCount >= MAX_SPANS_PER_REQUEST)
+                    break;
+                spanCount += 1;
+                const attrs = attrMap(span.attributes);
+                const route = routeOf(attrs);
+                const statusCode = statusCodeOf(attrs);
+                const startedAt = nanosToDate(span.startTimeUnixNano);
+                const durationMs = spanDurationMs(span);
+                const kind = spanKind(span.kind);
+                const isError = isErrorStatus(span.status?.code) ||
+                    (statusCode !== null && statusCode >= 500);
+                spans.push({
+                    service: resource.service,
+                    environment: resource.environment,
+                    release: resource.release,
+                    traceId: span.traceId ?? "",
+                    spanId: span.spanId ?? "",
+                    parentSpanId: span.parentSpanId ?? null,
+                    name: span.name ?? "unnamed",
+                    kind,
+                    status: isError ? "error" : "ok",
+                    route,
+                    statusCode,
+                    durationMs,
+                    attributes: null,
+                    startedAt,
+                });
+                // Error occurrences: one per exception event; if the span is
+                // errored without exception events, one from the span status.
+                const exceptionEvents = (span.events ?? []).filter((event) => event.name === "exception");
+                for (const event of exceptionEvents) {
+                    const eventAttrs = attrMap(event.attributes);
+                    occurrences.push({
+                        source: "server",
+                        severity: severityOf(eventAttrs, attrs),
+                        service: resource.service,
+                        environment: resource.environment,
+                        release: resource.release,
+                        errorType: eventAttrs.get("exception.type") ?? "Error",
+                        message: eventAttrs.get("exception.message") ??
+                            span.status?.message ??
+                            span.name ??
+                            "Unknown error",
+                        stack: eventAttrs.get("exception.stacktrace") ?? null,
+                        route,
+                        method: methodOf(attrs),
+                        statusCode,
+                        traceId: span.traceId ?? null,
+                        sessionId: null,
+                        attributes: null,
+                        occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
+                    });
+                }
+                if (exceptionEvents.length === 0 && isErrorStatus(span.status?.code)) {
+                    occurrences.push({
+                        source: "server",
+                        severity: severityOf(new Map(), attrs),
+                        service: resource.service,
+                        environment: resource.environment,
+                        release: resource.release,
+                        errorType: "SpanError",
+                        message: span.status?.message || `${span.name ?? "span"} failed`,
+                        stack: null,
+                        route,
+                        method: methodOf(attrs),
+                        statusCode,
+                        traceId: span.traceId ?? null,
+                        sessionId: null,
+                        attributes: null,
+                        occurredAt: startedAt,
+                    });
+                }
+                // Server spans fold into 1-minute usage rollups so traffic is
+                // tracked even when the metrics pipeline isn't wired.
+                if (kind === "server") {
+                    addToRollup(rollups, {
+                        service: resource.service,
+                        environment: resource.environment,
+                        release: resource.release,
+                        route: route ?? "",
+                        bucketAt: minuteBucket(startedAt),
+                        requestCount: 1,
+                        errorCount: isError ? 1 : 0,
+                        durationSumMs: durationMs,
+                        sessionCount: 0,
+                    });
+                }
+            }
+        }
+    }
+    return { occurrences, spans, metricPoints: [...rollups.values()], spanCount };
+}
+function minuteBucket(date) {
+    return new Date(Math.floor(date.getTime() / 60_000) * 60_000);
+}
+function rollupKey(p) {
+    return [
+        p.service,
+        p.environment,
+        p.release ?? "",
+        p.route,
+        p.bucketAt.getTime(),
+    ].join("");
+}
+function addToRollup(rollups, point) {
+    const key = rollupKey(point);
+    const existing = rollups.get(key);
+    if (!existing) {
+        rollups.set(key, { ...point });
+        return;
+    }
+    existing.requestCount += point.requestCount;
+    existing.errorCount += point.errorCount;
+    existing.durationSumMs += point.durationSumMs;
+    existing.sessionCount += point.sessionCount;
+}
+// HTTP-server duration instruments we know how to fold into rollups.
+// `http.server.duration` (old semconv) is milliseconds;
+// `http.server.request.duration` (current semconv) is seconds.
+const HTTP_DURATION_INSTRUMENTS = {
+    "http.server.duration": 1,
+    "http.server.request.duration": 1000,
+};
+export function normalizeMetrics(request) {
+    const rollups = new Map();
+    for (const resourceMetric of request.resourceMetrics ?? []) {
+        const resource = resourceInfo(resourceMetric.resource);
+        for (const scopeMetric of resourceMetric.scopeMetrics ?? []) {
+            for (const metric of scopeMetric.metrics ?? []) {
+                const multiplier = metric.name
+                    ? HTTP_DURATION_INSTRUMENTS[metric.name]
+                    : undefined;
+                if (multiplier === undefined)
+                    continue;
+                for (const dataPoint of metric.histogram?.dataPoints ?? []) {
+                    const attrs = attrMap(dataPoint.attributes);
+                    const statusCode = statusCodeOf(attrs);
+                    const count = Number(dataPoint.count ?? 0);
+                    if (count <= 0)
+                        continue;
+                    addToRollup(rollups, {
+                        service: resource.service,
+                        environment: resource.environment,
+                        release: resource.release,
+                        route: routeOf(attrs) ?? "",
+                        bucketAt: minuteBucket(nanosToDate(dataPoint.timeUnixNano)),
+                        requestCount: count,
+                        errorCount: statusCode !== null && statusCode >= 500 ? count : 0,
+                        durationSumMs: (dataPoint.sum ?? 0) * multiplier,
+                        sessionCount: 0,
+                    });
+                }
+            }
+        }
+    }
+    return [...rollups.values()];
+}

package/dist/otlp-proto.d.ts

@@ -0,0 +1,3 @@
+import type { OtlpMetricsRequest, OtlpTraceRequest } from "./normalize-otlp.js";
+export declare function decodeTraceRequest(body: Buffer): OtlpTraceRequest;
+export declare function decodeMetricsRequest(body: Buffer): OtlpMetricsRequest;

package/dist/otlp-proto.js

@@ -0,0 +1,107 @@
+import protobuf from "protobufjs";
+/**
+ * OTLP/HTTP protobuf decode (`content-type: application/x-protobuf`) —
+ * the default wire format for most OpenTelemetry SDKs (Go, Rust, Python,
+ * Java, .NET, and the JS proto exporters).
+ *
+ * The schema below is a TRIMMED mirror of opentelemetry-proto: only the
+ * fields the normaliser reads, with their exact field numbers. Protobuf
+ * skips unknown fields during decode, so payloads produced against the
+ * full schema parse correctly. Decoded messages are converted to the same
+ * structural shape as OTLP/JSON (hex ids, camelCase, stringified 64-bit
+ * ints) and fed through the existing normaliser.
+ */
+const PROTO = `
+syntax = "proto3";
+package otlp;
+
+message AnyValue {
+  oneof value {
+    string string_value = 1;
+    bool bool_value = 2;
+    int64 int_value = 3;
+    double double_value = 4;
+  }
+}
+message KeyValue { string key = 1; AnyValue value = 2; }
+message Resource { repeated KeyValue attributes = 1; }
+
+message ExportTraceServiceRequest { repeated ResourceSpans resource_spans = 1; }
+message ResourceSpans { Resource resource = 1; repeated ScopeSpans scope_spans = 2; }
+message ScopeSpans { repeated Span spans = 2; }
+message Span {
+  bytes trace_id = 1;
+  bytes span_id = 2;
+  bytes parent_span_id = 4;
+  string name = 5;
+  int32 kind = 6;
+  fixed64 start_time_unix_nano = 7;
+  fixed64 end_time_unix_nano = 8;
+  repeated KeyValue attributes = 9;
+  repeated Event events = 11;
+  Status status = 15;
+  message Event {
+    fixed64 time_unix_nano = 1;
+    string name = 2;
+    repeated KeyValue attributes = 3;
+  }
+}
+message Status { string message = 2; int32 code = 3; }
+
+message ExportMetricsServiceRequest { repeated ResourceMetrics resource_metrics = 1; }
+message ResourceMetrics { Resource resource = 1; repeated ScopeMetrics scope_metrics = 2; }
+message ScopeMetrics { repeated Metric metrics = 2; }
+message Metric {
+  string name = 1;
+  string unit = 3;
+  Sum sum = 7;
+  Histogram histogram = 9;
+}
+message Sum { repeated NumberDataPoint data_points = 1; }
+message NumberDataPoint {
+  fixed64 time_unix_nano = 3;
+  double as_double = 4;
+  sfixed64 as_int = 6;
+  repeated KeyValue attributes = 7;
+}
+message Histogram { repeated HistogramDataPoint data_points = 1; }
+message HistogramDataPoint {
+  fixed64 time_unix_nano = 3;
+  fixed64 count = 4;
+  optional double sum = 5;
+  repeated KeyValue attributes = 9;
+}
+`;
+const root = protobuf.parse(PROTO).root;
+const TraceRequest = root.lookupType("otlp.ExportTraceServiceRequest");
+const MetricsRequest = root.lookupType("otlp.ExportMetricsServiceRequest");
+const TO_OBJECT_OPTIONS = {
+    longs: String, // 64-bit ints → strings (matches OTLP/JSON)
+    defaults: false,
+};
+function hexifyIds(value) {
+    if (Array.isArray(value))
+        return value.map(hexifyIds);
+    if (value && typeof value === "object" && !(value instanceof Uint8Array)) {
+        const out = {};
+        for (const [k, v] of Object.entries(value)) {
+            if ((k === "traceId" || k === "spanId" || k === "parentSpanId") &&
+                v instanceof Uint8Array) {
+                out[k] = Buffer.from(v).toString("hex");
+            }
+            else {
+                out[k] = hexifyIds(v);
+            }
+        }
+        return out;
+    }
+    return value;
+}
+export function decodeTraceRequest(body) {
+    const message = TraceRequest.decode(body);
+    return hexifyIds(TraceRequest.toObject(message, TO_OBJECT_OPTIONS));
+}
+export function decodeMetricsRequest(body) {
+    const message = MetricsRequest.decode(body);
+    return hexifyIds(MetricsRequest.toObject(message, TO_OBJECT_OPTIONS));
+}

package/dist/server.d.ts

@@ -0,0 +1,8 @@
+import { type Express } from "express";
+import { ClickHouseStore } from "./clickhouse.js";
+import type { IngesterConfig } from "./config.js";
+export interface IngesterApp {
+    app: Express;
+    store: ClickHouseStore;
+}
+export declare function createIngesterApp(config: IngesterConfig): IngesterApp;

package/dist/server.js

@@ -0,0 +1,267 @@
+import { randomUUID } from "node:crypto";
+import express from "express";
+import { KeyResolver, RateLimiter } from "./auth.js";
+import { ClickHouseStore } from "./clickhouse.js";
+import { deriveFields, fingerprintOccurrence } from "./fingerprint.js";
+import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
+import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
+import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
+export function createIngesterApp(config) {
+    const store = new ClickHouseStore(config);
+    const keys = new KeyResolver(config);
+    const serverRateLimiter = new RateLimiter(config.rateLimitPerMinute);
+    const clientRateLimiter = new RateLimiter(config.clientRateLimitPerMinute);
+    const app = express();
+    app.disable("x-powered-by");
+    app.use(express.json({
+        limit: config.maxBodyBytes,
+        type: ["application/json"],
+    }));
+    // Cross-origin sendBeacon can only send CORS-safelisted content types
+    // without a preflight, so direct-from-browser payloads arrive as
+    // text/plain and are parsed in the /v1/browser handler.
+    app.use(express.text({
+        limit: config.maxBodyBytes,
+        type: ["text/plain"],
+    }));
+    // OTLP protobuf — the default wire format of most OTel SDKs (Go, Rust,
+    // Python, Java, .NET, JS proto exporters). body-parser inflates
+    // gzip/deflate request bodies automatically for all three parsers.
+    app.use(express.raw({
+        limit: config.maxBodyBytes,
+        type: ["application/x-protobuf"],
+    }));
+    // CORS for direct browser ingest (publishable client keys). Auth and the
+    // per-key origin allow-list are enforced at POST time; the CORS response
+    // itself is permissive so preflights never need key knowledge.
+    app.use("/v1/browser", (req, res, next) => {
+        res.setHeader("access-control-allow-origin", "*");
+        res.setHeader("access-control-allow-methods", "POST, OPTIONS");
+        res.setHeader("access-control-allow-headers", "content-type, authorization, x-autter-key");
+        res.setHeader("access-control-max-age", "86400");
+        if (req.method === "OPTIONS") {
+            res.status(204).end();
+            return;
+        }
+        next();
+    });
+    app.get("/healthz", async (_req, res) => {
+        if (!store.configured) {
+            res.status(200).json({ ok: true, clickhouse: "unconfigured" });
+            return;
+        }
+        try {
+            const ok = await store.ping();
+            res.status(ok ? 200 : 503).json({ ok, clickhouse: ok ? "up" : "down" });
+        }
+        catch {
+            res.status(503).json({ ok: false, clickhouse: "down" });
+        }
+    });
+    /** Auth + scope + rate limit; returns null (response sent) on failure. */
+    async function authenticate(req, res, surface) {
+        // A storage-less ingester must refuse, not accept-and-drop: exporters
+        // retry on 503, so telemetry survives a misconfigured deploy.
+        if (!store.configured) {
+            res.status(503).json({ error: "storage not configured" });
+            return null;
+        }
+        const key = keys.extractKey(req);
+        if (!key) {
+            res.status(401).json({ error: "missing ingest key" });
+            return null;
+        }
+        const ctx = await keys.resolve(key);
+        if (!ctx) {
+            res.status(401).json({ error: "invalid ingest key" });
+            return null;
+        }
+        if (ctx.scope === "client") {
+            // Publishable keys: browser surface only, origin allow-list, and
+            // the tighter rate window.
+            if (surface !== "browser") {
+                res.status(403).json({
+                    error: "client keys cannot send OTLP — use a server key",
+                });
+                return null;
+            }
+            const origin = req.headers.origin;
+            if (ctx.allowedOrigins.length > 0 &&
+                (!origin || !ctx.allowedOrigins.includes(origin))) {
+                res.status(403).json({ error: "origin not allowed for this key" });
+                return null;
+            }
+            if (!clientRateLimiter.allow(key)) {
+                res.status(429).json({ error: "rate limit exceeded" });
+                return null;
+            }
+            return ctx;
+        }
+        if (!serverRateLimiter.allow(key)) {
+            res.status(429).json({ error: "rate limit exceeded" });
+            return null;
+        }
+        return ctx;
+    }
+    function fingerprintAll(inputs) {
+        return inputs.map((input) => ({
+            ...input,
+            occurrenceId: randomUUID(),
+            fingerprint: fingerprintOccurrence(input),
+            ...deriveFields(input),
+        }));
+    }
+    /** Best-effort forward of fingerprinted occurrences for issue grouping. */
+    function forwardToSink(ctx, occurrences) {
+        if (!config.sinkUrl || occurrences.length === 0)
+            return;
+        void fetch(config.sinkUrl, {
+            method: "POST",
+            headers: {
+                "content-type": "application/json",
+                ...(config.sinkToken
+                    ? { authorization: `Bearer ${config.sinkToken}` }
+                    : {}),
+            },
+            body: JSON.stringify({
+                version: 1,
+                orgId: ctx.orgId,
+                repositoryId: ctx.repositoryId,
+                occurrences: occurrences.map((o) => ({
+                    ...o,
+                    occurredAt: o.occurredAt.toISOString(),
+                })),
+            }),
+            signal: AbortSignal.timeout(10_000),
+        }).catch((err) => {
+            console.warn("sink forward failed (non-fatal):", err?.message ?? err);
+        });
+    }
+    function storageError(res, err) {
+        console.error("clickhouse write failed:", err);
+        res.status(503).json({ error: "storage unavailable, retry later" });
+    }
+    /** OTLP success responses mirror the request encoding: an empty
+     * protobuf message body for proto clients, JSON otherwise. */
+    function otlpSuccess(req, res) {
+        if (req.is("application/x-protobuf")) {
+            res.status(200).type("application/x-protobuf").end();
+            return;
+        }
+        res.status(200).json({ partialSuccess: {} });
+    }
+    app.post("/v1/traces", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        let request;
+        if (req.is("application/x-protobuf")) {
+            try {
+                request = decodeTraceRequest(req.body);
+            }
+            catch {
+                res.status(400).json({ error: "invalid protobuf payload" });
+                return;
+            }
+        }
+        else {
+            request = req.body;
+        }
+        const { occurrences, spans, metricPoints } = normalizeTraces(request);
+        const fingerprinted = fingerprintAll(occurrences);
+        try {
+            await Promise.all([
+                store.insertOccurrences(ctx, fingerprinted),
+                store.insertSpans(ctx, spans),
+                store.insertMetricPoints(ctx, metricPoints),
+            ]);
+        }
+        catch (err) {
+            storageError(res, err);
+            return;
+        }
+        forwardToSink(ctx, fingerprinted);
+        otlpSuccess(req, res);
+    });
+    app.post("/v1/metrics", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        let request;
+        if (req.is("application/x-protobuf")) {
+            try {
+                request = decodeMetricsRequest(req.body);
+            }
+            catch {
+                res.status(400).json({ error: "invalid protobuf payload" });
+                return;
+            }
+        }
+        else {
+            request = req.body;
+        }
+        const metricPoints = normalizeMetrics(request);
+        try {
+            await store.insertMetricPoints(ctx, metricPoints);
+        }
+        catch (err) {
+            storageError(res, err);
+            return;
+        }
+        otlpSuccess(req, res);
+    });
+    app.post("/v1/browser", async (req, res) => {
+        const ctx = await authenticate(req, res, "browser");
+        if (!ctx)
+            return;
+        let body = req.body;
+        if (typeof body === "string") {
+            // text/plain from a cross-origin sendBeacon — see CORS note above.
+            try {
+                body = JSON.parse(body);
+            }
+            catch {
+                res.status(400).json({ error: "invalid json" });
+                return;
+            }
+        }
+        const parsed = browserPayloadSchema.safeParse(body);
+        if (!parsed.success) {
+            res.status(400).json({
+                error: "invalid payload",
+                issues: parsed.error.issues.slice(0, 5),
+            });
+            return;
+        }
+        const { occurrences, metricPoints } = normalizeBrowserPayload(parsed.data);
+        const fingerprinted = fingerprintAll(occurrences);
+        try {
+            await Promise.all([
+                store.insertOccurrences(ctx, fingerprinted),
+                store.insertMetricPoints(ctx, metricPoints),
+            ]);
+        }
+        catch (err) {
+            storageError(res, err);
+            return;
+        }
+        forwardToSink(ctx, fingerprinted);
+        res.status(202).json({ accepted: fingerprinted.length });
+    });
+    // Body-parser errors (oversized/malformed JSON) → clean 4xx, not a stack.
+    app.use((err, _req, res, next) => {
+        if (res.headersSent)
+            return next(err);
+        if (err.type === "entity.too.large") {
+            res.status(413).json({ error: "payload too large" });
+            return;
+        }
+        if (err.status && err.status < 500) {
+            res.status(err.status).json({ error: "bad request" });
+            return;
+        }
+        console.error("unhandled error:", err);
+        res.status(500).json({ error: "internal error" });
+    });
+    return { app, store };
+}