@autter/otlp-ingester 0.1.0

package/dist/fingerprint.js

@@ -0,0 +1,80 @@
+import { createHash } from "node:crypto";
+/**
+ * Deterministic error grouping. Two occurrences share an issue when they
+ * share source + service + error type + normalised message + top stack
+ * frames + normalised route. Volatile fragments (ids, numbers, minified
+ * line/column offsets) are stripped so re-deploys and per-user values don't
+ * fragment issues. This algorithm must stay in sync with the Autter backend
+ * so browser-relay and OTLP occurrences group identically.
+ */
+const UUID_RE = /[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}/gi;
+const LONG_HEX_RE = /\b[0-9a-f]{8,}\b/gi;
+// No trailing \b: numbers glued to units ("4200ms", "3.5s", "512kb") must
+// template too, or per-value messages fragment into separate fingerprints.
+const NUMBER_RE = /\b\d+(\.\d+)?/g;
+const QUOTED_RE = /(["'`])(?:\\.|(?!\1).)*\1/g;
+export function normalizeMessage(message) {
+    return message
+        .slice(0, 500)
+        .replace(QUOTED_RE, "<str>")
+        .replace(UUID_RE, "<uuid>")
+        .replace(LONG_HEX_RE, "<hex>")
+        .replace(NUMBER_RE, "<n>")
+        .replace(/\s+/g, " ")
+        .trim();
+}
+/** Replace id-like path segments so /orders/812 and /orders/44 group. */
+export function normalizeRoute(route) {
+    if (!route)
+        return "";
+    const path = route.split("?")[0] ?? "";
+    return path
+        .split("/")
+        .map((segment) => {
+        if (!segment)
+            return segment;
+        if (/^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/i.test(segment))
+            return ":id";
+        if (/^\d+$/.test(segment))
+            return ":id";
+        if (/^[0-9a-f]{8,}$/i.test(segment))
+            return ":id";
+        return segment;
+    })
+        .join("/");
+}
+const FRAME_LOCATION_RE = /:\d+(:\d+)?\)?$/;
+export function normalizeStackFrames(stack, topN = 5) {
+    if (!stack)
+        return [];
+    return stack
+        .split("\n")
+        .map((line) => line.trim())
+        .filter((line) => /^at\s|@|^\s*File\s/.test(line) || /\.[jt]sx?/.test(line))
+        .slice(0, topN)
+        .map((line) => line
+        .replace(/\?[^:\s)]*/g, "")
+        .replace(FRAME_LOCATION_RE, "")
+        .replace(/\s+/g, " ")
+        .trim());
+}
+export function fingerprintOccurrence(input) {
+    const parts = [
+        input.source,
+        input.service,
+        input.errorType,
+        normalizeMessage(input.message),
+        ...normalizeStackFrames(input.stack),
+        normalizeRoute(input.route),
+    ];
+    return createHash("sha256").update(parts.join(" ")).digest("hex").slice(0, 32);
+}
+export function deriveFields(input) {
+    const topFrames = normalizeStackFrames(input.stack);
+    return {
+        routeNormalized: normalizeRoute(input.route),
+        messageNormalized: normalizeMessage(input.message),
+        topFrames,
+        firstFrame: topFrames[0] ?? "",
+    };
+}

package/dist/index.d.ts

@@ -0,0 +1,3 @@
+export { createIngesterApp } from "./server.js";
+export { loadConfig } from "./config.js";
+export * from "./types.js";

package/dist/index.js

@@ -0,0 +1,26 @@
+import { loadConfig } from "./config.js";
+import { createIngesterApp } from "./server.js";
+const config = loadConfig();
+const { app, store } = createIngesterApp(config);
+const server = app.listen(config.port, () => {
+    console.log(`autter otlp-ingester listening on :${config.port} ` +
+        `(clickhouse: ${config.clickhouseUrl ? "configured" : "NOT configured"})`);
+});
+// Warm the schema at boot so the first ingest request doesn't pay for DDL.
+if (store.configured) {
+    store.ensureSchema().catch((err) => {
+        console.error("clickhouse schema bootstrap failed (will retry on first ingest):", err?.message ?? err);
+    });
+}
+async function shutdown(signal) {
+    console.log(`${signal} received, shutting down`);
+    server.close(() => {
+        void store.close().finally(() => process.exit(0));
+    });
+    setTimeout(() => process.exit(1), 10_000).unref();
+}
+process.on("SIGTERM", () => void shutdown("SIGTERM"));
+process.on("SIGINT", () => void shutdown("SIGINT"));
+export { createIngesterApp } from "./server.js";
+export { loadConfig } from "./config.js";
+export * from "./types.js";

package/dist/migrations.d.ts

@@ -0,0 +1,42 @@
+/**
+ * ClickHouse schema migrations.
+ *
+ * The baseline schema in clickhouse.ts uses `CREATE TABLE IF NOT EXISTS`,
+ * which provisions FRESH databases but silently no-ops on existing ones —
+ * so schema changes to already-deployed databases go here. Migrations run
+ * automatically at ingester boot (and lazily before the first ingest),
+ * strictly in array order, and each applied id is recorded in
+ * `<db>.schema_migrations` so it runs exactly once per database.
+ *
+ * Rules for adding a migration:
+ * 1. Statements MUST be idempotent (`ADD COLUMN IF NOT EXISTS`,
+ *    `DROP COLUMN IF EXISTS`, `MODIFY TTL`, …) — multiple ingester
+ *    replicas may boot concurrently and race; idempotency makes the race
+ *    harmless. The tracking table is bookkeeping, not a lock.
+ * 2. Additive only within a major version: new columns need a DEFAULT so
+ *    old ingester replicas that are still running (rolling deploy) can
+ *    keep inserting without naming them.
+ * 3. ALSO update the baseline in clickhouse.ts `schemaStatements()` so
+ *    fresh databases come up with the final shape — a migration alone
+ *    only fixes existing databases.
+ * 4. Never edit or reorder shipped migrations; append a corrective one.
+ *
+ * `{db}` is replaced with the configured database name at run time.
+ *
+ * Example (what adding a column looks like):
+ *   {
+ *     id: "0002-occurrences-sdk-version",
+ *     statements: [
+ *       `ALTER TABLE {db}.runtime_error_occurrences
+ *          ADD COLUMN IF NOT EXISTS sdk_version String DEFAULT ''`,
+ *     ],
+ *   },
+ */
+export interface Migration {
+    /** Unique, ordered id: "<serial>-<slug>". Never reuse or reorder. */
+    id: string;
+    statements: string[];
+}
+export declare const MIGRATIONS: Migration[];
+/** The tracking table itself — created by the runner before anything else. */
+export declare function migrationsTableDDL(db: string): string;

package/dist/migrations.js

@@ -0,0 +1,88 @@
+/**
+ * ClickHouse schema migrations.
+ *
+ * The baseline schema in clickhouse.ts uses `CREATE TABLE IF NOT EXISTS`,
+ * which provisions FRESH databases but silently no-ops on existing ones —
+ * so schema changes to already-deployed databases go here. Migrations run
+ * automatically at ingester boot (and lazily before the first ingest),
+ * strictly in array order, and each applied id is recorded in
+ * `<db>.schema_migrations` so it runs exactly once per database.
+ *
+ * Rules for adding a migration:
+ * 1. Statements MUST be idempotent (`ADD COLUMN IF NOT EXISTS`,
+ *    `DROP COLUMN IF EXISTS`, `MODIFY TTL`, …) — multiple ingester
+ *    replicas may boot concurrently and race; idempotency makes the race
+ *    harmless. The tracking table is bookkeeping, not a lock.
+ * 2. Additive only within a major version: new columns need a DEFAULT so
+ *    old ingester replicas that are still running (rolling deploy) can
+ *    keep inserting without naming them.
+ * 3. ALSO update the baseline in clickhouse.ts `schemaStatements()` so
+ *    fresh databases come up with the final shape — a migration alone
+ *    only fixes existing databases.
+ * 4. Never edit or reorder shipped migrations; append a corrective one.
+ *
+ * `{db}` is replaced with the configured database name at run time.
+ *
+ * Example (what adding a column looks like):
+ *   {
+ *     id: "0002-occurrences-sdk-version",
+ *     statements: [
+ *       `ALTER TABLE {db}.runtime_error_occurrences
+ *          ADD COLUMN IF NOT EXISTS sdk_version String DEFAULT ''`,
+ *     ],
+ *   },
+ */
+export const MIGRATIONS = [
+    // 0001 intentionally reserved as the baseline marker: databases created
+    // before the migration runner existed record it as applied without
+    // running anything (the baseline CREATEs already shaped them).
+    { id: "0001-baseline", statements: [] },
+    // Aggregation-ready occurrence shape: severity (warnings/info share the
+    // table with errors), pre-normalised route/message, extracted stack
+    // frames, and the request method — so later aggregations GROUP BY
+    // stored columns instead of re-parsing stacks/routes in SQL.
+    {
+        id: "0002-occurrences-aggregation-columns",
+        statements: [
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS severity LowCardinality(String) DEFAULT 'error' AFTER source`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS message_normalized String DEFAULT '' AFTER message`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS top_frames Array(String) DEFAULT [] AFTER stack`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS first_frame String DEFAULT '' AFTER top_frames`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS route_normalized String DEFAULT '' AFTER route`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				ADD COLUMN IF NOT EXISTS method LowCardinality(String) DEFAULT '' AFTER route_normalized`,
+        ],
+    },
+    // ZSTD compression on the fat text columns — stack traces and JSON
+    // attributes compress several times better under ZSTD than the LZ4
+    // default, and these columns dominate on-disk size. MODIFY COLUMN with
+    // a codec is idempotent and cheap: new parts use it immediately, old
+    // parts recompress on background merges.
+    {
+        id: "0003-compress-fat-columns",
+        statements: [
+            `ALTER TABLE {db}.runtime_error_occurrences
+				MODIFY COLUMN stack String DEFAULT '' CODEC(ZSTD(3))`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				MODIFY COLUMN message String CODEC(ZSTD(1))`,
+            `ALTER TABLE {db}.runtime_error_occurrences
+				MODIFY COLUMN attributes String DEFAULT '{}' CODEC(ZSTD(1))`,
+            `ALTER TABLE {db}.runtime_spans
+				MODIFY COLUMN attributes String DEFAULT '{}' CODEC(ZSTD(1))`,
+        ],
+    },
+];
+/** The tracking table itself — created by the runner before anything else. */
+export function migrationsTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.schema_migrations (
+		id String,
+		applied_at DateTime('UTC') DEFAULT now()
+	)
+	ENGINE = ReplacingMergeTree
+	ORDER BY id`;
+}

package/dist/normalize-browser.d.ts

@@ -0,0 +1,103 @@
+import { z } from "zod";
+import { type RuntimeMetricPoint, type RuntimeOccurrenceInput } from "./types.js";
+export declare const browserPayloadSchema: z.ZodObject<{
+    version: z.ZodLiteral<1>;
+    sessionId: z.ZodOptional<z.ZodString>;
+    service: z.ZodString;
+    environment: z.ZodString;
+    release: z.ZodOptional<z.ZodString>;
+    events: z.ZodArray<z.ZodObject<{
+        id: z.ZodOptional<z.ZodString>;
+        type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event"]>;
+        timestamp: z.ZodString;
+        /** exception/message events: signal level. Defaults per type. */
+        severity: z.ZodOptional<z.ZodEnum<["fatal", "error", "warning", "info"]>>;
+        message: z.ZodDefault<z.ZodString>;
+        /** track_event only: the event name (counted, never free-form PII). */
+        name: z.ZodOptional<z.ZodString>;
+        stack: z.ZodOptional<z.ZodString>;
+        errorType: z.ZodOptional<z.ZodString>;
+        filename: z.ZodOptional<z.ZodString>;
+        line: z.ZodOptional<z.ZodNumber>;
+        column: z.ZodOptional<z.ZodNumber>;
+        /** Path only; query strings are stripped defensively anyway. */
+        route: z.ZodOptional<z.ZodString>;
+        context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+    }, "strip", z.ZodTypeAny, {
+        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        message: string;
+        timestamp: string;
+        id?: string | undefined;
+        severity?: "fatal" | "error" | "warning" | "info" | undefined;
+        name?: string | undefined;
+        stack?: string | undefined;
+        errorType?: string | undefined;
+        filename?: string | undefined;
+        line?: number | undefined;
+        column?: number | undefined;
+        route?: string | undefined;
+        context?: Record<string, unknown> | undefined;
+    }, {
+        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        timestamp: string;
+        id?: string | undefined;
+        message?: string | undefined;
+        severity?: "fatal" | "error" | "warning" | "info" | undefined;
+        name?: string | undefined;
+        stack?: string | undefined;
+        errorType?: string | undefined;
+        filename?: string | undefined;
+        line?: number | undefined;
+        column?: number | undefined;
+        route?: string | undefined;
+        context?: Record<string, unknown> | undefined;
+    }>, "many">;
+}, "strip", z.ZodTypeAny, {
+    version: 1;
+    service: string;
+    environment: string;
+    events: {
+        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        message: string;
+        timestamp: string;
+        id?: string | undefined;
+        severity?: "fatal" | "error" | "warning" | "info" | undefined;
+        name?: string | undefined;
+        stack?: string | undefined;
+        errorType?: string | undefined;
+        filename?: string | undefined;
+        line?: number | undefined;
+        column?: number | undefined;
+        route?: string | undefined;
+        context?: Record<string, unknown> | undefined;
+    }[];
+    sessionId?: string | undefined;
+    release?: string | undefined;
+}, {
+    version: 1;
+    service: string;
+    environment: string;
+    events: {
+        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        timestamp: string;
+        id?: string | undefined;
+        message?: string | undefined;
+        severity?: "fatal" | "error" | "warning" | "info" | undefined;
+        name?: string | undefined;
+        stack?: string | undefined;
+        errorType?: string | undefined;
+        filename?: string | undefined;
+        line?: number | undefined;
+        column?: number | undefined;
+        route?: string | undefined;
+        context?: Record<string, unknown> | undefined;
+    }[];
+    sessionId?: string | undefined;
+    release?: string | undefined;
+}>;
+export type BrowserPayload = z.infer<typeof browserPayloadSchema>;
+export interface NormalizedBrowser {
+    occurrences: RuntimeOccurrenceInput[];
+    metricPoints: RuntimeMetricPoint[];
+}
+export declare function normalizeBrowserPayload(payload: BrowserPayload): NormalizedBrowser;

package/dist/normalize-browser.js

@@ -0,0 +1,113 @@
+import { z } from "zod";
+import { asSeverity, } from "./types.js";
+/**
+ * The compact browser payload (`version: 1`) emitted by
+ * `@autter/runtime-browser` and forwarded by the customer's same-origin
+ * relay. The schema is a privacy gate as much as a validator: anything not
+ * whitelisted here never reaches storage.
+ */
+const browserEventSchema = z.object({
+    id: z.string().max(64).optional(),
+    type: z.enum([
+        "exception",
+        "unhandled_rejection",
+        "message",
+        "session_start",
+        "track_event",
+    ]),
+    timestamp: z.string().datetime(),
+    /** exception/message events: signal level. Defaults per type. */
+    severity: z.enum(["fatal", "error", "warning", "info"]).optional(),
+    message: z.string().max(4000).default(""),
+    /** track_event only: the event name (counted, never free-form PII). */
+    name: z.string().max(200).optional(),
+    stack: z.string().max(32000).optional(),
+    errorType: z.string().max(200).optional(),
+    filename: z.string().max(1000).optional(),
+    line: z.number().int().nonnegative().optional(),
+    column: z.number().int().nonnegative().optional(),
+    /** Path only; query strings are stripped defensively anyway. */
+    route: z.string().max(1000).optional(),
+    context: z.record(z.unknown()).optional(),
+});
+export const browserPayloadSchema = z.object({
+    version: z.literal(1),
+    sessionId: z.string().max(100).optional(),
+    service: z.string().min(1).max(200),
+    environment: z.string().min(1).max(100),
+    release: z.string().max(200).optional(),
+    events: z.array(browserEventSchema).max(50),
+});
+const TYPE_TO_ERROR_TYPE = {
+    exception: "Error",
+    unhandled_rejection: "UnhandledRejection",
+    message: "Message",
+};
+/** Default severity per event type when the SDK doesn't say. */
+const TYPE_TO_SEVERITY = {
+    exception: "error",
+    unhandled_rejection: "error",
+    message: "warning",
+};
+export function normalizeBrowserPayload(payload) {
+    const occurrences = [];
+    const rollups = new Map();
+    function bumpRollup(route, occurredAt, field) {
+        const bucketAt = new Date(Math.floor(occurredAt.getTime() / 60_000) * 60_000);
+        const key = `${route} ${bucketAt.getTime()}`;
+        const existing = rollups.get(key);
+        if (existing) {
+            existing[field] += 1;
+            return;
+        }
+        rollups.set(key, {
+            service: payload.service,
+            environment: payload.environment,
+            release: payload.release ?? null,
+            route,
+            bucketAt,
+            requestCount: field === "requestCount" ? 1 : 0,
+            errorCount: 0,
+            durationSumMs: 0,
+            sessionCount: field === "sessionCount" ? 1 : 0,
+        });
+    }
+    for (const event of payload.events) {
+        const occurredAt = new Date(event.timestamp);
+        if (event.type === "session_start") {
+            bumpRollup("", occurredAt, "sessionCount");
+            continue;
+        }
+        // Coarse usage counters: track_event("checkout_opened") becomes a
+        // request_count increment on the synthetic route "event:checkout_opened".
+        if (event.type === "track_event") {
+            const name = (event.name ?? event.message ?? "").slice(0, 200);
+            if (name)
+                bumpRollup(`event:${name}`, occurredAt, "requestCount");
+            continue;
+        }
+        occurrences.push({
+            source: "browser",
+            severity: asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error"),
+            service: payload.service,
+            environment: payload.environment,
+            release: payload.release ?? null,
+            errorType: event.errorType ?? TYPE_TO_ERROR_TYPE[event.type] ?? "Error",
+            message: event.message || "Unknown error",
+            stack: event.stack ?? null,
+            route: event.route ? (event.route.split("?")[0] ?? null) : null,
+            method: null,
+            statusCode: null,
+            traceId: null,
+            sessionId: payload.sessionId ?? null,
+            attributes: {
+                ...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
+                ...(event.line !== undefined ? { line: event.line } : {}),
+                ...(event.column !== undefined ? { column: event.column } : {}),
+                ...(event.context ? { context: event.context } : {}),
+            },
+            occurredAt,
+        });
+    }
+    return { occurrences, metricPoints: [...rollups.values()] };
+}

package/dist/normalize-otlp.d.ts

@@ -0,0 +1,84 @@
+import { type RuntimeMetricPoint, type RuntimeOccurrenceInput, type RuntimeSpanRow } from "./types.js";
+/**
+ * OTLP/HTTP JSON → runtime signal. Structural types cover only the fields
+ * we read (the full OTLP schema is large and versioned; unknown fields pass
+ * through untouched). Protobuf decode is Milestone 1 — see docs/PLAN.md.
+ *
+ * Note: OTLP/JSON encodes trace/span ids as hex strings and enum fields as
+ * either numbers or `SPAN_KIND_*` / `STATUS_CODE_*` strings depending on the
+ * serialiser — both are accepted.
+ */
+interface OtlpKeyValue {
+    key?: string;
+    value?: {
+        stringValue?: string;
+        intValue?: string | number;
+        doubleValue?: number;
+        boolValue?: boolean;
+    };
+}
+interface OtlpEvent {
+    name?: string;
+    timeUnixNano?: string | number;
+    attributes?: OtlpKeyValue[];
+}
+interface OtlpSpan {
+    traceId?: string;
+    spanId?: string;
+    parentSpanId?: string;
+    name?: string;
+    kind?: number | string;
+    startTimeUnixNano?: string | number;
+    endTimeUnixNano?: string | number;
+    attributes?: OtlpKeyValue[];
+    status?: {
+        code?: number | string;
+        message?: string;
+    };
+    events?: OtlpEvent[];
+}
+interface OtlpResource {
+    attributes?: OtlpKeyValue[];
+}
+export interface OtlpTraceRequest {
+    resourceSpans?: Array<{
+        resource?: OtlpResource;
+        scopeSpans?: Array<{
+            spans?: OtlpSpan[];
+        }>;
+    }>;
+}
+interface OtlpDataPoint {
+    attributes?: OtlpKeyValue[];
+    timeUnixNano?: string | number;
+    count?: string | number;
+    sum?: number;
+    asInt?: string | number;
+    asDouble?: number;
+}
+export interface OtlpMetricsRequest {
+    resourceMetrics?: Array<{
+        resource?: OtlpResource;
+        scopeMetrics?: Array<{
+            metrics?: Array<{
+                name?: string;
+                unit?: string;
+                histogram?: {
+                    dataPoints?: OtlpDataPoint[];
+                };
+                sum?: {
+                    dataPoints?: OtlpDataPoint[];
+                };
+            }>;
+        }>;
+    }>;
+}
+export interface NormalizedTraces {
+    occurrences: RuntimeOccurrenceInput[];
+    spans: RuntimeSpanRow[];
+    metricPoints: RuntimeMetricPoint[];
+    spanCount: number;
+}
+export declare function normalizeTraces(request: OtlpTraceRequest): NormalizedTraces;
+export declare function normalizeMetrics(request: OtlpMetricsRequest): RuntimeMetricPoint[];
+export {};