@autter/otlp-ingester 1.5.0 → 1.6.1

package/dist/fingerprint-v3.js

@@ -0,0 +1,96 @@
+import { createHash } from "node:crypto";
+import { normalizeMessage, occurrenceFingerprint } from "./fingerprint.js";
+import { manualFingerprint } from "./fingerprint-override.js";
+/**
+ * `frames-v3` — the structural issue identity (contract §2). Byte-for-byte
+ * shared with the platform mirror (`runtime-frames-fingerprint.ts`); the
+ * vectors in fingerprint-v3-vectors.json pin both.
+ *
+ * It hashes service + error type + up to five in-app frames reduced to
+ * `function@file`, where file is a lowercased basename with build hashes
+ * (`index-5f3a9c1e.js` → `index-*.js`) templated out. Line and column
+ * numbers, route, source (browser vs server) and release are never part of
+ * it, so one bug stays one issue across deploys, pages, preview domains and
+ * tiers. The normalised message is hashed only when no frame was picked.
+ */
+export const FRAMES_V3 = "frames-v3";
+const MAX_PICKED_FRAMES = 5;
+/** §2.3 — the function half of a frame token. */
+export function frameTokenFunction(frame) {
+    let fn = frame.function ?? "";
+    fn = fn.replace(/^async /, "");
+    fn = fn.replace(/^new /, "");
+    fn = fn.replace(/^(?:Object|Module)\./, "");
+    if (frame.resolved === false && fn.length <= 2)
+        fn = "";
+    if (fn === "<anonymous>" || fn === "anonymous")
+        fn = "";
+    return fn;
+}
+/** §2.3 — the file half of a frame token. */
+export function frameTokenFile(frame) {
+    let file = frame.filename ?? frame.module ?? "";
+    const cut = file.search(/[?#]/);
+    if (cut !== -1)
+        file = file.slice(0, cut);
+    file = file.slice(Math.max(file.lastIndexOf("/"), file.lastIndexOf("\\")) + 1);
+    file = file.replace(/[A-Za-z0-9]{8,}/g, (run) => /[0-9]/.test(run) || run === run.toUpperCase() ? "*" : run);
+    return file.toLowerCase();
+}
+export function frameToken(frame) {
+    return `${frameTokenFunction(frame)}@${frameTokenFile(frame)}`;
+}
+/** §2.1–2.2 — in-app frames in order, else only the first frame; at most 5. */
+export function pickFrames(frames) {
+    const inApp = frames.filter((frame) => frame.inApp === true);
+    const picked = inApp.length > 0 ? inApp : frames.slice(0, 1);
+    return picked.slice(0, MAX_PICKED_FRAMES);
+}
+export function framesV3Fingerprint(input) {
+    const tokens = pickFrames(input.frames).map(frameToken);
+    const message = tokens.length === 0 ? normalizeMessage(input.message) : "";
+    const fingerprint = createHash("sha256")
+        .update([FRAMES_V3, input.service, input.errorType, tokens.join("\u0001"), message].join("\u0000"))
+        .digest("hex")
+        .slice(0, 32);
+    return {
+        fingerprint,
+        record: {
+            scheme: FRAMES_V3,
+            parts: [input.errorType, ...tokens, message].filter((part) => part !== ""),
+        },
+    };
+}
+/**
+ * The candidate list the sink carries (contract §1/§4), highest precedence
+ * first:
+ *   manual-v1 (SDK override) → code-v1 (valid error code) → frames-v3 →
+ *   message-v1 (legacy).
+ * A coded error's legacy identity IS code-v1, which outranks every auto
+ * scheme, so its list is [manual-v1?, code-v1]; an uncoded one is
+ * [manual-v1?, frames-v3, message-v1]. Either way the list ends with the
+ * legacy fingerprint, so the platform can always fall back to it.
+ */
+export function occurrenceIdentity(input, frames) {
+    const legacy = occurrenceFingerprint(input);
+    const candidates = [];
+    const manual = manualFingerprint(input);
+    if (manual)
+        candidates.push({ scheme: "manual-v1", fingerprint: manual });
+    let record = null;
+    if (legacy.scheme === "code-v1") {
+        candidates.push({ scheme: "code-v1", fingerprint: legacy.fingerprint });
+    }
+    else {
+        const v3 = framesV3Fingerprint({
+            service: input.service,
+            errorType: input.errorType,
+            message: input.message,
+            frames,
+        });
+        record = v3.record;
+        candidates.push({ scheme: FRAMES_V3, fingerprint: v3.fingerprint });
+        candidates.push({ scheme: "message-v1", fingerprint: legacy.fingerprint });
+    }
+    return { legacy, candidates, record };
+}

package/dist/fingerprint.d.ts

@@ -2,6 +2,31 @@ import type { RuntimeOccurrenceInput } from "./types.js";
 export declare function normalizeMessage(message: string): string;
 /** Replace id-like path segments so /orders/812 and /orders/44 group. */
 export declare function normalizeRoute(route: string | null): string;
+/**
+ * Backend exceptions reach us in many native stack formats. The fingerprint
+ * groups by the top stack frames, so a format we cannot parse yields an EMPTY
+ * frame list — and then every error sharing a message collapses into one
+ * issue (two unrelated Go panics, say, become a single group). To keep
+ * unrelated defects apart, each officially-supported language is parsed and
+ * every frame reduced to a stable `function (file)` token that preserves the
+ * function, module and file.
+ *
+ * Volatile parts are stripped: line/column numbers, `+0x` offsets, pointer
+ * arguments and memory addresses. Line numbers are parsed but deliberately
+ * left OUT of the token — they shift on every re-deploy, and the JS path has
+ * always dropped them for exactly this reason, so the same defect stays one
+ * issue across rebuilds. The raw stack is retained verbatim on its own column.
+ *
+ * JavaScript/TypeScript, Firefox `@` frames and Python `File` frames keep
+ * their historical normalisation byte-for-byte (parseScriptFrames) so existing
+ * issues never regroup.
+ */
+export type StackLanguage = "go" | "rust" | "jvm" | "dotnet" | "script";
+/**
+ * Classify a whole stack by language. Signatures are chosen to be unique to
+ * each runtime so a JS or Python stack always falls through to "script".
+ */
+export declare function detectStackLanguage(lines: string[]): StackLanguage;
 export declare function normalizeStackFrames(stack: string | null, topN?: number): string[];
 /**
  * Error codes (`autter.error.code`) — namespaced, stable, low-cardinality

package/dist/fingerprint.js

@@ -70,7 +70,7 @@ const RUST_AT_RS_RE = /^\s*at\s+\S+\.rs:\d+/;
  * Classify a whole stack by language. Signatures are chosen to be unique to
  * each runtime so a JS or Python stack always falls through to "script".
  */
-function detectStackLanguage(lines) {
+export function detectStackLanguage(lines) {
     let hasDotnet = false;
     let hasJvm = false;
     let hasRust = false;

package/dist/logs.js

@@ -1,6 +1,7 @@
 import { createHash } from "node:crypto";
 import { decodeOtlpAttributes, sanitizeRuntimeContext, } from "./context.js";
 import { declaredLink, liftErrorFields } from "./error-fields.js";
+import { fingerprintOverrideOf } from "./fingerprint-override.js";
 import { normalizeRoute } from "./fingerprint.js";
 export function normalizeLogs(request) {
     if (!request || !Array.isArray(request.resourceLogs))
@@ -122,6 +123,7 @@ export function normalizeLogs(request) {
                         attributes,
                         occurredAt,
                         ...declared,
+                        ...fingerprintOverrideOf(attributes["autter.fingerprint"]),
                     }
                     : undefined;
                 rows.push({

package/dist/memory.d.ts

@@ -24,16 +24,16 @@ export declare const platformEventSchema: z.ZodObject<{
     service: string;
     environment: string;
     release: string;
+    occurredAt: string;
     kind: "oom_kill" | "restart";
     instanceId: string;
     platform: "ecs" | "kubernetes" | "other";
-    occurredAt: string;
 }, {
     service: string;
+    occurredAt: string;
     kind: "oom_kill" | "restart";
     instanceId: string;
     platform: "ecs" | "kubernetes" | "other";
-    occurredAt: string;
     environment?: string | undefined;
     release?: string | undefined;
 }>;

package/dist/migrations.d.ts

@@ -35,6 +35,16 @@
 /** Migration 0013 columns — shared with the baseline DDL in clickhouse.ts. */
 export declare const OCCURRENCE_CODE_COLUMNS: string[];
 export declare const OCCURRENCE_REQUEST_ID_INDEX = "INDEX idx_occ_request_id request_id TYPE bloom_filter GRANULARITY 4";
+/**
+ * Migration 0016 columns — shared with the baseline DDL in clickhouse.ts.
+ * `frames`: the structured (symbolicated) frames as JSON, '' when none.
+ * `fingerprints`: every identity candidate (manual-v1, code-v1, frames-v3,
+ * message-v1 — highest precedence first, ending with the legacy one), so
+ * readers find an issue keyed by ANY scheme with
+ * `(fingerprint = X OR has(fingerprints, X))`; rows written before 0016
+ * read `[]` and still match on the legacy `fingerprint`.
+ */
+export declare const OCCURRENCE_GROUPING_COLUMNS: string[];
 export interface Migration {
     /** Unique, ordered id: "<serial>-<slug>". Never reuse or reorder. */
     id: string;

package/dist/migrations.js

@@ -34,7 +34,7 @@
  */
 import { latencyTableDDL } from "./latency.js";
 import { profileTableDDL } from "./profiles.js";
-import { sourceMapTableDDL } from "./source-maps.js";
+import { SOURCE_MAP_DEBUG_COLUMNS, SOURCE_MAP_DEBUG_ID_INDEX, sourceMapTableDDL, } from "./source-maps.js";
 import { LOG_REQUEST_COLUMNS, LOG_REQUEST_ID_INDEX, logTableDDL, requestRollupTableDDL, requestRollupViewDDL, } from "./logs.js";
 import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
 import { ingesterInfoTableDDL, sdkVersionTableDDL } from "./sdk-versions.js";
@@ -49,6 +49,19 @@ export const OCCURRENCE_CODE_COLUMNS = [
     "fingerprint_scheme LowCardinality(String) DEFAULT 'message-v1'",
 ];
 export const OCCURRENCE_REQUEST_ID_INDEX = "INDEX idx_occ_request_id request_id TYPE bloom_filter GRANULARITY 4";
+/**
+ * Migration 0016 columns — shared with the baseline DDL in clickhouse.ts.
+ * `frames`: the structured (symbolicated) frames as JSON, '' when none.
+ * `fingerprints`: every identity candidate (manual-v1, code-v1, frames-v3,
+ * message-v1 — highest precedence first, ending with the legacy one), so
+ * readers find an issue keyed by ANY scheme with
+ * `(fingerprint = X OR has(fingerprints, X))`; rows written before 0016
+ * read `[]` and still match on the legacy `fingerprint`.
+ */
+export const OCCURRENCE_GROUPING_COLUMNS = [
+    "frames String DEFAULT '' CODEC(ZSTD(3))",
+    "fingerprints Array(String) DEFAULT []",
+];
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -171,6 +184,24 @@ export const MIGRATIONS = [
     // Version compatibility: SDK name/version seen per service, and this
     // ingester's own version + schema level for the Autter dashboard.
     { id: "0015-runtime-compat", statements: [sdkVersionTableDDL("{db}"), ingesterInfoTableDDL("{db}")] },
+    // Structured stacks and multi-scheme identity (frames-v3): the platform
+    // resolves an issue from any candidate, so occurrence lookups by an
+    // issue's fingerprint must match every candidate a row carried.
+    {
+        id: "0016-occurrence-frames-fingerprints",
+        statements: OCCURRENCE_GROUPING_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
+    },
+    // Source map debug ids (EI-13): the bundle's debug id, so a map matches
+    // by the bytes it was built from whatever release the SDK reports, plus
+    // the stored map's sha256 so CI skips maps it already uploaded. The
+    // sorting key is unchanged: uploads still need a release.
+    {
+        id: "0017-source-maps-debug-id",
+        statements: [
+            ...SOURCE_MAP_DEBUG_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_source_maps ADD COLUMN IF NOT EXISTS ${column}`),
+            `ALTER TABLE {db}.runtime_source_maps ADD ${SOURCE_MAP_DEBUG_ID_INDEX.replace("INDEX ", "INDEX IF NOT EXISTS ")}`,
+        ],
+    },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -34,56 +34,85 @@ export declare const browserPayloadSchema: z.ZodObject<{
          * instead of rejecting the whole beacon.
          */
         context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
+        /** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
+        breadcrumbs: z.ZodCatch<z.ZodOptional<z.ZodArray<z.ZodUnknown, "many">>>;
+        /** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
+         * event's stack (EI-13). Malformed values are dropped, never the beacon. */
+        debugIds: z.ZodCatch<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
+        /**
+         * Fingerprint override (runtime-browser 1.6.0+): one string or a list.
+         * Bounded and scrubbed in fingerprint-override.ts; a malformed value is
+         * dropped on its own.
+         */
+        fingerprint: z.ZodCatch<z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>>;
+        /**
+         * runtime-browser ≥1.6.0: W3C trace id of the request behind the event
+         * (the `traceparent` it sent), joining it to the server trace. A malformed
+         * id is dropped on its own instead of rejecting the beacon.
+         */
+        traceId: z.ZodCatch<z.ZodOptional<z.ZodString>>;
     }, "strip", z.ZodTypeAny, {
-        type: "exception" | "message" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
+        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
-        durationMs?: number | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
+        errorType?: string | undefined;
+        stack?: string | undefined;
         route?: string | undefined;
+        traceId?: string | undefined;
+        breadcrumbs?: unknown[] | undefined;
+        debugIds?: Record<string, unknown> | undefined;
+        durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        severity?: "fatal" | "error" | "warning" | "info" | undefined;
-        name?: string | undefined;
-        stack?: string | undefined;
-        errorType?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
+        fingerprint?: string | string[] | undefined;
     }, {
-        type: "exception" | "message" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
-        durationMs?: number | undefined;
-        route?: string | undefined;
         message?: string | undefined;
-        filename?: string | undefined;
-        id?: string | undefined;
-        severity?: "fatal" | "error" | "warning" | "info" | undefined;
-        name?: string | undefined;
-        stack?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
-        line?: number | undefined;
+        stack?: string | undefined;
+        route?: string | undefined;
+        traceId?: unknown;
+        breadcrumbs?: unknown;
+        debugIds?: unknown;
+        durationMs?: number | undefined;
+        filename?: string | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
+        fingerprint?: unknown;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
     service: string;
     environment: string;
     version: 1;
     events: {
-        type: "exception" | "message" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
+        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
-        durationMs?: number | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
+        errorType?: string | undefined;
+        stack?: string | undefined;
         route?: string | undefined;
+        traceId?: string | undefined;
+        breadcrumbs?: unknown[] | undefined;
+        debugIds?: Record<string, unknown> | undefined;
+        durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        severity?: "fatal" | "error" | "warning" | "info" | undefined;
-        name?: string | undefined;
-        stack?: string | undefined;
-        errorType?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
+        fingerprint?: string | string[] | undefined;
     }[];
     release?: string | undefined;
     sessionId?: string | undefined;
@@ -93,20 +122,24 @@ export declare const browserPayloadSchema: z.ZodObject<{
     environment: string;
     version: 1;
     events: {
-        type: "exception" | "message" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
-        durationMs?: number | undefined;
-        route?: string | undefined;
         message?: string | undefined;
-        filename?: string | undefined;
-        id?: string | undefined;
-        severity?: "fatal" | "error" | "warning" | "info" | undefined;
-        name?: string | undefined;
-        stack?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
-        line?: number | undefined;
+        stack?: string | undefined;
+        route?: string | undefined;
+        traceId?: unknown;
+        breadcrumbs?: unknown;
+        debugIds?: unknown;
+        durationMs?: number | undefined;
+        filename?: string | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
+        fingerprint?: unknown;
     }[];
     release?: string | undefined;
     sessionId?: string | undefined;

package/dist/normalize-browser.js

@@ -1,8 +1,11 @@
 import { z } from "zod";
 import { liftErrorFields } from "./error-fields.js";
+import { fingerprintOverrideOf } from "./fingerprint-override.js";
 import { sanitizeRuntimeContext } from "./context.js";
 import { normalizeRoute } from "./fingerprint.js";
 import { scrubText } from "./redact.js";
+import { breadcrumbsField } from "./breadcrumbs.js";
+import { browserDebugIds } from "./source-maps.js";
 import { asSeverity, } from "./types.js";
 /**
  * The compact browser payload (`version: 1`) emitted by
@@ -46,6 +49,23 @@ const browserEventSchema = z.object({
      * instead of rejecting the whole beacon.
      */
     context: z.record(z.unknown()).optional(),
+    /** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
+    breadcrumbs: z.array(z.unknown()).optional().catch(undefined),
+    /** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
+     * event's stack (EI-13). Malformed values are dropped, never the beacon. */
+    debugIds: z.record(z.string().max(1000), z.unknown()).optional().catch(undefined),
+    /**
+     * Fingerprint override (runtime-browser 1.6.0+): one string or a list.
+     * Bounded and scrubbed in fingerprint-override.ts; a malformed value is
+     * dropped on its own.
+     */
+    fingerprint: z.union([z.string().max(2000), z.array(z.string().max(2000)).max(50)]).optional().catch(undefined),
+    /**
+     * runtime-browser ≥1.6.0: W3C trace id of the request behind the event
+     * (the `traceparent` it sent), joining it to the server trace. A malformed
+     * id is dropped on its own instead of rejecting the beacon.
+     */
+    traceId: z.string().regex(/^(?!0{32})[0-9a-f]{32}$/).optional().catch(undefined),
 });
 export const browserPayloadSchema = z.object({
     version: z.literal(1),
@@ -156,7 +176,7 @@ export function normalizeBrowserPayload(payload) {
             route: event.route ? (event.route.split("?")[0] ?? null) : null,
             method: null,
             statusCode: null,
-            traceId: null,
+            traceId: event.traceId ?? null,
             sessionId: payload.sessionId ?? null,
             attributes: {
                 ...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
@@ -166,6 +186,9 @@ export function normalizeBrowserPayload(payload) {
             },
             occurredAt,
             ...liftErrorFields(event.context),
+            ...breadcrumbsField(event.breadcrumbs),
+            ...browserDebugIds(event.debugIds),
+            ...fingerprintOverrideOf(event.fingerprint),
         });
     }
     return { occurrences, metricPoints: [...rollups.values()] };

package/dist/normalize-otlp.js

@@ -1,8 +1,10 @@
 import { decodeOtlpAttributes, sanitizeRuntimeContext } from "./context.js";
 import { liftErrorFields, validRequestId } from "./error-fields.js";
+import { fingerprintOverrideOf } from "./fingerprint-override.js";
 import { scrubText } from "./redact.js";
 import { normalizeRoute } from "./fingerprint.js";
 import { extractLlmCall } from "./llm.js";
+import { BREADCRUMBS_ATTRIBUTE, breadcrumbsField } from "./breadcrumbs.js";
 import { asSeverity, } from "./types.js";
 const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
 function attrMap(attributes) {
@@ -172,6 +174,8 @@ export function normalizeTraces(request) {
                         // Event attributes first: one span can carry several
                         // exceptions, each with its own code.
                         ...liftErrorFields(eventAttrs, attrs),
+                        ...breadcrumbsField(eventAttrs.get(BREADCRUMBS_ATTRIBUTE), attrs.get(BREADCRUMBS_ATTRIBUTE)),
+                        ...fingerprintOverrideOf(eventAttrs.get("autter.fingerprint") ?? attrs.get("autter.fingerprint")),
                     });
                 }
                 const failedOutcomes = (span.events ?? []).filter((event) => {