@autter/otlp-ingester 1.5.0 → 1.6.0

package/dist/symbolicate.js

@@ -0,0 +1,257 @@
+import { isIgnored, originalPositionFor, sourceContentFor, TraceMap, } from "@jridgewell/trace-mapping";
+import { isInAppPath, stripQueryAndHash } from "./stack-frames.js";
+const JS_FILE_RE = /\.[cm]?js$/i;
+const MAX_CONTEXT_CHARS = 300;
+const CONTEXT_LINES = 3;
+/** Path-only form of a script URL — the key uploads are stored under. */
+export function scriptPath(filename) {
+    try {
+        return new URL(filename, "https://autter.invalid").pathname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Decoded mappings take several times the JSON's size; the budget counts an
+ * estimate so a handful of large bundles cannot exhaust a small container.
+ */
+const DECODED_WEIGHT_FACTOR = 8;
+export class SourceMapCache {
+    entries = new Map();
+    inFlight = new Map();
+    weight = 0;
+    maxBytes;
+    maxEntries;
+    hitTtlMs;
+    missTtlMs;
+    errorTtlMs;
+    now;
+    /** Fetches made (cache misses) — for tests and /healthz-style debugging. */
+    fetches = 0;
+    constructor(options = {}) {
+        this.maxBytes = options.maxBytes ?? 64 * 1024 * 1024;
+        this.maxEntries = options.maxEntries ?? 500;
+        this.hitTtlMs = options.hitTtlMs ?? 60 * 60_000;
+        this.missTtlMs = options.missTtlMs ?? 5 * 60_000;
+        this.errorTtlMs = options.errorTtlMs ?? 30_000;
+        this.now = options.now ?? Date.now;
+    }
+    static keyOf(key) {
+        const scope = `${key.orgId}\u0000${key.repositoryId}\u0000`;
+        return key.debugId
+            ? `${scope}d\u0000${key.debugId}`
+            : `${scope}r\u0000${key.release ?? ""}\u0000${key.filename}`;
+    }
+    get size() {
+        return this.entries.size;
+    }
+    /** The parsed map for `key` (null when none), fetching at most once at a time. */
+    async get(key, fetchMap) {
+        const id = SourceMapCache.keyOf(key);
+        const hit = this.entries.get(id);
+        if (hit) {
+            if (hit.expiresAt > this.now()) {
+                // Refresh recency: Map iteration order is insertion order.
+                this.entries.delete(id);
+                this.entries.set(id, hit);
+                return hit.map;
+            }
+            this.remove(id);
+        }
+        const pending = this.inFlight.get(id);
+        if (pending)
+            return pending;
+        const load = this.load(id, key, fetchMap).finally(() => this.inFlight.delete(id));
+        this.inFlight.set(id, load);
+        return load;
+    }
+    async load(id, key, fetchMap) {
+        this.fetches += 1;
+        let json;
+        try {
+            json = await fetchMap(key);
+        }
+        catch (err) {
+            console.warn("source map lookup failed — frames stay unresolved:", err instanceof Error ? err.message : err);
+            this.store(id, null, 0, this.errorTtlMs);
+            return null;
+        }
+        if (!json) {
+            this.store(id, null, 0, this.missTtlMs);
+            return null;
+        }
+        let map;
+        try {
+            map = new TraceMap(json);
+        }
+        catch {
+            // A malformed upload is a miss until it is replaced.
+            this.store(id, null, 0, this.missTtlMs);
+            return null;
+        }
+        this.store(id, map, json.length * DECODED_WEIGHT_FACTOR, this.hitTtlMs);
+        return map;
+    }
+    store(id, map, weight, ttlMs) {
+        // A map bigger than the whole budget serves this lookup but is not kept:
+        // caching it would evict everything else.
+        if (weight > this.maxBytes)
+            return;
+        this.remove(id);
+        this.entries.set(id, { map, weight, expiresAt: this.now() + ttlMs });
+        this.weight += weight;
+        while ((this.weight > this.maxBytes || this.entries.size > this.maxEntries) &&
+            this.entries.size > 1) {
+            const oldest = this.entries.keys().next().value;
+            this.remove(oldest);
+        }
+    }
+    remove(id) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            return;
+        this.entries.delete(id);
+        this.weight -= entry.weight;
+    }
+}
+// ── Resolution ────────────────────────────────────────────────────────────
+/** A name a minifier evidently produced ("a", "Xe", "Object.t"). */
+function looksMangled(name) {
+    const bare = name.replace(/^(?:async\s+)?(?:new\s+)?/, "");
+    const last = bare.split(/[.\s]/).filter(Boolean).pop() ?? "";
+    return last.length <= 2;
+}
+function contextFor(map, source, line) {
+    let content = null;
+    try {
+        content = sourceContentFor(map, source);
+    }
+    catch {
+        content = null;
+    }
+    if (!content)
+        return {};
+    const lines = content.split(/\r?\n/);
+    const index = line - 1;
+    if (index < 0 || index >= lines.length)
+        return {};
+    const cut = (value) => value.slice(0, MAX_CONTEXT_CHARS);
+    return {
+        contextLine: cut(lines[index]),
+        preContext: lines.slice(Math.max(0, index - CONTEXT_LINES), index).map(cut),
+        postContext: lines.slice(index + 1, index + 1 + CONTEXT_LINES).map(cut),
+    };
+}
+export class Symbolicator {
+    fetchMap;
+    cache;
+    constructor(fetchMap, cache = new SourceMapCache()) {
+        this.fetchMap = fetchMap;
+        this.cache = cache;
+    }
+    /**
+     * Resolve the JS frames of one stack. Returns a new array; frames that
+     * cannot be resolved (no map, no position, non-JS) are returned as given.
+     */
+    async symbolicate(frames, scope) {
+        if (frames.length === 0)
+            return frames;
+        try {
+            return await this.resolveAll(frames, scope);
+        }
+        catch (err) {
+            console.warn("symbolication failed — using parsed frames:", err instanceof Error ? err.message : err);
+            return frames;
+        }
+    }
+    debugIdFor(filename, debugIds) {
+        if (!debugIds)
+            return null;
+        const direct = debugIds[filename];
+        if (typeof direct === "string" && direct)
+            return direct;
+        for (const [url, id] of Object.entries(debugIds)) {
+            if (typeof id === "string" && id && stripQueryAndHash(url) === filename)
+                return id;
+        }
+        return null;
+    }
+    keyFor(frame, scope) {
+        if (!frame.filename || frame.lineno == null || frame.lineno < 1)
+            return null;
+        if (!JS_FILE_RE.test(frame.filename))
+            return null;
+        const path = scriptPath(frame.filename);
+        if (!path)
+            return null;
+        const debugId = this.debugIdFor(frame.filename, scope.debugIds);
+        if (!debugId && !scope.release)
+            return null;
+        return {
+            orgId: scope.orgId,
+            repositoryId: scope.repositoryId,
+            release: scope.release,
+            filename: path,
+            ...(debugId ? { debugId } : {}),
+        };
+    }
+    async resolveAll(frames, scope) {
+        const keys = frames.map((frame) => this.keyFor(frame, scope));
+        if (!keys.some(Boolean))
+            return frames;
+        const maps = await Promise.all(keys.map((key) => (key ? this.cache.get(key, this.fetchMap) : Promise.resolve(null))));
+        const resolutions = frames.map((frame, i) => {
+            const map = maps[i];
+            if (!map)
+                return null;
+            try {
+                return resolveFrame(map, frame);
+            }
+            catch {
+                return null;
+            }
+        });
+        if (!resolutions.some(Boolean))
+            return frames;
+        return frames.map((frame, i) => {
+            const resolution = resolutions[i];
+            if (!resolution)
+                return frame;
+            // Innermost first: the caller is the next frame.
+            const callerName = resolutions[i + 1]?.nameHere ?? null;
+            const parsed = frame.function;
+            const name = callerName ?? (parsed && !looksMangled(parsed) ? parsed : null);
+            return { ...resolution.frame, function: name };
+        });
+    }
+}
+function resolveFrame(map, frame) {
+    const position = originalPositionFor(map, {
+        line: frame.lineno,
+        // Stack columns are 1-based, source map columns 0-based.
+        column: Math.max(0, (frame.colno ?? 1) - 1),
+    });
+    if (!position.source || position.line == null)
+        return null;
+    const filename = stripQueryAndHash(position.source);
+    const ignored = (() => {
+        try {
+            return isIgnored(map, position.source);
+        }
+        catch {
+            return false;
+        }
+    })();
+    const resolved = {
+        function: frame.function,
+        module: null,
+        filename,
+        lineno: position.line,
+        colno: position.column == null ? null : position.column + 1,
+        inApp: !ignored && isInAppPath(filename),
+        resolved: true,
+        ...contextFor(map, position.source, position.line),
+    };
+    return { frame: resolved, nameHere: position.name ?? null };
+}

package/dist/types.d.ts

@@ -1,3 +1,4 @@
+import type { Breadcrumb } from "./breadcrumbs.js";
 export type RuntimeSource = "browser" | "server";
 /**
  * Signal severity. Errors AND warnings/info land in the same occurrences
@@ -44,6 +45,60 @@ export interface RuntimeOccurrenceInput {
     expected?: boolean;
     /** `autter.request.id` — joins the occurrence to its request summary. */
     requestId?: string;
+    /**
+     * SDK fingerprint override (browser `fingerprint`, OTLP/log
+     * `autter.fingerprint`): ≤10 scrubbed values, each ≤200 chars. Hashed into
+     * the `manual-v1` candidate by `manualFingerprint` (fingerprint-override.ts).
+     * Omitted when absent.
+     */
+    fingerprintOverride?: string[];
+    /** Steps before the error, oldest first, ≤ 30 (validated by breadcrumbs.ts). */
+    breadcrumbs?: Breadcrumb[];
+    /**
+     * Browser only: script URL → debug id for the scripts on the page, used
+     * to find their source maps (EI-13). Input-only: never stored or sent to
+     * the sink.
+     */
+    debugIds?: Record<string, string>;
+}
+/**
+ * One stack frame, innermost (crash site) first. The sink wire shape
+ * (contract §1) — also stored as JSON in `runtime_error_occurrences.frames`.
+ */
+export interface StackFrame {
+    /** Resolved name when symbolicated, else as parsed. */
+    function: string | null;
+    /** Package / module / class when the language has one. */
+    module: string | null;
+    /** Path as resolved (the source map `sources` entry when resolved), query/hash stripped. */
+    filename: string | null;
+    lineno: number | null;
+    colno: number | null;
+    /** False for dependency, runtime and extension frames (stack-frames.ts §3). */
+    inApp: boolean;
+    /** True when a source map resolved the frame. */
+    resolved: boolean;
+    /** Only when `sourcesContent` supplied it (≤ 300 chars). */
+    contextLine?: string;
+    /** ≤ 3 lines each, ≤ 300 chars per line. */
+    preContext?: string[];
+    postContext?: string[];
+}
+/**
+ * Every grouping scheme an issue identity can come from (contract §1).
+ * The ingester computes `manual-v1`, `code-v1`, `frames-v3` and
+ * `message-v1`; `rule-v1` and `external-v1` exist only on the platform.
+ */
+export type FingerprintSchemeV2 = "manual-v1" | "rule-v1" | "code-v1" | "frames-v3" | "message-v1" | "external-v1";
+export interface FingerprintCandidate {
+    scheme: FingerprintSchemeV2;
+    /** 32 lowercase hex characters. */
+    fingerprint: string;
+}
+/** What went into a fingerprint, for the "why these were grouped" UI. */
+export interface FingerprintRecord {
+    scheme: string;
+    parts: string[];
 }
 /**
  * Derived, aggregation-ready fields — computed once at ingest (from the
@@ -52,9 +107,22 @@ export interface RuntimeOccurrenceInput {
  */
 export interface RuntimeOccurrence extends RuntimeOccurrenceInput {
     occurrenceId: string;
+    /**
+     * The LEGACY identity, byte-for-byte as before 1.6.0 so an older
+     * platform groups exactly as it did: "code-v1" or "message-v1".
+     */
     fingerprint: string;
     /** "code-v1" when grouped by a valid error code, else "message-v1". */
     fingerprintScheme: "message-v1" | "code-v1";
+    /** Parsed (and, for browser JS, symbolicated) frames, innermost first, ≤ 50. */
+    frames?: StackFrame[];
+    /**
+     * Every applicable identity, highest precedence first (manual-v1,
+     * code-v1, frames-v3, message-v1); always ends with the legacy one.
+     */
+    fingerprints?: FingerprintCandidate[];
+    /** What went into the frames-v3 fingerprint, when one was computed. */
+    fingerprintRecord?: FingerprintRecord;
     /** normalizeRoute(route) — "/users/:id/orders/:id"; GROUP BY-safe. */
     routeNormalized: string;
     /** normalizeMessage(message) — ids/numbers/strings templated out. */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.5.0",
+	"version": "1.6.0",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",
@@ -19,6 +19,7 @@
 	},
 	"dependencies": {
 		"@clickhouse/client": "^1.12.0",
+		"@jridgewell/trace-mapping": "^0.3.31",
 		"express": "^4.21.2",
 		"protobufjs": "^7.6.5",
 		"zod": "^3.24.1"