@autter/otlp-ingester 1.5.0 → 1.6.1

package/dist/symbolicate.js

@@ -0,0 +1,363 @@
+import { isIgnored, sourceContentFor, TraceMap, } from "@jridgewell/trace-mapping";
+import { DecodedMappings, decodedMappingsBytes, findInMappings, } from "./source-map-mappings.js";
+import { SCOPES_KEY, ScopeIndex } from "./source-map-scopes.js";
+import { isInAppPath, stripQueryAndHash } from "./stack-frames.js";
+const JS_FILE_RE = /\.[cm]?js$/i;
+const MAX_CONTEXT_CHARS = 300;
+const CONTEXT_LINES = 3;
+/** Path-only form of a script URL — the key uploads are stored under. */
+export function scriptPath(filename) {
+    try {
+        return new URL(filename, "https://autter.invalid").pathname;
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * The largest share of the budget one map may take. A bigger one is used
+ * for its lookup and dropped: keeping it would evict most other maps, and
+ * every event in those files would then refetch theirs.
+ */
+const MAX_SHARE_OF_BUDGET = 0.5;
+export class SourceMapCache {
+    entries = new Map();
+    inFlight = new Map();
+    weight = 0;
+    maxBytes;
+    maxEntries;
+    hitTtlMs;
+    missTtlMs;
+    errorTtlMs;
+    now;
+    /** Fetches made (cache misses) — for tests and /healthz-style debugging. */
+    fetches = 0;
+    /** Maps used without being cached because they were too big to keep. */
+    bypassed = 0;
+    constructor(options = {}) {
+        this.maxBytes = options.maxBytes ?? 64 * 1024 * 1024;
+        this.maxEntries = options.maxEntries ?? 500;
+        this.hitTtlMs = options.hitTtlMs ?? 60 * 60_000;
+        this.missTtlMs = options.missTtlMs ?? 5 * 60_000;
+        this.errorTtlMs = options.errorTtlMs ?? 30_000;
+        this.now = options.now ?? Date.now;
+    }
+    static keyOf(key) {
+        const scope = `${key.orgId}\u0000${key.repositoryId}\u0000`;
+        return key.debugId
+            ? `${scope}d\u0000${key.debugId}`
+            : `${scope}r\u0000${key.release ?? ""}\u0000${key.filename}`;
+    }
+    get size() {
+        return this.entries.size;
+    }
+    /** Estimated bytes of the cached maps (what `maxBytes` bounds). */
+    get bytes() {
+        return this.weight;
+    }
+    /** The parsed map for `key` (null when none), fetching at most once at a time. */
+    async get(key, fetchMap) {
+        const id = SourceMapCache.keyOf(key);
+        const hit = this.entries.get(id);
+        if (hit) {
+            if (hit.expiresAt > this.now()) {
+                // Refresh recency: Map iteration order is insertion order.
+                this.entries.delete(id);
+                this.entries.set(id, hit);
+                return hit.map;
+            }
+            this.remove(id);
+        }
+        const pending = this.inFlight.get(id);
+        if (pending)
+            return pending;
+        const load = this.load(id, key, fetchMap).finally(() => this.inFlight.delete(id));
+        this.inFlight.set(id, load);
+        return load;
+    }
+    async load(id, key, fetchMap) {
+        this.fetches += 1;
+        let json;
+        try {
+            json = await fetchMap(key);
+        }
+        catch (err) {
+            console.warn("source map lookup failed — frames stay unresolved:", err instanceof Error ? err.message : err);
+            this.store(id, null, 0, this.errorTtlMs);
+            return null;
+        }
+        if (!json) {
+            this.store(id, null, 0, this.missTtlMs);
+            return null;
+        }
+        let loaded;
+        try {
+            loaded = parseSourceMap(json, this.maxBytes * MAX_SHARE_OF_BUDGET);
+        }
+        catch {
+            // A malformed upload is a miss until it is replaced.
+            this.store(id, null, 0, this.missTtlMs);
+            return null;
+        }
+        if (loaded.cacheable)
+            this.store(id, loaded, loaded.bytes, this.hitTtlMs);
+        else
+            this.bypassed += 1;
+        return loaded;
+    }
+    store(id, map, weight, ttlMs) {
+        // A map bigger than the whole budget serves this lookup but is not kept:
+        // caching it would evict everything else.
+        if (weight > this.maxBytes)
+            return;
+        this.remove(id);
+        this.entries.set(id, { map, weight, expiresAt: this.now() + ttlMs });
+        this.weight += weight;
+        while ((this.weight > this.maxBytes || this.entries.size > this.maxEntries) &&
+            this.entries.size > 1) {
+            const oldest = this.entries.keys().next().value;
+            this.remove(oldest);
+        }
+    }
+    remove(id) {
+        const entry = this.entries.get(id);
+        if (!entry)
+            return;
+        this.entries.delete(id);
+        this.weight -= entry.weight;
+    }
+}
+/** Rough heap of a list of strings (V8 strings plus array slots). */
+function stringsBytes(list) {
+    if (!Array.isArray(list))
+        return 0;
+    let bytes = 16;
+    for (const item of list)
+        bytes += 8 + (typeof item === "string" ? 2 * item.length + 24 : 0);
+    return bytes;
+}
+/**
+ * Parse a stored map once: metadata, scopes (null when absent or malformed
+ * — a bad scope field never loses the map) and position lookups. When the
+ * estimated heap is at most `maxDecodedBytes`, the mappings are decoded
+ * once into a compact table; otherwise they are left encoded and each
+ * lookup reads them in one pass (`cacheable: false`). Throws when the map
+ * itself is unusable.
+ */
+export function parseSourceMap(json, maxDecodedBytes = Number.POSITIVE_INFINITY) {
+    const raw = JSON.parse(json);
+    if (!raw || typeof raw !== "object" || Array.isArray(raw))
+        throw new Error("not a source map");
+    const fields = raw;
+    const { mappings } = fields;
+    if (typeof mappings !== "string")
+        throw new Error("no mappings");
+    // TraceMap copies only the standard fields, but given an object that has
+    // its private `_decodedMemo` it would return that object as is. It gets
+    // no mappings: decoding them is this module's job.
+    delete fields._decodedMemo;
+    const map = new TraceMap({ ...fields, mappings: "" });
+    let scopes = null;
+    try {
+        scopes = ScopeIndex.from(fields[SCOPES_KEY], map.names);
+    }
+    catch {
+        scopes = null;
+    }
+    const bytes = decodedMappingsBytes(mappings) +
+        stringsBytes(map.names) +
+        stringsBytes(map.sources) +
+        stringsBytes(map.resolvedSources) +
+        stringsBytes(map.sourcesContent) +
+        (scopes?.bytes ?? 0);
+    if (bytes <= maxDecodedBytes) {
+        const decoded = DecodedMappings.decode(mappings);
+        return {
+            map,
+            scopes,
+            bytes,
+            cacheable: true,
+            lookup: (positions) => positions.map((position) => decoded.find(position.line, position.column)),
+        };
+    }
+    return { map, scopes, bytes, cacheable: false, lookup: (positions) => findInMappings(mappings, positions) };
+}
+// ── Resolution ────────────────────────────────────────────────────────────
+/** A name a minifier evidently produced ("a", "Xe", "Object.t"). */
+function looksMangled(name) {
+    const bare = name.replace(/^(?:async\s+)?(?:new\s+)?/, "");
+    const last = bare.split(/[.\s]/).filter(Boolean).pop() ?? "";
+    return last.length <= 2;
+}
+function contextFor(map, source, line) {
+    let content = null;
+    try {
+        content = sourceContentFor(map, source);
+    }
+    catch {
+        content = null;
+    }
+    if (!content)
+        return {};
+    const lines = content.split(/\r?\n/);
+    const index = line - 1;
+    if (index < 0 || index >= lines.length)
+        return {};
+    const cut = (value) => value.slice(0, MAX_CONTEXT_CHARS);
+    return {
+        contextLine: cut(lines[index]),
+        preContext: lines.slice(Math.max(0, index - CONTEXT_LINES), index).map(cut),
+        postContext: lines.slice(index + 1, index + 1 + CONTEXT_LINES).map(cut),
+    };
+}
+export class Symbolicator {
+    fetchMap;
+    cache;
+    constructor(fetchMap, cache = new SourceMapCache()) {
+        this.fetchMap = fetchMap;
+        this.cache = cache;
+    }
+    /**
+     * Resolve the JS frames of one stack. Returns a new array; frames that
+     * cannot be resolved (no map, no position, non-JS) are returned as given.
+     */
+    async symbolicate(frames, scope) {
+        if (frames.length === 0)
+            return frames;
+        try {
+            return await this.resolveAll(frames, scope);
+        }
+        catch (err) {
+            console.warn("symbolication failed — using parsed frames:", err instanceof Error ? err.message : err);
+            return frames;
+        }
+    }
+    debugIdFor(filename, debugIds) {
+        if (!debugIds)
+            return null;
+        const direct = debugIds[filename];
+        if (typeof direct === "string" && direct)
+            return direct;
+        for (const [url, id] of Object.entries(debugIds)) {
+            if (typeof id === "string" && id && stripQueryAndHash(url) === filename)
+                return id;
+        }
+        return null;
+    }
+    keyFor(frame, scope) {
+        if (!frame.filename || frame.lineno == null || frame.lineno < 1)
+            return null;
+        if (!JS_FILE_RE.test(frame.filename))
+            return null;
+        const path = scriptPath(frame.filename);
+        if (!path)
+            return null;
+        const debugId = this.debugIdFor(frame.filename, scope.debugIds);
+        if (!debugId && !scope.release)
+            return null;
+        return {
+            orgId: scope.orgId,
+            repositoryId: scope.repositoryId,
+            release: scope.release,
+            filename: path,
+            ...(debugId ? { debugId } : {}),
+        };
+    }
+    async resolveAll(frames, scope) {
+        const keys = frames.map((frame) => this.keyFor(frame, scope));
+        if (!keys.some(Boolean))
+            return frames;
+        const maps = await Promise.all(keys.map((key) => (key ? this.cache.get(key, this.fetchMap) : Promise.resolve(null))));
+        const resolutions = resolveFrames(frames, maps);
+        if (!resolutions.some(Boolean))
+            return frames;
+        return frames.map((frame, i) => {
+            const resolution = resolutions[i];
+            if (!resolution)
+                return frame;
+            const { scope } = resolution;
+            if (scope?.kind === "named")
+                return { ...resolution.frame, function: scope.name };
+            if (scope?.kind === "anonymous")
+                return { ...resolution.frame, function: null };
+            // No scopes (an older upload), or a function whose name the CLI
+            // could not map: innermost first, so the caller is the next frame.
+            const callerName = resolutions[i + 1]?.nameHere ?? null;
+            const parsed = frame.function;
+            const name = callerName ?? (parsed && !looksMangled(parsed) ? parsed : null);
+            return { ...resolution.frame, function: name };
+        });
+    }
+}
+/** A frame's 0-based generated position (stack lines and columns are 1-based). */
+function generatedPosition(frame) {
+    return { line: frame.lineno - 1, column: Math.max(0, (frame.colno ?? 1) - 1) };
+}
+/** Resolve every frame that has a map, with one lookup per map: a map too
+ * big to keep is read in a single pass for the whole stack. */
+function resolveFrames(frames, maps) {
+    const resolutions = frames.map(() => null);
+    const byMap = new Map();
+    maps.forEach((loaded, index) => {
+        if (!loaded)
+            return;
+        const indexes = byMap.get(loaded);
+        if (indexes)
+            indexes.push(index);
+        else
+            byMap.set(loaded, [index]);
+    });
+    for (const [loaded, indexes] of byMap) {
+        const positions = indexes.map((index) => generatedPosition(frames[index]));
+        let segments;
+        try {
+            segments = loaded.lookup(positions);
+        }
+        catch {
+            continue;
+        }
+        indexes.forEach((frameIndex, k) => {
+            try {
+                resolutions[frameIndex] = resolveFrame(loaded, frames[frameIndex], positions[k], segments[k] ?? null);
+            }
+            catch {
+                resolutions[frameIndex] = null;
+            }
+        });
+    }
+    return resolutions;
+}
+function resolveFrame(loaded, frame, position, segment) {
+    const { map } = loaded;
+    if (!segment)
+        return null;
+    const source = map.resolvedSources[segment.source];
+    if (!source)
+        return null;
+    const filename = stripQueryAndHash(source);
+    const ignored = (() => {
+        try {
+            return isIgnored(map, source);
+        }
+        catch {
+            return false;
+        }
+    })();
+    const line = segment.line + 1;
+    const resolved = {
+        function: frame.function,
+        module: null,
+        filename,
+        lineno: line,
+        colno: segment.column + 1,
+        inApp: !ignored && isInAppPath(filename),
+        resolved: true,
+        ...contextFor(map, source, line),
+    };
+    const name = segment.name >= 0 ? map.names[segment.name] : undefined;
+    return {
+        frame: resolved,
+        nameHere: typeof name === "string" ? name : null,
+        scope: loaded.scopes ? loaded.scopes.nameAt(position.line, position.column) : null,
+    };
+}

package/dist/types.d.ts

@@ -1,3 +1,4 @@
+import type { Breadcrumb } from "./breadcrumbs.js";
 export type RuntimeSource = "browser" | "server";
 /**
  * Signal severity. Errors AND warnings/info land in the same occurrences
@@ -44,6 +45,60 @@ export interface RuntimeOccurrenceInput {
     expected?: boolean;
     /** `autter.request.id` — joins the occurrence to its request summary. */
     requestId?: string;
+    /**
+     * SDK fingerprint override (browser `fingerprint`, OTLP/log
+     * `autter.fingerprint`): ≤10 scrubbed values, each ≤200 chars. Hashed into
+     * the `manual-v1` candidate by `manualFingerprint` (fingerprint-override.ts).
+     * Omitted when absent.
+     */
+    fingerprintOverride?: string[];
+    /** Steps before the error, oldest first, ≤ 30 (validated by breadcrumbs.ts). */
+    breadcrumbs?: Breadcrumb[];
+    /**
+     * Browser only: script URL → debug id for the scripts on the page, used
+     * to find their source maps (EI-13). Input-only: never stored or sent to
+     * the sink.
+     */
+    debugIds?: Record<string, string>;
+}
+/**
+ * One stack frame, innermost (crash site) first. The sink wire shape
+ * (contract §1) — also stored as JSON in `runtime_error_occurrences.frames`.
+ */
+export interface StackFrame {
+    /** Resolved name when symbolicated, else as parsed. */
+    function: string | null;
+    /** Package / module / class when the language has one. */
+    module: string | null;
+    /** Path as resolved (the source map `sources` entry when resolved), query/hash stripped. */
+    filename: string | null;
+    lineno: number | null;
+    colno: number | null;
+    /** False for dependency, runtime and extension frames (stack-frames.ts §3). */
+    inApp: boolean;
+    /** True when a source map resolved the frame. */
+    resolved: boolean;
+    /** Only when `sourcesContent` supplied it (≤ 300 chars). */
+    contextLine?: string;
+    /** ≤ 3 lines each, ≤ 300 chars per line. */
+    preContext?: string[];
+    postContext?: string[];
+}
+/**
+ * Every grouping scheme an issue identity can come from (contract §1).
+ * The ingester computes `manual-v1`, `code-v1`, `frames-v3` and
+ * `message-v1`; `rule-v1` and `external-v1` exist only on the platform.
+ */
+export type FingerprintSchemeV2 = "manual-v1" | "rule-v1" | "code-v1" | "frames-v3" | "message-v1" | "external-v1";
+export interface FingerprintCandidate {
+    scheme: FingerprintSchemeV2;
+    /** 32 lowercase hex characters. */
+    fingerprint: string;
+}
+/** What went into a fingerprint, for the "why these were grouped" UI. */
+export interface FingerprintRecord {
+    scheme: string;
+    parts: string[];
 }
 /**
  * Derived, aggregation-ready fields — computed once at ingest (from the
@@ -52,9 +107,22 @@ export interface RuntimeOccurrenceInput {
  */
 export interface RuntimeOccurrence extends RuntimeOccurrenceInput {
     occurrenceId: string;
+    /**
+     * The LEGACY identity, byte-for-byte as before 1.6.0 so an older
+     * platform groups exactly as it did: "code-v1" or "message-v1".
+     */
     fingerprint: string;
     /** "code-v1" when grouped by a valid error code, else "message-v1". */
     fingerprintScheme: "message-v1" | "code-v1";
+    /** Parsed (and, for browser JS, symbolicated) frames, innermost first, ≤ 50. */
+    frames?: StackFrame[];
+    /**
+     * Every applicable identity, highest precedence first (manual-v1,
+     * code-v1, frames-v3, message-v1); always ends with the legacy one.
+     */
+    fingerprints?: FingerprintCandidate[];
+    /** What went into the frames-v3 fingerprint, when one was computed. */
+    fingerprintRecord?: FingerprintRecord;
     /** normalizeRoute(route) — "/users/:id/orders/:id"; GROUP BY-safe. */
     routeNormalized: string;
     /** normalizeMessage(message) — ids/numbers/strings templated out. */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.5.0",
+	"version": "1.6.1",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",
@@ -19,6 +19,7 @@
 	},
 	"dependencies": {
 		"@clickhouse/client": "^1.12.0",
+		"@jridgewell/trace-mapping": "^0.3.31",
 		"express": "^4.21.2",
 		"protobufjs": "^7.6.5",
 		"zod": "^3.24.1"