@autter/otlp-ingester 1.10.0 → 1.11.0

package/dist/symbolicate.js

@@ -1,10 +1,12 @@
-import { isIgnored, sourceContentFor, TraceMap, } from "@jridgewell/trace-mapping";
+import { isIgnored, TraceMap } from "@jridgewell/trace-mapping";
 import { DecodedMappings, decodedMappingsBytes, findInMappings, } from "./source-map-mappings.js";
+import { scrubText } from "./redact.js";
 import { SCOPES_KEY, ScopeIndex } from "./source-map-scopes.js";
 import { isInAppPath, stripQueryAndHash } from "./stack-frames.js";
 const JS_FILE_RE = /\.[cm]?js$/i;
 const MAX_CONTEXT_CHARS = 300;
-const CONTEXT_LINES = 3;
+/** Lines kept before and after an in-app frame's own line (RF-05). */
+export const CONTEXT_LINES = 5;
 /**
  * In-app browser frames a source map could resolve, counted after
  * symbolication (EI-27): `resolved` ones, and `unresolved` .js/.mjs/.cjs
@@ -36,6 +38,25 @@ export function scriptPath(filename) {
         return null;
     }
 }
+/**
+ * The path form of a server frame's file, comparable with stored upload
+ * paths: `file:///app/dist/index.mjs` → `/app/dist/index.mjs`,
+ * `C:\app\dist\index.js` → `/app/dist/index.js`, `index.js` (a Worker
+ * script) → `/index.js`. Null when it is not a path.
+ */
+export function serverScriptPath(filename) {
+    let path = filename;
+    if (/^file:\/\//i.test(path)) {
+        try {
+            path = new URL(path).pathname;
+        }
+        catch {
+            return null;
+        }
+    }
+    path = path.replace(/\\/g, "/").replace(/^\/?[A-Za-z]:(?=\/)/, "");
+    return scriptPath(path);
+}
 /**
  * The largest share of the budget one map may take. A bigger one is used
  * for its lookup and dropped: keeping it would evict most other maps, and
@@ -46,7 +67,13 @@ export class SourceMapCache {
     entries = new Map();
     inFlight = new Map();
     weight = 0;
+    /** Source contents of cached maps, keyed by parse id and source index. */
+    contents = new Map();
+    contentWeight = 0;
+    /** Distinguishes every parse, so a re-parsed (maybe re-uploaded) map never reads an older parse's contents. */
+    parses = 0;
     maxBytes;
+    maxContentBytes;
     maxEntries;
     hitTtlMs;
     missTtlMs;
@@ -58,6 +85,7 @@ export class SourceMapCache {
     bypassed = 0;
     constructor(options = {}) {
         this.maxBytes = options.maxBytes ?? 64 * 1024 * 1024;
+        this.maxContentBytes = options.maxContentBytes ?? this.maxBytes;
         this.maxEntries = options.maxEntries ?? 500;
         this.hitTtlMs = options.hitTtlMs ?? 60 * 60_000;
         this.missTtlMs = options.missTtlMs ?? 5 * 60_000;
@@ -66,9 +94,11 @@ export class SourceMapCache {
     }
     static keyOf(key) {
         const scope = `${key.orgId}\u0000${key.repositoryId}\u0000`;
+        // A suffix lookup (server frame) can find another map than an exact one
+        // for the same path, so the two never share an entry.
         return key.debugId
             ? `${scope}d\u0000${key.debugId}`
-            : `${scope}r\u0000${key.release ?? ""}\u0000${key.filename}`;
+            : `${scope}${key.suffix ? "s" : "r"}\u0000${key.release ?? ""}\u0000${key.filename}`;
     }
     get size() {
         return this.entries.size;
@@ -77,6 +107,10 @@ export class SourceMapCache {
     get bytes() {
         return this.weight;
     }
+    /** Estimated bytes of the cached source contents (what `maxContentBytes` bounds). */
+    get contentBytes() {
+        return this.contentWeight;
+    }
     /** The parsed map for `key` (null when none), fetching at most once at a time. */
     async get(key, fetchMap) {
         const id = SourceMapCache.keyOf(key);
@@ -112,28 +146,74 @@ export class SourceMapCache {
             this.store(id, null, 0, this.missTtlMs);
             return null;
         }
-        let loaded;
+        let parsed;
         try {
-            loaded = parseSourceMap(json, this.maxBytes * MAX_SHARE_OF_BUDGET);
+            parsed = parseSourceMapWithContents(json, this.maxBytes * MAX_SHARE_OF_BUDGET);
         }
         catch {
             // A malformed upload is a miss until it is replaced.
             this.store(id, null, 0, this.missTtlMs);
             return null;
         }
-        if (loaded.cacheable)
-            this.store(id, loaded, loaded.bytes, this.hitTtlMs);
-        else
+        const { loaded, contents } = parsed;
+        if (!loaded.cacheable) {
+            // Used for this lookup only: its contents go with it.
             this.bypassed += 1;
+            return loaded;
+        }
+        // The map's entry never holds the contents (it weighs what it did
+        // before RF-05): they go to their own store, one file per entry.
+        const parse = ++this.parses;
+        const contentKeys = [];
+        contents?.forEach((content, index) => {
+            if (typeof content !== "string" || !content)
+                return;
+            const key = `${parse}\u0000${index}`;
+            contentKeys.push(key);
+            this.storeContent(key, content);
+        });
+        loaded.contentOf = (index) => this.contentOf(`${parse}\u0000${index}`);
+        this.store(id, loaded, loaded.bytes, this.hitTtlMs, contentKeys);
         return loaded;
     }
-    store(id, map, weight, ttlMs) {
+    contentOf(key) {
+        const entry = this.contents.get(key);
+        if (!entry)
+            return null;
+        if (entry.expiresAt <= this.now()) {
+            this.removeContent(key);
+            return null;
+        }
+        this.contents.delete(key);
+        this.contents.set(key, entry);
+        return entry.content;
+    }
+    storeContent(key, content) {
+        const weight = 2 * content.length + 64;
+        // Bounded like the maps: one file may take at most half the budget.
+        if (weight > this.maxContentBytes * MAX_SHARE_OF_BUDGET)
+            return;
+        this.removeContent(key);
+        this.contents.set(key, { content, weight, expiresAt: this.now() + this.hitTtlMs });
+        this.contentWeight += weight;
+        while (this.contentWeight > this.maxContentBytes && this.contents.size > 1) {
+            this.removeContent(this.contents.keys().next().value);
+        }
+    }
+    removeContent(key) {
+        const entry = this.contents.get(key);
+        if (!entry)
+            return;
+        this.contents.delete(key);
+        this.contentWeight -= entry.weight;
+    }
+    store(id, map, weight, ttlMs, contentKeys) {
         // A map bigger than the whole budget serves this lookup but is not kept:
         // caching it would evict everything else.
         if (weight > this.maxBytes)
             return;
         this.remove(id);
-        this.entries.set(id, { map, weight, expiresAt: this.now() + ttlMs });
+        this.entries.set(id, { map, weight, expiresAt: this.now() + ttlMs, ...(contentKeys?.length ? { contentKeys } : {}) });
         this.weight += weight;
         while ((this.weight > this.maxBytes || this.entries.size > this.maxEntries) &&
             this.entries.size > 1) {
@@ -147,6 +227,8 @@ export class SourceMapCache {
             return;
         this.entries.delete(id);
         this.weight -= entry.weight;
+        for (const key of entry.contentKeys ?? [])
+            this.removeContent(key);
     }
 }
 /** Rough heap of a list of strings (V8 strings plus array slots). */
@@ -165,15 +247,24 @@ function stringsBytes(list) {
  * once into a compact table; otherwise they are left encoded and each
  * lookup reads them in one pass (`cacheable: false`). Throws when the map
  * itself is unusable.
+ *
+ * `sourcesContent` (RF-05) is taken out before anything else: the TraceMap
+ * never holds it and `bytes` never counts it, so whether a map is decoded
+ * and cached is decided exactly as for a map uploaded without contents.
+ * They are returned beside the map; SourceMapCache keeps them in their own
+ * store.
  */
-export function parseSourceMap(json, maxDecodedBytes = Number.POSITIVE_INFINITY) {
+export function parseSourceMapWithContents(json, maxDecodedBytes = Number.POSITIVE_INFINITY) {
     const raw = JSON.parse(json);
     if (!raw || typeof raw !== "object" || Array.isArray(raw))
         throw new Error("not a source map");
-    const fields = raw;
+    const { sourcesContent, ...fields } = raw;
     const { mappings } = fields;
     if (typeof mappings !== "string")
         throw new Error("no mappings");
+    const contents = Array.isArray(sourcesContent)
+        ? sourcesContent.map((content) => (typeof content === "string" ? content : null))
+        : null;
     // TraceMap copies only the standard fields, but given an object that has
     // its private `_decodedMemo` it would return that object as is. It gets
     // no mappings: decoding them is this module's job.
@@ -190,19 +281,41 @@ export function parseSourceMap(json, maxDecodedBytes = Number.POSITIVE_INFINITY)
         stringsBytes(map.names) +
         stringsBytes(map.sources) +
         stringsBytes(map.resolvedSources) +
-        stringsBytes(map.sourcesContent) +
         (scopes?.bytes ?? 0);
+    // Standalone use (and a map too big to cache): the contents travel with
+    // this parse; SourceMapCache replaces this for the maps it keeps. Built
+    // outside this scope so `lookup`'s closure never holds the contents.
+    const contentOf = contentsReader(contents);
     if (bytes <= maxDecodedBytes) {
         const decoded = DecodedMappings.decode(mappings);
         return {
-            map,
-            scopes,
-            bytes,
-            cacheable: true,
-            lookup: (positions) => positions.map((position) => decoded.find(position.line, position.column)),
+            loaded: {
+                map,
+                scopes,
+                bytes,
+                cacheable: true,
+                lookup: (positions) => positions.map((position) => decoded.find(position.line, position.column)),
+                contentOf,
+            },
+            contents,
         };
     }
-    return { map, scopes, bytes, cacheable: false, lookup: (positions) => findInMappings(mappings, positions) };
+    return {
+        loaded: { map, scopes, bytes, cacheable: false, lookup: (positions) => findInMappings(mappings, positions), contentOf },
+        contents,
+    };
+}
+/**
+ * `contentOf` over a parse's own contents. A separate function on purpose:
+ * closures made in one scope share it, so a reader made next to `lookup`
+ * would keep every source content alive for as long as the cached map.
+ */
+function contentsReader(contents) {
+    return (index) => contents?.[index] ?? null;
+}
+/** parseSourceMapWithContents, the map alone (its contents through `contentOf`). */
+export function parseSourceMap(json, maxDecodedBytes = Number.POSITIVE_INFINITY) {
+    return parseSourceMapWithContents(json, maxDecodedBytes).loaded;
 }
 // ── Resolution ────────────────────────────────────────────────────────────
 /** A name a minifier evidently produced ("a", "Xe", "Object.t"). */
@@ -211,25 +324,56 @@ function looksMangled(name) {
     const last = bare.split(/[.\s]/).filter(Boolean).pop() ?? "";
     return last.length <= 2;
 }
-function contextFor(map, source, line) {
+/**
+ * Lines `index - radius` … `index + radius` (0-based) of `content`, found
+ * without splitting the whole file (a busy frame in a big file would split
+ * it on every event); `at` is where line `index` sits in `lines`. Null when
+ * the file is shorter than `index + 1` lines.
+ */
+function linesAround(content, index, radius) {
+    if (index < 0)
+        return null;
+    const first = Math.max(0, index - radius);
+    let start = 0;
+    for (let line = 0; line < first; line++) {
+        const newline = content.indexOf("\n", start);
+        if (newline < 0)
+            return null;
+        start = newline + 1;
+    }
+    const lines = [];
+    for (let line = first; line <= index + radius; line++) {
+        const newline = content.indexOf("\n", start);
+        const end = newline < 0 ? content.length : newline;
+        lines.push(content.slice(start, end).replace(/\r$/, ""));
+        if (newline < 0)
+            break;
+        start = newline + 1;
+    }
+    const at = index - first;
+    return at < lines.length ? { lines, at } : null;
+}
+function contextFor(loaded, sourceIndex, line) {
     let content = null;
     try {
-        content = sourceContentFor(map, source);
+        content = loaded.contentOf(sourceIndex);
     }
     catch {
         content = null;
     }
     if (!content)
         return {};
-    const lines = content.split(/\r?\n/);
-    const index = line - 1;
-    if (index < 0 || index >= lines.length)
+    const window = linesAround(content, line - 1, CONTEXT_LINES);
+    if (!window)
         return {};
-    const cut = (value) => value.slice(0, MAX_CONTEXT_CHARS);
+    // Source code can hold secrets: scrubbed like the stack. Cut first so the
+    // scrub work stays bounded, with slack so a secret straddling the cut is
+    // still seen whole.
+    const clean = (value) => scrubText(value.slice(0, MAX_CONTEXT_CHARS + 512)).slice(0, MAX_CONTEXT_CHARS);
     return {
-        contextLine: cut(lines[index]),
-        preContext: lines.slice(Math.max(0, index - CONTEXT_LINES), index).map(cut),
-        postContext: lines.slice(index + 1, index + 1 + CONTEXT_LINES).map(cut),
+        contextLine: clean(window.lines[window.at]),
+        preContext: window.lines.slice(0, window.at).map(clean),
+        postContext: window.lines.slice(window.at + 1).map(clean),
     };
 }
 export class Symbolicator {
@@ -271,7 +415,12 @@ export class Symbolicator {
             return null;
         if (!JS_FILE_RE.test(frame.filename))
             return null;
-        const path = scriptPath(frame.filename);
+        // A server's dependency and runtime frames (node_modules/, node:) run
+        // from files no upload covers; looking them up by suffix could even
+        // pick an app bundle that happens to share the file name.
+        if (scope.server && !frame.inApp)
+            return null;
+        const path = scope.server ? serverScriptPath(frame.filename) : scriptPath(frame.filename);
         if (!path)
             return null;
         const debugId = this.debugIdFor(frame.filename, scope.debugIds);
@@ -283,6 +432,7 @@ export class Symbolicator {
             release: scope.release,
             filename: path,
             ...(debugId ? { debugId } : {}),
+            ...(scope.server ? { suffix: true } : {}),
         };
     }
     async resolveAll(frames, scope) {
@@ -366,15 +516,17 @@ function resolveFrame(loaded, frame, position, segment) {
         }
     })();
     const line = segment.line + 1;
+    const inApp = !ignored && isInAppPath(filename);
     const resolved = {
         function: frame.function,
         module: null,
         filename,
         lineno: line,
         colno: segment.column + 1,
-        inApp: !ignored && isInAppPath(filename),
+        inApp,
         resolved: true,
-        ...contextFor(map, source, line),
+        // Context lines are the evidence for the app's own code (RF-05).
+        ...(inApp ? contextFor(loaded, segment.source, line) : {}),
     };
     const name = segment.name >= 0 ? map.names[segment.name] : undefined;
     return {

package/dist/trace-dedupe.d.ts

@@ -43,7 +43,7 @@ import type { RuntimeOccurrenceInput } from "./types.js";
  * message, severity, time and batch position (so its deterministic id), and
  * takes from the others what it lacks: attributes (its own win), stack,
  * route, method, status, request id, declared error fields, fingerprint
- * override and breadcrumbs. `autter.merged_recordings` counts the merged
+ * override, breadcrumbs and debug ids. `autter.merged_recordings` counts the merged
  * recordings.
  *
  * Rule 2, redundant status-only occurrences. A status-only `HttpServerError`

package/dist/trace-dedupe.js

@@ -46,7 +46,7 @@ import { parseStackFrames } from "./stack-frames.js";
  * message, severity, time and batch position (so its deterministic id), and
  * takes from the others what it lacks: attributes (its own win), stack,
  * route, method, status, request id, declared error fields, fingerprint
- * override and breadcrumbs. `autter.merged_recordings` counts the merged
+ * override, breadcrumbs and debug ids. `autter.merged_recordings` counts the merged
  * recordings.
  *
  * Rule 2, redundant status-only occurrences. A status-only `HttpServerError`
@@ -245,6 +245,8 @@ function mergeInto(kept, others) {
         }
         if (!merged.breadcrumbs?.length && other.breadcrumbs?.length)
             merged.breadcrumbs = other.breadcrumbs;
+        if (!merged.debugIds && other.debugIds)
+            merged.debugIds = other.debugIds;
     }
     attributes["autter.merged_recordings"] = others.length + 1;
     // Same bounds as every other occurrence (128 keys, kept's own first).

package/dist/types.d.ts

@@ -1,4 +1,5 @@
 import type { Breadcrumb } from "./breadcrumbs.js";
+import type { LatencySamples } from "./latency.js";
 import type { SuppressionRule } from "./suppression.js";
 export type RuntimeSource = "browser" | "server";
 /**
@@ -56,9 +57,10 @@ export interface RuntimeOccurrenceInput {
     /** Steps before the error, oldest first, ≤ 30 (validated by breadcrumbs.ts). */
     breadcrumbs?: Breadcrumb[];
     /**
-     * Browser only: script URL → debug id for the scripts on the page, used
-     * to find their source maps (EI-13). Input-only: never stored or sent to
-     * the sink.
+     * Script URL (browser) or script path (server, the SDK's
+     * `autter.debug_ids`) → debug id for the bundles in the stack, used to
+     * find their source maps (EI-13, RF-05). Input-only: never stored or
+     * sent to the sink.
      */
     debugIds?: Record<string, string>;
 }
@@ -79,18 +81,23 @@ export interface StackFrame {
     inApp: boolean;
     /** True when a source map resolved the frame. */
     resolved: boolean;
-    /** Only when `sourcesContent` supplied it (≤ 300 chars). */
+    /**
+     * The frame's own source line, on in-app resolved frames whose map
+     * carried the source's `sourcesContent` (≤ 300 chars, scrubbed).
+     */
     contextLine?: string;
-    /** ≤ 3 lines each, ≤ 300 chars per line. */
+    /** Up to 5 lines before the frame's line, in file order (RF-05; 3 before), ≤ 300 chars each. */
     preContext?: string[];
+    /** Up to 5 lines after it, in file order, ≤ 300 chars each. */
     postContext?: string[];
 }
 /**
  * Every grouping scheme an issue identity can come from (contract §1).
- * The ingester computes `manual-v1`, `code-v1`, `frames-v3` and
- * `message-v1`; `rule-v1` and `external-v1` exist only on the platform.
+ * The ingester computes `manual-v1`, `code-v1`, `frames-v3`, `message-v1`
+ * and `cause-v1` (RF-06); `rule-v1` and `external-v1` exist only on the
+ * platform.
  */
-export type FingerprintSchemeV2 = "manual-v1" | "rule-v1" | "code-v1" | "frames-v3" | "message-v1" | "external-v1";
+export type FingerprintSchemeV2 = "manual-v1" | "rule-v1" | "code-v1" | "frames-v3" | "message-v1" | "cause-v1" | "external-v1";
 export interface FingerprintCandidate {
     scheme: FingerprintSchemeV2;
     /** 32 lowercase hex characters. */
@@ -115,11 +122,12 @@ export interface RuntimeOccurrence extends RuntimeOccurrenceInput {
     fingerprint: string;
     /** "code-v1" when grouped by a valid error code, else "message-v1". */
     fingerprintScheme: "message-v1" | "code-v1";
-    /** Parsed (and, for browser JS, symbolicated) frames, innermost first, ≤ 50. */
+    /** Parsed (and, for bundled JS, symbolicated) frames, innermost first, ≤ 50. */
     frames?: StackFrame[];
     /**
-     * Every applicable identity, highest precedence first (manual-v1,
-     * code-v1, frames-v3, message-v1); always ends with the legacy one.
+     * Every applicable identity, in precedence order (manual-v1, code-v1,
+     * frames-v3, message-v1, cause-v1); always includes the legacy one,
+     * which is last unless a cause-v1 candidate follows it.
      */
     fingerprints?: FingerprintCandidate[];
     /** What went into the frames-v3 fingerprint, when one was computed. */
@@ -200,6 +208,13 @@ export interface RuntimeMetricPoint {
     errorCount: number;
     durationSumMs: number;
     sessionCount: number;
+    /**
+     * Latency samples behind `durationSumMs` (IP-02, latency.ts), written to
+     * `runtime_route_latency_1m`. Absent when the feed has no durations
+     * (errors outside requests, browser usage counters). Not forwarded to
+     * the sink.
+     */
+    latency?: LatencySamples;
 }
 /**
  * The tenant a validated ingest key resolves to.

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.10.0",
+	"version": "1.11.0",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",