@autter/otlp-ingester 1.4.0 → 1.6.0

package/dist/source-maps-lookup.js

@@ -0,0 +1,64 @@
+import { normalizeDebugId, sourceMapPath, } from "./source-maps.js";
+async function firstMap(client, database, where, params) {
+    const rows = await client.query({
+        query: `SELECT map FROM ${database}.runtime_source_maps
+			WHERE org_id = {org:String} AND repository_id = {repo:String} AND ${where}
+			ORDER BY uploaded_at DESC LIMIT 1`,
+        query_params: params,
+        format: "JSONEachRow",
+    });
+    const [row] = await rows.json();
+    return row?.map ?? null;
+}
+/** The stored map JSON for a minified file: by debug id first when given, else by release + filename (path only, no origin/query). Null when none. */
+export async function lookupSourceMap(client, database, key) {
+    const tenant = { org: key.orgId, repo: key.repositoryId };
+    // A debug id names the exact bundle bytes, so it matches whatever release
+    // the SDK reported (or none). Fall through to release + path when no map
+    // was uploaded with this id (old CLI, plain curl upload).
+    const debugId = normalizeDebugId(key.debugId);
+    if (debugId) {
+        const map = await firstMap(client, database, "debug_id = {debugId:String}", { ...tenant, debugId });
+        if (map !== null)
+            return map;
+    }
+    const filename = sourceMapPath(key.filename);
+    if (!key.release || !filename)
+        return null;
+    return firstMap(client, database, "release = {release:String} AND filename = {filename:String}", { ...tenant, release: key.release, filename });
+}
+/**
+ * Re-uploading a map refreshes its 30-day TTL, so a map only counts as
+ * already uploaded when it was stored this recently; older ones are sent
+ * again and stay alive while the release is still being built.
+ */
+export const SOURCE_MAP_FRESH_DAYS = 7;
+/**
+ * Which check entries the tenant already has: same release, path, debug id
+ * and content hash, uploaded in the last SOURCE_MAP_FRESH_DAYS. Returns the
+ * indexes of `check.entries` that need no upload.
+ */
+export async function presentSourceMaps(client, database, tenant, check) {
+    if (check.entries.length === 0)
+        return [];
+    const rows = await client.query({
+        query: `SELECT DISTINCT filename, debug_id, content_hash FROM ${database}.runtime_source_maps
+			WHERE org_id = {org:String} AND repository_id = {repo:String} AND release = {release:String}
+				AND has({hashes:Array(String)}, content_hash)
+				AND uploaded_at >= now64(3) - INTERVAL ${SOURCE_MAP_FRESH_DAYS} DAY`,
+        query_params: {
+            org: tenant.orgId,
+            repo: tenant.repositoryId,
+            release: check.release,
+            hashes: [...new Set(check.entries.map((entry) => entry.hash))],
+        },
+        format: "JSONEachRow",
+    });
+    const stored = new Set((await rows.json()).map((row) => `${row.filename}\u0000${row.debug_id}\u0000${row.content_hash}`));
+    const present = [];
+    check.entries.forEach((entry, index) => {
+        if (stored.has(`${entry.filename}\u0000${entry.debugId}\u0000${entry.hash}`))
+            present.push(index);
+    });
+    return present;
+}

package/dist/source-maps.d.ts

@@ -1,6 +1,67 @@
+/** Largest stored map (after `sourcesContent` is removed), in characters. */
+export declare const SOURCE_MAP_MAX_CHARS: number;
+/** A debug id: 8-4-4-4-12 lowercase hex. Written by `autter-runtime
+ * sourcemaps inject`, and by Rollup, Turbopack, esbuild and Sentry tooling. */
+export declare const DEBUG_ID_RE: RegExp;
+/** Lowercased debug id, or null when the value isn't one. */
+export declare function normalizeDebugId(value: unknown): string | null;
+/**
+ * The stored form of a bundle's filename: path only, with no origin, query
+ * or hash ("https://cdn.example.com/assets/app.js?v=2" → "/assets/app.js").
+ * Uploads and lookups both go through it so they always agree.
+ */
+export declare function sourceMapPath(filename: string): string | null;
+/** Migration 0016 columns — shared with the baseline DDL below. */
+export declare const SOURCE_MAP_DEBUG_COLUMNS: string[];
+export declare const SOURCE_MAP_DEBUG_ID_INDEX = "INDEX idx_sm_debug_id debug_id TYPE bloom_filter GRANULARITY 1";
 export declare function sourceMapTableDDL(db: string): string;
-export declare function validateSourceMap(body: unknown): {
+export interface ValidSourceMap {
     release: string;
+    /** sourceMapPath() of the uploaded filename. */
     filename: string;
+    /** The map JSON without `sourcesContent`. */
     map: string;
-} | null;
+    /** Lowercase debug id, or "" when the upload has none. */
+    debugId: string;
+    /** sha256 hex of `map`. */
+    contentHash: string;
+}
+/** The content hash of a stored map — `autter-runtime sourcemaps upload`
+ * computes the same value to skip maps the ingester already has. */
+export declare function sourceMapContentHash(map: string): string;
+/**
+ * Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
+ * `map` may be an object or a JSON string. The debug id comes from the body,
+ * else from the map's own `debugId` (or legacy `debug_id`); an invalid one
+ * rejects the upload. Null when the upload is invalid.
+ */
+export declare function validateSourceMap(body: unknown): ValidSourceMap | null;
+/** One entry of a `/v1/sourcemaps/check` request. */
+export interface SourceMapCheckEntry {
+    filename: string;
+    debugId: string;
+    hash: string;
+}
+export interface SourceMapCheck {
+    release: string;
+    entries: SourceMapCheckEntry[];
+}
+/** Most entries one check request may name. */
+export declare const SOURCE_MAP_CHECK_MAX = 500;
+/**
+ * Validate a `/v1/sourcemaps/check` request:
+ * `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
+ * filenames and debug ids are normalised exactly as an upload would store
+ * them. Null when invalid.
+ */
+export declare function validateSourceMapCheck(body: unknown): SourceMapCheck | null;
+/** Most scripts one browser event may name in `debugIds`. */
+export declare const DEBUG_IDS_MAX = 50;
+/**
+ * The `debugIds` of a browser event (runtime-browser 1.6.0+, EI-13): script
+ * URL → debug id. Keys lose their query and hash and are scrubbed, invalid ids
+ * are dropped one by one, at most 50 are kept. `{ debugIds }` for the occurrence, or `{}`.
+ */
+export declare function browserDebugIds(raw: Record<string, unknown> | undefined): {
+    debugIds?: Record<string, string>;
+};

package/dist/source-maps.js

@@ -1,27 +1,74 @@
+import { createHash } from "node:crypto";
+import { scrubText } from "./redact.js";
+/** Largest stored map (after `sourcesContent` is removed), in characters. */
+export const SOURCE_MAP_MAX_CHARS = 5 * 1024 * 1024;
+/** Bundles a map can be uploaded for: `.js`, `.mjs` and `.cjs` (1.6.0+). */
+const BUNDLE_PATH_RE = /\.[cm]?js$/;
+/** A debug id: 8-4-4-4-12 lowercase hex. Written by `autter-runtime
+ * sourcemaps inject`, and by Rollup, Turbopack, esbuild and Sentry tooling. */
+export const DEBUG_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
+/** Lowercased debug id, or null when the value isn't one. */
+export function normalizeDebugId(value) {
+    if (typeof value !== "string")
+        return null;
+    const id = value.trim().toLowerCase();
+    return DEBUG_ID_RE.test(id) ? id : null;
+}
+/**
+ * The stored form of a bundle's filename: path only, with no origin, query
+ * or hash ("https://cdn.example.com/assets/app.js?v=2" → "/assets/app.js").
+ * Uploads and lookups both go through it so they always agree.
+ */
+export function sourceMapPath(filename) {
+    if (filename.length > 1000)
+        return null;
+    try {
+        return new URL(filename, "https://autter.invalid").pathname;
+    }
+    catch {
+        return null;
+    }
+}
+/** Migration 0016 columns — shared with the baseline DDL below. */
+export const SOURCE_MAP_DEBUG_COLUMNS = [
+    // The bundle's debug id ('' for maps uploaded without one).
+    "debug_id String DEFAULT ''",
+    // sha256 of the stored map, so CI can skip maps it already uploaded.
+    "content_hash String DEFAULT ''",
+];
+export const SOURCE_MAP_DEBUG_ID_INDEX = "INDEX idx_sm_debug_id debug_id TYPE bloom_filter GRANULARITY 1";
 export function sourceMapTableDDL(db) {
     return `CREATE TABLE IF NOT EXISTS ${db}.runtime_source_maps (
 		org_id String, repository_id String, release String, filename String,
-		map String CODEC(ZSTD(3)), uploaded_at DateTime64(3, 'UTC') DEFAULT now64(3)
+		map String CODEC(ZSTD(3)), uploaded_at DateTime64(3, 'UTC') DEFAULT now64(3),
+		${SOURCE_MAP_DEBUG_COLUMNS.join(", ")},
+		${SOURCE_MAP_DEBUG_ID_INDEX}
 	) ENGINE = ReplacingMergeTree(uploaded_at)
 	ORDER BY (org_id, repository_id, release, filename)
 	TTL toDateTime(uploaded_at) + INTERVAL 30 DAY`;
 }
+const RELEASE_RE = /^[a-zA-Z0-9._-]{1,200}$/;
+/** The content hash of a stored map — `autter-runtime sourcemaps upload`
+ * computes the same value to skip maps the ingester already has. */
+export function sourceMapContentHash(map) {
+    return createHash("sha256").update(map).digest("hex");
+}
+/**
+ * Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
+ * `map` may be an object or a JSON string. The debug id comes from the body,
+ * else from the map's own `debugId` (or legacy `debug_id`); an invalid one
+ * rejects the upload. Null when the upload is invalid.
+ */
 export function validateSourceMap(body) {
     if (!body || typeof body !== "object")
         return null;
     const input = body;
-    if (typeof input.release !== "string" || !/^[a-zA-Z0-9._-]{1,200}$/.test(input.release))
+    if (typeof input.release !== "string" || !RELEASE_RE.test(input.release))
         return null;
-    if (typeof input.filename !== "string" || input.filename.length > 1000)
+    if (typeof input.filename !== "string")
         return null;
-    let filename;
-    try {
-        filename = new URL(input.filename, "https://autter.invalid").pathname;
-    }
-    catch {
-        return null;
-    }
-    if (!filename.endsWith(".js"))
+    const filename = sourceMapPath(input.filename);
+    if (!filename || !BUNDLE_PATH_RE.test(filename))
         return null;
     let parsed;
     try {
@@ -32,10 +79,71 @@ export function validateSourceMap(body) {
     }
     if (!parsed || parsed.version !== 3 || typeof parsed.mappings !== "string" || !Array.isArray(parsed.sources))
         return null;
+    const rawDebugId = input.debugId ?? parsed.debugId ?? parsed.debug_id;
+    const debugId = rawDebugId === undefined || rawDebugId === null || rawDebugId === "" ? "" : normalizeDebugId(rawDebugId);
+    if (debugId === null)
+        return null;
     // Source contents are unnecessary for position lookup and may contain secrets.
     const { sourcesContent: _omitted, ...safe } = parsed;
     const map = JSON.stringify(safe);
-    if (map.length > 5 * 1024 * 1024)
+    if (map.length > SOURCE_MAP_MAX_CHARS)
+        return null;
+    return { release: input.release, filename, map, debugId, contentHash: sourceMapContentHash(map) };
+}
+/** Most entries one check request may name. */
+export const SOURCE_MAP_CHECK_MAX = 500;
+/**
+ * Validate a `/v1/sourcemaps/check` request:
+ * `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
+ * filenames and debug ids are normalised exactly as an upload would store
+ * them. Null when invalid.
+ */
+export function validateSourceMapCheck(body) {
+    if (!body || typeof body !== "object")
         return null;
-    return { release: input.release, filename, map };
+    const input = body;
+    if (typeof input.release !== "string" || !RELEASE_RE.test(input.release))
+        return null;
+    if (!Array.isArray(input.files) || input.files.length > SOURCE_MAP_CHECK_MAX)
+        return null;
+    const entries = [];
+    for (const raw of input.files) {
+        if (!raw || typeof raw !== "object")
+            return null;
+        const file = raw;
+        if (typeof file.filename !== "string" || typeof file.hash !== "string" || !/^[0-9a-f]{64}$/.test(file.hash))
+            return null;
+        const filename = sourceMapPath(file.filename);
+        if (!filename)
+            return null;
+        const debugId = file.debugId === undefined || file.debugId === null || file.debugId === "" ? "" : normalizeDebugId(file.debugId);
+        if (debugId === null)
+            return null;
+        entries.push({ filename, debugId, hash: file.hash });
+    }
+    return { release: input.release, entries };
+}
+/** Most scripts one browser event may name in `debugIds`. */
+export const DEBUG_IDS_MAX = 50;
+/**
+ * The `debugIds` of a browser event (runtime-browser 1.6.0+, EI-13): script
+ * URL → debug id. Keys lose their query and hash and are scrubbed, invalid ids
+ * are dropped one by one, at most 50 are kept. `{ debugIds }` for the occurrence, or `{}`.
+ */
+export function browserDebugIds(raw) {
+    if (!raw)
+        return {};
+    const debugIds = {};
+    let count = 0;
+    for (const [url, value] of Object.entries(raw)) {
+        const id = normalizeDebugId(value);
+        // Scrubbed like the stack it was taken from, so keys still match its frames.
+        const script = scrubText(url.split(/[?#]/)[0] ?? "");
+        if (!id || !script)
+            continue;
+        debugIds[script] = id;
+        if (++count >= DEBUG_IDS_MAX)
+            break;
+    }
+    return count ? { debugIds } : {};
 }

package/dist/stack-fixtures.d.ts

@@ -12,8 +12,10 @@
  *     parsing, Go/Rust frames were dropped and these collided into one issue).
  *   - `redeploy` — the primary defect after a rebuild that shifted every line
  *     number (and, for Go, pointer offsets/addresses/goroutine ids). It must
- *     fingerprint IDENTICALLY to `primary`, proving grouping is stable across
- *     re-deploys and repeated ingestion.
+ *     fingerprint IDENTICALLY to `primary` under frames-v3, proving grouping
+ *     is stable across re-deploys and repeated ingestion — and under the
+ *     legacy message-v1 too, except where `legacyKeepsLineNumbers` says the
+ *     frozen legacy normaliser hashes line numbers (Python).
  */
 export interface StackFixture {
     primary: string;
@@ -21,5 +23,11 @@ export interface StackFixture {
     redeploy: string;
     /** Golden normalised top frames for `primary` (locks the parser output). */
     primaryFrames: string[];
+    /**
+     * message-v1 keeps this language's line numbers (Python `File "…", line
+     * N` lines are hashed verbatim, and must stay so for existing issues), so
+     * only frames-v3 survives `redeploy`.
+     */
+    legacyKeepsLineNumbers?: boolean;
 }
 export declare const STACK_FIXTURES: Record<string, StackFixture>;

package/dist/stack-fixtures.js

@@ -12,8 +12,10 @@
  *     parsing, Go/Rust frames were dropped and these collided into one issue).
  *   - `redeploy` — the primary defect after a rebuild that shifted every line
  *     number (and, for Go, pointer offsets/addresses/goroutine ids). It must
- *     fingerprint IDENTICALLY to `primary`, proving grouping is stable across
- *     re-deploys and repeated ingestion.
+ *     fingerprint IDENTICALLY to `primary` under frames-v3, proving grouping
+ *     is stable across re-deploys and repeated ingestion — and under the
+ *     legacy message-v1 too, except where `legacyKeepsLineNumbers` says the
+ *     frozen legacy normaliser hashes line numbers (Python).
  */
 // ── Go ──────────────────────────────────────────────────────────────────────
 const GO_PRIMARY = `panic: runtime error: invalid memory address or nil pointer dereference
@@ -154,6 +156,13 @@ const PYTHON_PRIMARY = `Traceback (most recent call last):
   File "/app/orders/service.py", line 88, in process
     raise ValueError(f"order {order_id} is invalid")
 ValueError: order 4821 is invalid`;
+// Same defect after a deploy that shifted both functions down the file.
+const PYTHON_REDEPLOY = `Traceback (most recent call last):
+  File "/app/web/handler.py", line 47, in handle
+    return self.service.process(order)
+  File "/app/orders/service.py", line 95, in process
+    raise ValueError(f"order {order_id} is invalid")
+ValueError: order 9113 is invalid`;
 const PYTHON_SIBLING = `Traceback (most recent call last):
   File "/app/web/handler.py", line 42, in handle
     return self.service.process(order)
@@ -222,10 +231,11 @@ export const STACK_FIXTURES = {
     python: {
         primary: PYTHON_PRIMARY,
         sibling: PYTHON_SIBLING,
-        redeploy: PYTHON_PRIMARY,
+        redeploy: PYTHON_REDEPLOY,
         primaryFrames: [
             'File "/app/web/handler.py", line 42, in handle',
             'File "/app/orders/service.py", line 88, in process',
         ],
+        legacyKeepsLineNumbers: true,
     },
 };

package/dist/stack-frames.d.ts

@@ -0,0 +1,48 @@
+import type { StackFrame } from "./types.js";
+/**
+ * Raw stack text → structured frames, INNERMOST (crash site) FIRST.
+ *
+ * The legacy `message-v1` fingerprint hashes stack lines as text, which
+ * keeps Python line numbers, takes Python's outermost frames, and counts
+ * dependency frames. `frames-v3` (fingerprint-v3.ts) hashes structured
+ * frames instead, so every supported language is parsed into the sink's
+ * StackFrame shape here, in one consistent orientation:
+ *
+ * - V8 (Node, Chrome, Edge): `at fn (file:line:col)` / `at file:line:col`
+ * - Firefox / Safari: `fn@url:line:col` / `@url:line:col`
+ * - Python: `File "…", line N, in fn` — tracebacks list the OUTERMOST frame
+ *   first, so the frames are reversed; only the last (reported) traceback
+ *   of a chained exception is used
+ * - Go: function line + `\t<file>.go:<line> +0x…`, first goroutine only
+ * - JVM: `at pkg.Class.method(File.java:N)`, first exception block only
+ *   (before `Caused by:`)
+ * - .NET: `at Ns.Type.Method(args) in <file>:line N`; for wrapped
+ *   exceptions only the frames after the last "End of inner exception"
+ * - Rust: `N: path::to::fn` + `at <file>:<line>:<col>`
+ * - Ruby / PHP best effort, so those stacks do not collapse to the message
+ *
+ * `inApp` follows the contract §3 rule (isInAppFrame). `resolved` is always
+ * false here; symbolicate.ts sets it for frames a source map resolved.
+ * Shared parity vectors: stack-parse-vectors.json.
+ */
+/** Frames kept per stack (contract §1: at most 50). */
+export declare const MAX_FRAMES = 50;
+export type FrameLanguage = "javascript" | "python" | "go" | "jvm" | "dotnet" | "rust" | "ruby" | "php";
+/**
+ * Path half of §3: dependency directories, Node internals and browser
+ * extension scripts are not in-app. Windows separators are normalised so
+ * `C:\app\node_modules\x.js` counts too. A frame with no path is in-app.
+ */
+export declare function isInAppPath(path: string | null): boolean;
+/**
+ * Contract §3. Source-map ignore lists are applied by symbolicate.ts; the
+ * language rules apply only to frames of that language, so a JS object
+ * that happens to be called `runtime` or `System` stays in-app.
+ */
+export declare function isInAppFrame(frame: Pick<StackFrame, "function" | "module" | "filename">, language: FrameLanguage | null): boolean;
+/** Strip a query string and/or hash from a path or URL. */
+export declare function stripQueryAndHash(path: string): string;
+/** The language of a raw stack, or null when it has no recognisable frames. */
+export declare function detectFrameLanguage(lines: string[]): FrameLanguage | null;
+/** Parse a raw stack into frames, innermost first (≤ MAX_FRAMES). */
+export declare function parseStackFrames(stack: string | null | undefined): StackFrame[];