@autter/otlp-ingester 1.6.1 → 1.7.0

package/dist/server.js

@@ -19,6 +19,14 @@ import { configureRedaction, scrubOccurrence } from "./redact.js";
 import { buildCompatReport, evaluateCompat, INGESTER_VERSION_HEADER, } from "./compat.js";
 import { MIGRATIONS } from "./migrations.js";
 import { BROWSER_SDK_NAME, ingesterVersion, SdkVersionTracker, sdkSightingsFromOtlp, } from "./sdk-versions.js";
+/** The only routes an upload key (`autter_rtu_…`, scope "upload") may call. */
+export const UPLOAD_KEY_ROUTES = ["/v1/sourcemaps", "/v1/sourcemaps/check"];
+/** The 403 an upload key gets on every other authenticated route. */
+export const UPLOAD_KEY_NOT_ALLOWED = {
+    error: "upload_key_not_allowed",
+    detail: "autter_rtu_ keys can only upload source maps (POST /v1/sourcemaps and /v1/sourcemaps/check); " +
+        "telemetry needs a server key (autter_rt_…), or a client key (autter_rtc_…) for /v1/browser",
+};
 /** Re-report the ingester version at most this often (keeps the row inside
  * its TTL for long-running ingesters). */
 const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000;
@@ -94,7 +102,7 @@ export function createIngesterApp(config) {
     const sourceMapBody = express.json({ limit: SOURCE_MAP_MAX_BODY_BYTES });
     app.post("/v1/sourcemaps", async (req, res, next) => {
         // On failure the body is never read (Node discards it).
-        const ctx = await authenticate(req, res, "otlp");
+        const ctx = await authenticate(req, res, "sourcemaps");
         if (!ctx)
             return;
         sourceMapBody(req, res, async (err) => {
@@ -197,7 +205,12 @@ export function createIngesterApp(config) {
         res.setHeader("cache-control", "no-store");
         res.status(200).json(body);
     });
-    /** Auth + scope + rate limit; returns null (response sent) on failure. */
+    /**
+     * Auth + scope + rate limit; returns null (response sent) on failure.
+     * Server keys pass everywhere; client keys only on "browser"; upload
+     * keys only on "sourcemaps". A new route that passes "otlp" is closed to
+     * client and upload keys without further changes.
+     */
     async function authenticate(req, res, surface) {
         // A storage-less ingester must refuse, not accept-and-drop: exporters
         // retry on 503, so telemetry survives a misconfigured deploy.
@@ -215,12 +228,22 @@ export function createIngesterApp(config) {
             res.status(401).json({ error: "invalid ingest key" });
             return null;
         }
-        if (ctx.scope === "client") {
+        if (ctx.scope === "upload") {
+            // CI keys: source map upload and check only. Server rate window
+            // (each key has its own).
+            if (surface !== "sourcemaps") {
+                res.status(403).json(UPLOAD_KEY_NOT_ALLOWED);
+                return null;
+            }
+        }
+        else if (ctx.scope === "client") {
             // Publishable keys: browser surface only, origin allow-list, and
             // the tighter rate window.
             if (surface !== "browser") {
                 res.status(403).json({
-                    error: "client keys cannot send OTLP — use a server key",
+                    error: surface === "sourcemaps"
+                        ? "client keys cannot upload source maps — use an upload key (autter_rtu_…) or a server key"
+                        : "client keys cannot send OTLP — use a server key",
                 });
                 return null;
             }
@@ -236,6 +259,12 @@ export function createIngesterApp(config) {
             }
             return ctx;
         }
+        else if (ctx.scope !== "server") {
+            // KeyResolver never yields another scope; refuse rather than
+            // treat an unknown one as a server key.
+            res.status(403).json({ error: "key scope not allowed" });
+            return null;
+        }
         if (!serverRateLimiter.allow(key)) {
             res.status(429).json({ error: "rate limit exceeded" });
             return null;
@@ -275,8 +304,13 @@ export function createIngesterApp(config) {
      * `fingerprint` / `fingerprintScheme` stay the legacy identity
      * (message-v1 or code-v1) byte-for-byte; `fingerprints` carries every
      * candidate (manual-v1, code-v1, frames-v3, message-v1) for the platform
-     * to resolve the issue from. */
-    async function fingerprintAll(ctx, inputs) {
+     * to resolve the issue from.
+     *
+     * `batchIndexes` (traces): each occurrence's position among the batch's
+     * signals before the per-trace dedupe, so a merged occurrence keeps the
+     * id of the recording it kept and a drop never shifts another's id.
+     * Defaults to the position in `inputs`. */
+    async function fingerprintAll(ctx, inputs, batchIndexes) {
         return Promise.all(inputs.map(async (raw, index) => {
             // Second line of defence, before fingerprinting, storage, and the
             // sink (which feeds LLM fix generation): scrub secrets from the
@@ -291,7 +325,7 @@ export function createIngesterApp(config) {
             const { debugIds: _debugIds, ...stored } = input;
             return {
                 ...stored,
-                occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index),
+                occurrenceId: occurrenceIdFor(ctx, input, fingerprint, batchIndexes?.[index] ?? index),
                 fingerprint,
                 fingerprintScheme: scheme,
                 ...deriveFields(input),
@@ -377,7 +411,7 @@ export function createIngesterApp(config) {
     /** Which of these maps does this tenant already have? Lets
      * `autter-runtime sourcemaps upload` skip unchanged maps (1.6.0+). */
     app.post("/v1/sourcemaps/check", async (req, res) => {
-        const ctx = await authenticate(req, res, "otlp");
+        const ctx = await authenticate(req, res, "sourcemaps");
         if (!ctx)
             return;
         const check = validateSourceMapCheck(req.body);
@@ -432,8 +466,8 @@ export function createIngesterApp(config) {
         else {
             request = req.body;
         }
-        const { occurrences, spans, metricPoints, llmCalls } = normalizeTraces(request);
-        const fingerprinted = await fingerprintAll(ctx, occurrences);
+        const { occurrences, occurrenceBatchIndexes, spans, metricPoints, llmCalls } = normalizeTraces(request);
+        const fingerprinted = await fingerprintAll(ctx, occurrences, occurrenceBatchIndexes);
         // ClickHouse has no cross-table transaction, so these four inserts can
         // partially commit. Recovery boundary: any failure → 503 → the exporter
         // retries the whole batch. Deterministic occurrence ids make the retry

package/dist/source-maps-lookup.d.ts

@@ -21,9 +21,12 @@ export declare function lookupSourceMap(client: ClickHouseClient, database: stri
  */
 export declare const SOURCE_MAP_FRESH_DAYS = 7;
 /**
- * Which check entries the tenant already has: same release, path, debug id
- * and content hash, uploaded in the last SOURCE_MAP_FRESH_DAYS. Returns the
- * indexes of `check.entries` that need no upload.
+ * Which check entries the tenant already has. An entry with a hash: same
+ * release, path, debug id and content hash, uploaded in the last
+ * SOURCE_MAP_FRESH_DAYS (the upload skips those). An entry with only a
+ * debug id: any stored map with that id, whatever its release and path —
+ * exactly what lookupSourceMap would find for a stack from that bundle.
+ * Returns the indexes of `check.entries` that are present.
  */
 export declare function presentSourceMaps(client: ClickHouseClient, database: string, tenant: {
     orgId: string;

package/dist/source-maps-lookup.js

@@ -34,13 +34,41 @@ export async function lookupSourceMap(client, database, key) {
  */
 export const SOURCE_MAP_FRESH_DAYS = 7;
 /**
- * Which check entries the tenant already has: same release, path, debug id
- * and content hash, uploaded in the last SOURCE_MAP_FRESH_DAYS. Returns the
- * indexes of `check.entries` that need no upload.
+ * Which check entries the tenant already has. An entry with a hash: same
+ * release, path, debug id and content hash, uploaded in the last
+ * SOURCE_MAP_FRESH_DAYS (the upload skips those). An entry with only a
+ * debug id: any stored map with that id, whatever its release and path —
+ * exactly what lookupSourceMap would find for a stack from that bundle.
+ * Returns the indexes of `check.entries` that are present.
  */
 export async function presentSourceMaps(client, database, tenant, check) {
-    if (check.entries.length === 0)
-        return [];
+    const hashed = check.entries.some((entry) => entry.hash);
+    const byId = check.entries.filter((entry) => !entry.hash).map((entry) => entry.debugId);
+    const [withHash, ids] = await Promise.all([
+        hashed ? presentByHash(client, database, tenant, check) : new Set(),
+        byId.length ? presentDebugIds(client, database, tenant, byId) : new Set(),
+    ]);
+    const present = [];
+    check.entries.forEach((entry, index) => {
+        const found = entry.hash
+            ? withHash.has(`${entry.filename}\u0000${entry.debugId}\u0000${entry.hash}`)
+            : ids.has(entry.debugId);
+        if (found)
+            present.push(index);
+    });
+    return present;
+}
+async function presentDebugIds(client, database, tenant, debugIds) {
+    const rows = await client.query({
+        query: `SELECT DISTINCT debug_id FROM ${database}.runtime_source_maps
+			WHERE org_id = {org:String} AND repository_id = {repo:String}
+				AND has({debugIds:Array(String)}, debug_id)`,
+        query_params: { org: tenant.orgId, repo: tenant.repositoryId, debugIds: [...new Set(debugIds)] },
+        format: "JSONEachRow",
+    });
+    return new Set((await rows.json()).map((row) => row.debug_id));
+}
+async function presentByHash(client, database, tenant, check) {
     const rows = await client.query({
         query: `SELECT DISTINCT filename, debug_id, content_hash FROM ${database}.runtime_source_maps
 			WHERE org_id = {org:String} AND repository_id = {repo:String} AND release = {release:String}
@@ -50,15 +78,9 @@ export async function presentSourceMaps(client, database, tenant, check) {
             org: tenant.orgId,
             repo: tenant.repositoryId,
             release: check.release,
-            hashes: [...new Set(check.entries.map((entry) => entry.hash))],
+            hashes: [...new Set(check.entries.filter((entry) => entry.hash).map((entry) => entry.hash))],
         },
         format: "JSONEachRow",
     });
-    const stored = new Set((await rows.json()).map((row) => `${row.filename}\u0000${row.debug_id}\u0000${row.content_hash}`));
-    const present = [];
-    check.entries.forEach((entry, index) => {
-        if (stored.has(`${entry.filename}\u0000${entry.debugId}\u0000${entry.hash}`))
-            present.push(index);
-    });
-    return present;
+    return new Set((await rows.json()).map((row) => `${row.filename}\u0000${row.debug_id}\u0000${row.content_hash}`));
 }

package/dist/source-maps.d.ts

@@ -49,6 +49,7 @@ export declare function validateSourceMap(body: unknown): ValidSourceMap | null;
 export interface SourceMapCheckEntry {
     filename: string;
     debugId: string;
+    /** sha256 of the stored map, or "" for a check by debug id alone (1.7.0+). */
     hash: string;
 }
 export interface SourceMapCheck {
@@ -59,9 +60,12 @@ export interface SourceMapCheck {
 export declare const SOURCE_MAP_CHECK_MAX = 500;
 /**
  * Validate a `/v1/sourcemaps/check` request:
- * `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
+ * `{ release, files: [{ filename, debugId?, hash? }] }` (≤ 500 files). The
  * filenames and debug ids are normalised exactly as an upload would store
- * them. Null when invalid.
+ * them. Each file needs a hash (is this exact map stored? — what `upload`
+ * asks to skip maps) or, from 1.7.0, a debug id alone (is any map stored
+ * for this bundle? — what `sourcemaps verify` asks after the maps were
+ * deleted). Null when invalid.
  */
 export declare function validateSourceMapCheck(body: unknown): SourceMapCheck | null;
 /** Most scripts one browser event may name in `debugIds`. */

package/dist/source-maps.js

@@ -106,9 +106,12 @@ export function validateSourceMap(body) {
 export const SOURCE_MAP_CHECK_MAX = 500;
 /**
  * Validate a `/v1/sourcemaps/check` request:
- * `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
+ * `{ release, files: [{ filename, debugId?, hash? }] }` (≤ 500 files). The
  * filenames and debug ids are normalised exactly as an upload would store
- * them. Null when invalid.
+ * them. Each file needs a hash (is this exact map stored? — what `upload`
+ * asks to skip maps) or, from 1.7.0, a debug id alone (is any map stored
+ * for this bundle? — what `sourcemaps verify` asks after the maps were
+ * deleted). Null when invalid.
  */
 export function validateSourceMapCheck(body) {
     if (!body || typeof body !== "object")
@@ -123,7 +126,10 @@ export function validateSourceMapCheck(body) {
         if (!raw || typeof raw !== "object")
             return null;
         const file = raw;
-        if (typeof file.filename !== "string" || typeof file.hash !== "string" || !/^[0-9a-f]{64}$/.test(file.hash))
+        if (typeof file.filename !== "string")
+            return null;
+        const hash = file.hash === undefined || file.hash === null || file.hash === "" ? "" : file.hash;
+        if (typeof hash !== "string" || (hash !== "" && !/^[0-9a-f]{64}$/.test(hash)))
             return null;
         const filename = sourceMapPath(file.filename);
         if (!filename)
@@ -131,7 +137,9 @@ export function validateSourceMapCheck(body) {
         const debugId = file.debugId === undefined || file.debugId === null || file.debugId === "" ? "" : normalizeDebugId(file.debugId);
         if (debugId === null)
             return null;
-        entries.push({ filename, debugId, hash: file.hash });
+        if (!hash && !debugId)
+            return null;
+        entries.push({ filename, debugId, hash });
     }
     return { release: input.release, entries };
 }

package/dist/trace-dedupe.d.ts

@@ -0,0 +1,98 @@
+import type { RuntimeOccurrenceInput } from "./types.js";
+/**
+ * One error can be recorded on several spans of a trace. In a Next.js app a
+ * route handler's error is recorded by @autter/runtime-node on the request's
+ * HTTP server span AND by Next's own "executing api route" / render span
+ * (OpenTelemetry `recordException`); Express, Fastify, NestJS and gRPC
+ * instrumentations do the same, and errored ancestor spans with no event
+ * would add a stackless `HttpServerError` / `SpanError` on top. Each of
+ * those used to become its own occurrence, so one thrown error was counted
+ * two or three times. This pass, run on one export batch, keeps one
+ * occurrence per thrown error.
+ *
+ * Rule 1, duplicate recordings. Two exception occurrences are recordings of
+ * the same thrown error when ALL of these hold:
+ *   - same trace, service, environment and release (a thrown error lives in
+ *     one process);
+ *   - on DIFFERENT spans, one an ancestor of the other through parent links
+ *     present in the batch (an error propagating outward through nested
+ *     spans). Two events on one span are two recordings made on purpose (a
+ *     retry loop capturing each attempt on the request span) and are never
+ *     merged; nor are errors on sibling spans (one span per attempt);
+ *   - the same error type, or the OpenTelemetry JS `recordException`
+ *     convention: a third-party recording (no `autter.*` on its event) of an
+ *     error with a `code` reports the code as `exception.type` ("P2002",
+ *     "ECONNREFUSED"), so its type may instead equal the other's declared
+ *     `autter.error.code`, or its stack header may name the other's type;
+ *   - the same message after scrubbing and `normalizeMessage`;
+ *   - the same stack: the parsed in-app frames (function, file, line) are
+ *     equal or one list is a prefix (truncated) or suffix (captured at record
+ *     time, higher up) of the other. A side without a stack is no evidence
+ *     against; without in-app frames all frames are compared;
+ *   - recorded within DUPLICATE_WINDOW_MS of the group's latest recording.
+ * Recordings are grouped greedily in time order: each joins the eligible
+ * group (no member on its span, every member on its ancestor/descendant
+ * chain and matching it) whose latest recording is the most recent, so when
+ * every attempt of a retry loop is recorded on a child span and again on the
+ * parent, each attempt pairs with its own parent recording.
+ *
+ * The group keeps the occurrence with the most information: the one the
+ * Autter SDK recorded (`autter.*` on the event, or `autter.severity` on the
+ * span), then the one carrying runtime-next's `onRequestError` context, then
+ * the one on the span closest to the root. It keeps that occurrence's type,
+ * message, severity, time and batch position (so its deterministic id), and
+ * takes from the others what it lacks: attributes (its own win), stack,
+ * route, method, status, request id, declared error fields, fingerprint
+ * override and breadcrumbs. `autter.merged_recordings` counts the merged
+ * recordings.
+ *
+ * Rule 2, redundant status-only occurrences. A status-only `HttpServerError`
+ * / `SpanError` (an errored span without exception or failed-outcome events)
+ * is dropped when the same trace and service in this batch has an exception
+ * or failed outcome of severity error or fatal, not `autter.handled`, on an
+ * ancestor or descendant of its span: the span failed because of that
+ * error. It is kept when it is the trace's only signal, when the only
+ * errors are warnings or handled ones, or when no ancestor/descendant link
+ * is visible in the batch (sibling branches, a parent exported elsewhere).
+ *
+ * Scope: one export batch. A request's spans normally end, and are
+ * exported, together; when the exporter's timer splits them across batches,
+ * the recordings in different batches are still counted separately.
+ */
+/** Two recordings of one thrown error happen as it propagates outward:
+ * milliseconds apart, seconds at most for an error handler that awaits. */
+export declare const DUPLICATE_WINDOW_MS = 5000;
+export interface TraceSignal {
+    occurrence: RuntimeOccurrenceInput;
+    traceId: string;
+    spanId: string;
+    /** What produced it: an `exception` event, a failed `autter.outcome`
+     * event, or the span's error status alone. */
+    signal: "exception" | "outcome" | "status";
+    /** The Autter SDK recorded it: `autter.*` on the event, or
+     * `autter.severity` (captureException's marker) on the span. */
+    autterCapture: boolean;
+    /** `autter.*` attributes on the event itself. */
+    eventAutter: boolean;
+    /** runtime-next's `onRequestError` context on the span or event. */
+    nextContext: boolean;
+    /** `autter.handled` on the event: handled on purpose, not why a span failed. */
+    handled: boolean;
+}
+export interface SpanLink {
+    traceId: string;
+    spanId: string;
+    parentSpanId: string | null;
+}
+export interface DedupedSignals {
+    /** The surviving occurrences, in batch order (merged where rule 1 applied). */
+    occurrences: RuntimeOccurrenceInput[];
+    /** Input index of each survivor: its position in the batch, which its
+     * deterministic occurrence id is derived from. */
+    sourceIndexes: number[];
+    /** Input indexes each survivor stands for (itself first). */
+    members: number[][];
+}
+export declare function hasNextContext(...sources: Array<Map<string, string>>): boolean;
+export declare function hasAutterKey(attrs: Map<string, string>): boolean;
+export declare function dedupeTraceSignals(signals: TraceSignal[], spans: SpanLink[]): DedupedSignals;