@autter/otlp-ingester 1.6.0 → 1.7.0

package/README.md

@@ -31,6 +31,16 @@ works with its default exporter settings.
 | --- | --- | --- | --- |
 | `server` (default) | `autter_rt_…` (secret) | all endpoints | 300 req/min |
 | `client` | `autter_rtc_…` (publishable, safe in frontend bundles) | `/v1/browser` only | origin allow-list, 120 req/min |
+| `upload` (1.7.0+) | `autter_rtu_…` (secret, for CI) | `/v1/sourcemaps` and `/v1/sourcemaps/check` only | 300 req/min |
+
+An upload key gets `403 {"error":"upload_key_not_allowed","detail":"autter_rtu_ keys can only upload source maps …"}`
+on every other route, so a CI secret that leaks can upload source maps for
+one repository and nothing else. A static entry without `scope` is an upload
+key when its key starts with `autter_rtu_`, else a server key; an entry (or a
+validator answer) with any other `scope` is ignored rather than treated as a
+server key. Ingesters before 1.7.0 treat every scope but `client` as
+`server`, so a validator must not answer `"scope": "upload"` to them: from
+1.7.0 every validator request carries `x-autter-ingester-version`.
 
 `/v1/browser` answers CORS preflights permissively; real enforcement (key +
 origin allow-list) happens on the POST. Cross-origin browsers send
@@ -41,12 +51,14 @@ route parses as JSON.
 AUTTER_INGEST_KEYS='[
   {"key":"autter_rt_…","orgId":"org1","repositoryId":"repo1"},
   {"key":"autter_rtc_…","orgId":"org1","repositoryId":"repo1",
-   "scope":"client","allowedOrigins":["https://app.example.com"]}
+   "scope":"client","allowedOrigins":["https://app.example.com"]},
+  {"key":"autter_rtu_…","orgId":"org1","repositoryId":"repo1","scope":"upload"}
 ]'
 ```
 
 The validator webhook may return the same extra fields:
-`{ orgId, repositoryId, scope?, allowedOrigins? }`.
+`{ orgId, repositoryId, scope?, allowedOrigins? }`, with `scope` one of
+`server`, `client`, `upload`.
 
 ### Version compatibility
 
@@ -75,7 +87,7 @@ the Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIB
 | `SINK_MAX_ATTEMPTS` | `12` | Delivery attempts per batch (1–60 s backoff) |
 | `SINK_MAX_BUFFERED_BATCHES` | `1000` | Retry buffer cap; oldest drops are logged |
 | `SINK_MAX_BUFFERED_MB` | `64` | Sink retry buffer cap (memory) |
-| `SOURCE_MAP_CACHE_MB` | `64` | Memory budget (estimated decoded size) for parsed source maps used to symbolicate browser frames at ingest; least recently used maps are evicted |
+| `SOURCE_MAP_CACHE_MB` | `64` | Memory budget (decoded size, function scope tables included) for parsed source maps used to symbolicate browser frames at ingest; least recently used maps are evicted, and a map that needs more than half of it is used without being kept |
 | `MAX_BODY_BYTES` | `1048576` | Request body cap |
 | `RATE_LIMIT_PER_MINUTE` | `300` | Per-key fixed window (server keys) |
 | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) |

package/dist/auth.d.ts

@@ -1,5 +1,5 @@
 import type { Request } from "express";
-import type { IngesterConfig } from "./config.js";
+import { type IngesterConfig } from "./config.js";
 import type { IngestContext } from "./types.js";
 export declare class KeyResolver {
     private readonly config;

package/dist/auth.js

@@ -1,3 +1,6 @@
+import { INGESTER_VERSION_HEADER } from "./compat.js";
+import { defaultScopeFor, isIngestKeyScope } from "./config.js";
+import { ingesterVersion } from "./sdk-versions.js";
 const VALIDATOR_CACHE_TTL_MS = 60_000;
 export class KeyResolver {
     config;
@@ -6,10 +9,13 @@ export class KeyResolver {
     constructor(config) {
         this.config = config;
         for (const entry of config.ingestKeys) {
+            const scope = entry.scope ?? defaultScopeFor(entry.key);
+            if (!isIngestKeyScope(scope))
+                continue;
             this.staticKeys.set(entry.key, {
                 orgId: entry.orgId,
                 repositoryId: entry.repositoryId,
-                scope: entry.scope ?? "server",
+                scope,
                 allowedOrigins: entry.allowedOrigins ?? [],
             });
         }
@@ -44,6 +50,11 @@ export class KeyResolver {
                 method: "POST",
                 headers: {
                     "content-type": "application/json",
+                    // Lets the validator tell which scopes this ingester enforces:
+                    // before 1.7.0 any scope but "client" meant a server key, so an
+                    // upload key must not be answered as "upload" to an ingester
+                    // that doesn't send this header (or sends an older version).
+                    [INGESTER_VERSION_HEADER]: ingesterVersion(),
                     ...(this.config.keyValidatorToken
                         ? { authorization: `Bearer ${this.config.keyValidatorToken}` }
                         : {}),
@@ -53,11 +64,12 @@ export class KeyResolver {
             });
             if (response.ok) {
                 const body = (await response.json());
-                if (body.orgId && body.repositoryId) {
+                const scope = body.scope === undefined || body.scope === null ? defaultScopeFor(key) : body.scope;
+                if (body.orgId && body.repositoryId && isIngestKeyScope(scope)) {
                     ctx = {
                         orgId: body.orgId,
                         repositoryId: body.repositoryId,
-                        scope: body.scope ?? "server",
+                        scope,
                         allowedOrigins: body.allowedOrigins ?? [],
                     };
                 }

package/dist/clickhouse.js

@@ -19,6 +19,9 @@ const INSERT_SETTINGS = {
     async_insert: 1,
     wait_for_async_insert: 1,
 };
+/** Source maps above this go in with a plain insert (below ClickHouse's
+ * default async_insert_max_data_size of 10 MiB). */
+const ASYNC_INSERT_MAX_MAP_CHARS = 8 * 1024 * 1024;
 /** "YYYY-MM-DD hh:mm:ss.sss" (UTC) — the DateTime64 query-parameter form. */
 function chDateTime(date) {
     return date.toISOString().replace("T", " ").replace("Z", "");
@@ -474,7 +477,15 @@ export class ClickHouseStore {
         await this.ensureSchema();
         await this.getClient().insert({
             table: this.table("runtime_source_maps"), format: "JSONEachRow",
-            clickhouse_settings: INSERT_SETTINGS,
+            // One row of up to 32 MiB. Parallel parsing cuts input into
+            // 10 MiB chunks, refuses single JSON rows far past that, and gains
+            // nothing on one row. A map past the async-insert buffer (10 MiB
+            // by default) is inserted directly rather than buffered.
+            clickhouse_settings: {
+                ...INSERT_SETTINGS,
+                input_format_parallel_parsing: 0,
+                ...(sourceMap.map.length > ASYNC_INSERT_MAX_MAP_CHARS ? { async_insert: 0 } : {}),
+            },
             values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId, release: sourceMap.release,
                     filename: sourceMap.filename, map: sourceMap.map, debug_id: sourceMap.debugId,
                     content_hash: sourceMap.contentHash }],

package/dist/compat-manifest.json

@@ -118,6 +118,22 @@
             "ingester": "1.6.0",
             "migrations": [],
             "sdks": { "@autter/runtime-browser": "1.6.0", "@autter/runtime-next": "1.6.0", "@autter/runtime-node": "1.6.0" }
+        },
+        {
+            "id": "sourcemap_upload_keys",
+            "title": "Upload-only keys for source maps (autter_rtu_)",
+            "ingester": "1.7.0",
+            "migrations": [],
+            "route": "/v1/sourcemaps",
+            "sdks": {}
+        },
+        {
+            "id": "source_map_verify",
+            "title": "Source map check by debug id (autter-runtime sourcemaps verify)",
+            "ingester": "1.7.0",
+            "migrations": ["0017-source-maps-debug-id"],
+            "route": "/v1/sourcemaps/check",
+            "sdks": {}
         }
     ]
 }

package/dist/config.d.ts

@@ -1,9 +1,26 @@
+import type { IngestKeyScope } from "./types.js";
+export declare const INGEST_KEY_SCOPES: readonly IngestKeyScope[];
+/** Prefix of upload-only keys (`autter_rtu_…`): source map uploads from CI. */
+export declare const UPLOAD_KEY_PREFIX = "autter_rtu_";
+export declare function isIngestKeyScope(value: unknown): value is IngestKeyScope;
+/**
+ * The scope of a key whose static entry or validator answer names none:
+ * "upload" for an `autter_rtu_…` key (its prefix promises no more than
+ * that), else "server" as before 1.7.0.
+ */
+export declare function defaultScopeFor(key: string): IngestKeyScope;
 export interface StaticIngestKey {
     key: string;
     orgId: string;
     repositoryId: string;
-    /** "server" (default) = secret backend key; "client" = publishable browser key. */
-    scope?: "client" | "server";
+    /**
+     * "server" = secret backend key, every route; "client" = publishable
+     * browser key, `/v1/browser` only; "upload" = CI key, `/v1/sourcemaps`
+     * and `/v1/sourcemaps/check` only. Default: "upload" for an
+     * `autter_rtu_…` key, else "server". An entry with any other value is
+     * ignored (never treated as a server key).
+     */
+    scope?: IngestKeyScope;
     /** client keys: exact origins allowed to send (e.g. https://app.example.com). */
     allowedOrigins?: string[];
 }
@@ -55,10 +72,14 @@ export interface IngesterConfig {
     redactValuePatterns?: string[];
     redactKeyPatterns?: string[];
     /**
-     * In-memory budget (MiB, estimated decoded size) for parsed source maps
-     * used to symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB,
-     * default 64).
+     * In-memory budget (MiB, decoded size) for parsed source maps used to
+     * symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB, default 64).
+     * A map that needs more than half of it is used without being kept.
      */
     sourceMapCacheMb?: number;
 }
+/** Parse AUTTER_INGEST_KEYS. Entries without a string key, orgId and
+ * repositoryId, or with an unknown scope, are dropped (a typo in `scope`
+ * must not turn a restricted key into a server key). */
+export declare function parseIngestKeys(raw: string | undefined): StaticIngestKey[];
 export declare function loadConfig(): IngesterConfig;

package/dist/config.js

@@ -1,3 +1,17 @@
+export const INGEST_KEY_SCOPES = ["server", "client", "upload"];
+/** Prefix of upload-only keys (`autter_rtu_…`): source map uploads from CI. */
+export const UPLOAD_KEY_PREFIX = "autter_rtu_";
+export function isIngestKeyScope(value) {
+    return typeof value === "string" && INGEST_KEY_SCOPES.includes(value);
+}
+/**
+ * The scope of a key whose static entry or validator answer names none:
+ * "upload" for an `autter_rtu_…` key (its prefix promises no more than
+ * that), else "server" as before 1.7.0.
+ */
+export function defaultScopeFor(key) {
+    return key.startsWith(UPLOAD_KEY_PREFIX) ? "upload" : "server";
+}
 function intEnv(name, fallback) {
     const raw = process.env[name];
     if (!raw)
@@ -19,22 +33,33 @@ function patternListEnv(name) {
         new RegExp(source);
     return parsed;
 }
-function parseIngestKeys(raw) {
+/** Parse AUTTER_INGEST_KEYS. Entries without a string key, orgId and
+ * repositoryId, or with an unknown scope, are dropped (a typo in `scope`
+ * must not turn a restricted key into a server key). */
+export function parseIngestKeys(raw) {
     if (!raw)
         return [];
+    let parsed;
     try {
-        const parsed = JSON.parse(raw);
-        if (!Array.isArray(parsed))
-            return [];
-        return parsed.filter((entry) => entry &&
-            typeof entry.key === "string" &&
-            typeof entry.orgId === "string" &&
-            typeof entry.repositoryId === "string");
+        parsed = JSON.parse(raw);
     }
     catch {
         console.error("AUTTER_INGEST_KEYS is not valid JSON — ignoring");
         return [];
     }
+    if (!Array.isArray(parsed))
+        return [];
+    const keys = [];
+    parsed.forEach((entry, index) => {
+        if (!entry || typeof entry.key !== "string" || typeof entry.orgId !== "string" || typeof entry.repositoryId !== "string")
+            return;
+        if (entry.scope !== undefined && !isIngestKeyScope(entry.scope)) {
+            console.error(`AUTTER_INGEST_KEYS[${index}] has scope ${JSON.stringify(entry.scope)}; expected one of ${INGEST_KEY_SCOPES.join(", ")} — ignoring this key`);
+            return;
+        }
+        keys.push({ ...entry, scope: entry.scope ?? defaultScopeFor(entry.key) });
+    });
+    return keys;
 }
 export function loadConfig() {
     const config = {

package/dist/normalize-browser.d.ts

@@ -52,65 +52,65 @@ export declare const browserPayloadSchema: z.ZodObject<{
          */
         traceId: z.ZodCatch<z.ZodOptional<z.ZodString>>;
     }, "strip", z.ZodTypeAny, {
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown[] | undefined;
+        route?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: string | undefined;
-        breadcrumbs?: unknown[] | undefined;
         debugIds?: Record<string, unknown> | undefined;
         durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        name?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
     }, {
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown;
+        route?: string | undefined;
         message?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: unknown;
-        breadcrumbs?: unknown;
         debugIds?: unknown;
         durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        name?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
+    version: 1;
     service: string;
     environment: string;
-    version: 1;
     events: {
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown[] | undefined;
+        route?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: string | undefined;
-        breadcrumbs?: unknown[] | undefined;
         debugIds?: Record<string, unknown> | undefined;
         durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        name?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
     }[];
@@ -118,26 +118,26 @@ export declare const browserPayloadSchema: z.ZodObject<{
     sessionId?: string | undefined;
     sdk?: string | undefined;
 }, {
+    version: 1;
     service: string;
     environment: string;
-    version: 1;
     events: {
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown;
+        route?: string | undefined;
         message?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: unknown;
-        breadcrumbs?: unknown;
         debugIds?: unknown;
         durationMs?: number | undefined;
         filename?: string | undefined;
-        id?: string | undefined;
-        name?: string | undefined;
-        line?: number | undefined;
         column?: number | undefined;
+        line?: number | undefined;
+        name?: string | undefined;
+        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
     }[];

package/dist/normalize-otlp.d.ts

@@ -78,7 +78,13 @@ export interface OtlpMetricsRequest {
     }>;
 }
 export interface NormalizedTraces {
+    /** One per thrown error: duplicate recordings across a trace's spans are
+     * merged and redundant status-only occurrences dropped (trace-dedupe.ts). */
     occurrences: RuntimeOccurrenceInput[];
+    /** Each occurrence's position among every signal the batch recorded,
+     * before deduplication: the batch index its deterministic id is derived
+     * from, so a merge or a drop never shifts another occurrence's id. */
+    occurrenceBatchIndexes: number[];
     spans: RuntimeSpanRow[];
     metricPoints: RuntimeMetricPoint[];
     llmCalls: RuntimeLlmCall[];