@autter/otlp-ingester 1.7.0 → 1.8.0

package/README.md

@@ -60,6 +60,19 @@ The validator webhook may return the same extra fields:
 `{ orgId, repositoryId, scope?, allowedOrigins? }`, with `scope` one of
 `server`, `client`, `upload`.
 
+### Rejections and drops (1.8.0)
+
+Everything the ingester refuses (401, 403, 404, 413, 415, 429, 400, 503) or
+does not store from an accepted request (spans past 5,000, log records past
+2,000, cumulative HTTP histograms, invalid browser events, …) is counted per
+repository, minute, route and reason in `runtime_ingest_drops`; keys that
+don't resolve count against an empty tenant with the sha256 of the key when
+it is an Autter key. OTLP answers name dropped records in `partial_success`
+(JSON and protobuf), so collectors log them. Counts are written every 5 s in
+one batch, bounded in memory, and a failed write is logged and dropped:
+counting never slows or fails ingest. Reasons and queries:
+[docs/ARCHITECTURE.md § Rejections and drops](../../docs/ARCHITECTURE.md#rejections-and-drops).
+
 ### Version compatibility
 
 `GET /v1/compat` (public, no tenant data) returns this ingester's version,
@@ -67,9 +80,10 @@ its ClickHouse schema state, and which features from
 `src/compat-manifest.json` it supports. Add
 `?features=operation_logging&sdk=@autter/runtime-node@1.3.0` to get the evaluated
 incompatibilities. Every response carries `x-autter-ingester-version`. The
-ingester records the SDK name and version each service sends with
-(`runtime_sdk_versions`) and its own version (`runtime_ingester_info`), and
-the Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILITY.md).
+ingester records the SDK name and version each service sends with, and
+whether it came through an OpenTelemetry Collector (`runtime_sdk_versions`,
+`via` from 1.8.0), and its own version (`runtime_ingester_info`), and the
+Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILITY.md).
 
 ## Configuration
 

package/dist/auth.d.ts

@@ -1,6 +1,19 @@
 import type { Request } from "express";
 import { type IngesterConfig } from "./config.js";
 import type { IngestContext } from "./types.js";
+/**
+ * Why a key did not resolve: `unknown_key` — not a static key and the
+ * validator did not accept it; `key_unverified` — the validator could not
+ * answer (unreachable, timed out or a 5xx) and no earlier answer was cached.
+ */
+export type KeyFailure = "unknown_key" | "key_unverified";
+export type KeyResolution = {
+    ctx: IngestContext;
+    failure?: undefined;
+} | {
+    ctx: null;
+    failure: KeyFailure;
+};
 export declare class KeyResolver {
     private readonly config;
     private readonly staticKeys;
@@ -8,6 +21,8 @@ export declare class KeyResolver {
     constructor(config: IngesterConfig);
     extractKey(req: Request): string | null;
     resolve(key: string): Promise<IngestContext | null>;
+    /** `resolve`, plus why a key that did not resolve failed (for drop counts). */
+    resolveDetailed(key: string): Promise<KeyResolution>;
 }
 /** Fixed-window per-key rate limiter (single-node; Redis backing is M1). */
 export declare class RateLimiter {

package/dist/auth.js

@@ -36,15 +36,20 @@ export class KeyResolver {
         return null;
     }
     async resolve(key) {
+        return (await this.resolveDetailed(key)).ctx;
+    }
+    /** `resolve`, plus why a key that did not resolve failed (for drop counts). */
+    async resolveDetailed(key) {
         const staticCtx = this.staticKeys.get(key);
         if (staticCtx)
-            return staticCtx;
+            return { ctx: staticCtx };
         if (!this.config.keyValidatorUrl)
-            return null;
+            return { ctx: null, failure: "unknown_key" };
         const cached = this.cache.get(key);
         if (cached && cached.expiresAt > Date.now())
-            return cached.ctx;
+            return resolution(cached);
         let ctx = null;
+        let failure = "unknown_key";
         try {
             const response = await fetch(this.config.keyValidatorUrl, {
                 method: "POST",
@@ -74,24 +79,31 @@ export class KeyResolver {
                     };
                 }
             }
+            else if (response.status >= 500) {
+                failure = "key_unverified";
+            }
         }
         catch {
             // Validator unreachable: fail closed for unknown keys, but reuse a
             // stale cache entry if we have one so transient validator outages
             // don't drop telemetry from known-good keys.
             if (cached)
-                return cached.ctx;
-            return null;
+                return resolution(cached);
+            return { ctx: null, failure: "key_unverified" };
         }
-        this.cache.set(key, { ctx, expiresAt: Date.now() + VALIDATOR_CACHE_TTL_MS });
+        const entry = { ctx, expiresAt: Date.now() + VALIDATOR_CACHE_TTL_MS, ...(ctx ? {} : { failure }) };
+        this.cache.set(key, entry);
         if (this.cache.size > 10_000) {
             const oldest = this.cache.keys().next().value;
             if (oldest !== undefined)
                 this.cache.delete(oldest);
         }
-        return ctx;
+        return resolution(entry);
     }
 }
+function resolution(entry) {
+    return entry.ctx ? { ctx: entry.ctx } : { ctx: null, failure: entry.failure ?? "unknown_key" };
+}
 /** Fixed-window per-key rate limiter (single-node; Redis backing is M1). */
 export class RateLimiter {
     limitPerMinute;

package/dist/clickhouse.d.ts

@@ -6,6 +6,7 @@ import { type SourceMapCheck, type ValidSourceMap } from "./source-maps.js";
 import { type SourceMapKey } from "./source-maps-lookup.js";
 import { type RuntimeLogRecord } from "./logs.js";
 import { type SdkVersionRow } from "./sdk-versions.js";
+import { type IngestDropRow } from "./ingest-drops.js";
 import type { SchemaStatus } from "./compat.js";
 import { type Migration } from "./migrations.js";
 import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
@@ -54,6 +55,8 @@ export declare class ClickHouseStore {
         applied: string[];
     };
     insertSdkVersions(ctx: IngestContext, rows: SdkVersionRow[]): Promise<void>;
+    /** Rejection and drop counts (ingest-drops.ts); rows carry their own tenant. */
+    insertIngestDrops(rows: IngestDropRow[]): Promise<void>;
     /** Record this ingester's version and schema level (global, not tenant data). */
     insertIngesterInfo(info: {
         version: string;

package/dist/clickhouse.js

@@ -7,7 +7,9 @@ import { attributesWithBreadcrumbs } from "./breadcrumbs.js";
 import { lookupSourceMap, presentSourceMaps } from "./source-maps-lookup.js";
 import { logTableDDL, requestRollupTableDDL, } from "./logs.js";
 import { ingesterInfoTableDDL, sdkVersionTableDDL, } from "./sdk-versions.js";
-import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_GROUPING_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
+import { ingestDropsTableDDL } from "./ingest-drops.js";
+import { inAppFrameResolution } from "./symbolicate.js";
+import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_FRAME_RESOLUTION_COLUMNS, OCCURRENCE_GROUPING_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
 /**
  * ClickHouse persistence. The schema is idempotent and applied lazily once
  * per process. Every row is keyed by (org_id, repository_id) — tenant
@@ -22,6 +24,14 @@ const INSERT_SETTINGS = {
 /** Source maps above this go in with a plain insert (below ClickHouse's
  * default async_insert_max_data_size of 10 MiB). */
 const ASYNC_INSERT_MAX_MAP_CHARS = 8 * 1024 * 1024;
+/** Migration 0020 columns: browser occurrences only (server frames are not symbolicated). */
+function occurrenceFrameResolution(o) {
+    const counts = o.source === "browser" ? inAppFrameResolution(o.frames) : { resolved: 0, unresolved: 0 };
+    return {
+        in_app_frames_resolved: Math.min(65_535, counts.resolved),
+        in_app_frames_unresolved: Math.min(65_535, counts.unresolved),
+    };
+}
 /** "YYYY-MM-DD hh:mm:ss.sss" (UTC) — the DateTime64 query-parameter form. */
 function chDateTime(date) {
     return date.toISOString().replace("T", " ").replace("Z", "");
@@ -74,6 +84,7 @@ export class ClickHouseStore {
             requestRollupTableDDL(db),
             sdkVersionTableDDL(db, metricsTtlDays),
             ingesterInfoTableDDL(db),
+            ingestDropsTableDDL(db),
             `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
 				org_id             String,
 				repository_id      String,
@@ -101,6 +112,7 @@ export class ClickHouseStore {
 				ingested_at        DateTime64(3, 'UTC') DEFAULT now64(3),
 				${OCCURRENCE_CODE_COLUMNS.join(",\n\t\t\t\t")},
 				${OCCURRENCE_GROUPING_COLUMNS.join(",\n\t\t\t\t")},
+				${OCCURRENCE_FRAME_RESOLUTION_COLUMNS.join(",\n\t\t\t\t")},
 				${OCCURRENCE_REQUEST_ID_INDEX}
 			)
 			ENGINE = MergeTree
@@ -317,6 +329,30 @@ export class ClickHouseStore {
                 sdk_language: row.sdkLanguage,
                 ingester_version: row.ingesterVersion,
                 last_seen: row.seenAt.toISOString(),
+                via: row.via,
+            })),
+        });
+    }
+    /** Rejection and drop counts (ingest-drops.ts); rows carry their own tenant. */
+    async insertIngestDrops(rows) {
+        if (!rows.length || !this.configured)
+            return;
+        await this.ensureSchema();
+        await this.getClient().insert({
+            table: this.table("runtime_ingest_drops"),
+            format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: rows.map((row) => ({
+                org_id: row.orgId,
+                repository_id: row.repositoryId,
+                minute: row.minute.toISOString(),
+                route: row.route,
+                reason: row.reason,
+                kind: row.kind,
+                status: row.status,
+                key_hash: row.keyHash,
+                requests: row.requests,
+                records: row.records,
             })),
         });
     }
@@ -352,6 +388,7 @@ export class ClickHouseStore {
             format: "JSONEachRow",
             clickhouse_settings: INSERT_SETTINGS,
             values: occurrences.map((o) => ({
+                ...occurrenceFrameResolution(o),
                 org_id: ctx.orgId,
                 repository_id: ctx.repositoryId,
                 occurrence_id: o.occurrenceId,

package/dist/compat-manifest.json

@@ -134,6 +134,27 @@
             "migrations": ["0017-source-maps-debug-id"],
             "route": "/v1/sourcemaps/check",
             "sdks": {}
+        },
+        {
+            "id": "ingest_drop_counts",
+            "title": "Rejected and dropped records per repository",
+            "ingester": "1.8.0",
+            "migrations": ["0018-runtime-ingest-drops"],
+            "sdks": {}
+        },
+        {
+            "id": "telemetry_sender_via",
+            "title": "How each service's telemetry arrives (SDK, OpenTelemetry Collector or browser relay)",
+            "ingester": "1.8.0",
+            "migrations": ["0019-sdk-versions-via"],
+            "sdks": {}
+        },
+        {
+            "id": "frame_resolution_counts",
+            "title": "Resolved and unresolved browser frames per occurrence",
+            "ingester": "1.8.0",
+            "migrations": ["0020-occurrence-frame-resolution"],
+            "sdks": {}
         }
     ]
 }

package/dist/context.d.ts

@@ -1,3 +1,8 @@
+/** Run a synchronous normaliser and return what it cut along with its result. */
+export declare function countTruncations<T>(normalise: () => T): {
+    result: T;
+    truncated: number;
+};
 /** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
 export declare function sanitizeRuntimeContext(input: unknown): Record<string, unknown>;
 export interface OtlpValue {

package/dist/context.js

@@ -1,5 +1,20 @@
 import { isSensitiveKey, scrubText } from "./redact.js";
 import { BREADCRUMBS_ATTRIBUTE } from "./breadcrumbs.js";
+/**
+ * Attribute values the bounds below cut or left out (a string past its
+ * length cap, list items past 64, map keys past 128, nesting past 6, the
+ * 512-value budget), counted since the process started. The normalisers are
+ * synchronous, so the difference across one call is that call's own count:
+ * see `countTruncations`. Privacy rewrites (redaction, query stripping) are
+ * not truncation and are not counted.
+ */
+let truncatedValues = 0;
+/** Run a synchronous normaliser and return what it cut along with its result. */
+export function countTruncations(normalise) {
+    const before = truncatedValues;
+    const result = normalise();
+    return { result, truncated: truncatedValues - before };
+}
 /** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
 export function sanitizeRuntimeContext(input) {
     let budget = 512;
@@ -15,8 +30,10 @@ export function sanitizeRuntimeContext(input) {
         return cut <= 0 ? url : url.slice(0, url.length - rest.length + cut);
     });
     const visit = (value, key, depth) => {
-        if (--budget < 0 || depth > 6)
+        if (--budget < 0 || depth > 6) {
+            truncatedValues += 1;
             return "[truncated]";
+        }
         if (/__proto__|constructor|prototype/i.test(key))
             return undefined;
         const usage = 
@@ -44,6 +61,8 @@ export function sanitizeRuntimeContext(input) {
             // Cut before scrubbing so scrub work is bounded by the stored size;
             // the slack keeps a secret straddling the cut whole for the scrubber.
             const limit = /stack/i.test(key) ? 32000 : 2048;
+            if (text.length > limit)
+                truncatedValues += 1;
             return scrub(text.slice(0, limit + 512)).slice(0, limit);
         }
         if (typeof value === "number")
@@ -55,11 +74,19 @@ export function sanitizeRuntimeContext(input) {
         if (seen.has(value))
             return "[circular]";
         seen.add(value);
-        if (Array.isArray(value))
+        if (Array.isArray(value)) {
+            if (value.length > 64)
+                truncatedValues += value.length - 64;
             return value.slice(0, 64).map((v) => visit(v, key, depth + 1));
+        }
         const result = {};
-        for (const [k, v] of Object.entries(value).slice(0, 128)) {
+        const entries = Object.entries(value);
+        if (entries.length > 128)
+            truncatedValues += entries.length - 128;
+        for (const [k, v] of entries.slice(0, 128)) {
             const safe = visit(v, k, depth + 1);
+            if (k.length > 200)
+                truncatedValues += 1;
             if (safe !== undefined)
                 result[k.slice(0, 200)] = safe;
         }
@@ -72,8 +99,12 @@ export function sanitizeRuntimeContext(input) {
 }
 export function decodeOtlpAttributes(attributes) {
     const decode = (v, depth = 0) => {
-        if (!v || depth > 6)
+        if (!v)
+            return undefined;
+        if (depth > 6) {
+            truncatedValues += 1;
             return undefined;
+        }
         if (v.stringValue !== undefined)
             return v.stringValue;
         if (v.intValue !== undefined)
@@ -82,17 +113,25 @@ export function decodeOtlpAttributes(attributes) {
             return v.doubleValue;
         if (v.boolValue !== undefined)
             return v.boolValue;
-        if (v.arrayValue)
-            return (v.arrayValue.values ?? [])
-                .slice(0, 64)
-                .map((value) => decode(value, depth + 1));
-        if (v.kvlistValue)
-            return Object.fromEntries((v.kvlistValue.values ?? [])
+        if (v.arrayValue) {
+            const values = v.arrayValue.values ?? [];
+            if (values.length > 64)
+                truncatedValues += values.length - 64;
+            return values.slice(0, 64).map((value) => decode(value, depth + 1));
+        }
+        if (v.kvlistValue) {
+            const values = v.kvlistValue.values ?? [];
+            if (values.length > 128)
+                truncatedValues += values.length - 128;
+            return Object.fromEntries(values
                 .slice(0, 128)
                 .filter((a) => a.key && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
                 .map((a) => [a.key, decode(a.value, depth + 1)]));
+        }
         return undefined;
     };
+    if (Array.isArray(attributes) && attributes.length > 128)
+        truncatedValues += attributes.length - 128;
     return sanitizeRuntimeContext(Object.fromEntries((attributes ?? [])
         .slice()
         .sort((a, b) => Number(/^autter\.(?:operation|event)\./.test(b.key ?? "")) -

package/dist/index.d.ts

@@ -1,5 +1,6 @@
 export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
 export * from "./compat.js";
-export { SinkForwarder, type SinkStats, type SinkTuning } from "./sink.js";
+export { SinkForwarder, type SinkDrop, type SinkStats, type SinkTuning } from "./sink.js";
+export { DROP_REASONS, IngestDropCounter, type DropKind, type DropReason, type DropRoute, type IngestDrop, type IngestDropRow, } from "./ingest-drops.js";
 export * from "./types.js";

package/dist/index.js

@@ -2,7 +2,7 @@ import { loadConfig } from "./config.js";
 import { createIngesterApp } from "./server.js";
 import { ingesterVersion } from "./sdk-versions.js";
 const config = loadConfig();
-const { app, store, sink, reportIngesterInfo } = createIngesterApp(config);
+const { app, store, sink, reportIngesterInfo, drops } = createIngesterApp(config);
 const server = app.listen(config.port, () => {
     console.log(`autter otlp-ingester ${ingesterVersion()} listening on :${config.port} ` +
         `(clickhouse: ${config.clickhouseUrl ? "configured" : "NOT configured"})`);
@@ -25,7 +25,12 @@ async function shutdown(signal) {
         }
     }
     server.close(() => {
-        void store.close().finally(() => process.exit(0));
+        // Write the rejection/drop counts still buffered (best effort: a
+        // failed write is logged, never retried).
+        void drops
+            .stop()
+            .finally(() => store.close())
+            .finally(() => process.exit(0));
     });
     setTimeout(() => process.exit(1), 10_000).unref();
 }
@@ -35,4 +40,5 @@ export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
 export * from "./compat.js";
 export { SinkForwarder } from "./sink.js";
+export { DROP_REASONS, IngestDropCounter, } from "./ingest-drops.js";
 export * from "./types.js";