@autter/otlp-ingester 1.6.1 → 1.8.0

package/dist/migrations.js

@@ -37,7 +37,8 @@ import { profileTableDDL } from "./profiles.js";
 import { SOURCE_MAP_DEBUG_COLUMNS, SOURCE_MAP_DEBUG_ID_INDEX, sourceMapTableDDL, } from "./source-maps.js";
 import { LOG_REQUEST_COLUMNS, LOG_REQUEST_ID_INDEX, logTableDDL, requestRollupTableDDL, requestRollupViewDDL, } from "./logs.js";
 import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
-import { ingesterInfoTableDDL, sdkVersionTableDDL } from "./sdk-versions.js";
+import { ingesterInfoTableDDL, SDK_VERSION_VIA_COLUMN, sdkVersionTableDDL } from "./sdk-versions.js";
+import { ingestDropsTableDDL } from "./ingest-drops.js";
 /** Migration 0013 columns — shared with the baseline DDL in clickhouse.ts. */
 export const OCCURRENCE_CODE_COLUMNS = [
     "error_code String DEFAULT ''",
@@ -62,6 +63,17 @@ export const OCCURRENCE_GROUPING_COLUMNS = [
     "frames String DEFAULT '' CODEC(ZSTD(3))",
     "fingerprints Array(String) DEFAULT []",
 ];
+/**
+ * Migration 0020 columns — shared with the baseline DDL in clickhouse.ts.
+ * In-app browser JS frames a source map resolved / left unresolved at
+ * ingest (symbolicate.ts inAppFrameResolution); 0 and 0 for server
+ * occurrences, browser ones with no in-app JS frame, and rows written
+ * before 0020.
+ */
+export const OCCURRENCE_FRAME_RESOLUTION_COLUMNS = [
+    "in_app_frames_resolved UInt16 DEFAULT 0",
+    "in_app_frames_unresolved UInt16 DEFAULT 0",
+];
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -202,6 +214,22 @@ export const MIGRATIONS = [
             `ALTER TABLE {db}.runtime_source_maps ADD ${SOURCE_MAP_DEBUG_ID_INDEX.replace("INDEX ", "INDEX IF NOT EXISTS ")}`,
         ],
     },
+    // Rejections and drops per tenant, minute, route and reason (EI-27):
+    // what the ingester refused or did not store, so a repository's missing
+    // telemetry has a reason (ingest-drops.ts).
+    { id: "0018-runtime-ingest-drops", statements: [ingestDropsTableDDL("{db}")] },
+    // How each service's telemetry arrived: an OpenTelemetry Collector, an
+    // SDK exporter directly, or the browser relay (sdk-versions.ts).
+    {
+        id: "0019-sdk-versions-via",
+        statements: [`ALTER TABLE {db}.runtime_sdk_versions ADD COLUMN IF NOT EXISTS ${SDK_VERSION_VIA_COLUMN}`],
+    },
+    // Resolved and unresolved in-app browser frames per occurrence, after
+    // symbolication — the share the EI-14 source map rule reads.
+    {
+        id: "0020-occurrence-frame-resolution",
+        statements: OCCURRENCE_FRAME_RESOLUTION_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
+    },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -1,13 +1,9 @@
 import { z } from "zod";
 import { type RuntimeMetricPoint, type RuntimeOccurrenceInput } from "./types.js";
+/** Events past this in one payload are dropped and counted (events_over_limit). */
+export declare const MAX_BROWSER_EVENTS = 50;
+/** A whole payload, every event valid: what normalizeBrowserPayload takes. */
 export declare const browserPayloadSchema: z.ZodObject<{
-    version: z.ZodLiteral<1>;
-    sessionId: z.ZodOptional<z.ZodString>;
-    service: z.ZodString;
-    environment: z.ZodString;
-    release: z.ZodOptional<z.ZodString>;
-    /** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
-    sdk: z.ZodCatch<z.ZodOptional<z.ZodString>>;
     events: z.ZodArray<z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
         type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event", "outcome", "request_failure", "csp_violation", "timing"]>;
@@ -52,64 +48,71 @@ export declare const browserPayloadSchema: z.ZodObject<{
          */
         traceId: z.ZodCatch<z.ZodOptional<z.ZodString>>;
     }, "strip", z.ZodTypeAny, {
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown[] | undefined;
+        route?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: string | undefined;
-        breadcrumbs?: unknown[] | undefined;
         debugIds?: Record<string, unknown> | undefined;
         durationMs?: number | undefined;
         filename?: string | undefined;
+        name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        name?: string | undefined;
         id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
     }, {
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown;
+        route?: string | undefined;
         message?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: unknown;
-        breadcrumbs?: unknown;
         debugIds?: unknown;
         durationMs?: number | undefined;
         filename?: string | undefined;
+        name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        name?: string | undefined;
         id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
     }>, "many">;
+    version: z.ZodLiteral<1>;
+    sessionId: z.ZodOptional<z.ZodString>;
+    service: z.ZodString;
+    environment: z.ZodString;
+    release: z.ZodOptional<z.ZodString>;
+    /** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
+    sdk: z.ZodCatch<z.ZodOptional<z.ZodString>>;
 }, "strip", z.ZodTypeAny, {
+    version: 1;
     service: string;
     environment: string;
-    version: 1;
     events: {
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown[] | undefined;
+        route?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: string | undefined;
-        breadcrumbs?: unknown[] | undefined;
         debugIds?: Record<string, unknown> | undefined;
         durationMs?: number | undefined;
         filename?: string | undefined;
+        name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        name?: string | undefined;
         id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
@@ -118,25 +121,25 @@ export declare const browserPayloadSchema: z.ZodObject<{
     sessionId?: string | undefined;
     sdk?: string | undefined;
 }, {
+    version: 1;
     service: string;
     environment: string;
-    version: 1;
     events: {
-        type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
+        breadcrumbs?: unknown;
+        route?: string | undefined;
         message?: string | undefined;
         severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
         stack?: string | undefined;
-        route?: string | undefined;
         traceId?: unknown;
-        breadcrumbs?: unknown;
         debugIds?: unknown;
         durationMs?: number | undefined;
         filename?: string | undefined;
+        name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        name?: string | undefined;
         id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
@@ -146,6 +149,31 @@ export declare const browserPayloadSchema: z.ZodObject<{
     sdk?: unknown;
 }>;
 export type BrowserPayload = z.infer<typeof browserPayloadSchema>;
+export interface ParsedBrowserPayload {
+    /** The envelope with its valid events (at most MAX_BROWSER_EVENTS). */
+    payload: BrowserPayload;
+    /** Events dropped because they failed validation (invalid_event). */
+    invalidEvents: number;
+    /** Events past MAX_BROWSER_EVENTS, not looked at (events_over_limit). */
+    overLimitEvents: number;
+    /** The first validation issues of the dropped events; `path` starts with ["events", index]. */
+    issues: z.ZodIssue[];
+}
+/**
+ * Validate a browser payload event by event (1.8.0). The envelope (version,
+ * service, environment) must be valid or the payload is refused; an event
+ * that fails validation — a type this ingester doesn't know, a bad
+ * timestamp, an oversized field — is dropped on its own and counted, so it
+ * no longer costs the errors sent in the same beacon. Events past
+ * MAX_BROWSER_EVENTS are dropped and counted unread.
+ */
+export declare function parseBrowserPayload(body: unknown): {
+    success: true;
+    data: ParsedBrowserPayload;
+} | {
+    success: false;
+    issues: z.ZodIssue[];
+};
 export interface NormalizedBrowser {
     occurrences: RuntimeOccurrenceInput[];
     metricPoints: RuntimeMetricPoint[];

package/dist/normalize-browser.js

@@ -67,7 +67,9 @@ const browserEventSchema = z.object({
      */
     traceId: z.string().regex(/^(?!0{32})[0-9a-f]{32}$/).optional().catch(undefined),
 });
-export const browserPayloadSchema = z.object({
+/** Events past this in one payload are dropped and counted (events_over_limit). */
+export const MAX_BROWSER_EVENTS = 50;
+const browserEnvelope = {
     version: z.literal(1),
     sessionId: z.string().max(100).optional(),
     service: z.string().min(1).max(200),
@@ -75,8 +77,53 @@ export const browserPayloadSchema = z.object({
     release: z.string().max(200).optional(),
     /** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
     sdk: z.string().regex(/^\d{1,6}\.\d{1,6}\.\d{1,6}[0-9A-Za-z.+-]{0,20}$/).optional().catch(undefined),
-    events: z.array(browserEventSchema).max(50),
+};
+/** A whole payload, every event valid: what normalizeBrowserPayload takes. */
+export const browserPayloadSchema = z.object({
+    ...browserEnvelope,
+    events: z.array(browserEventSchema).max(MAX_BROWSER_EVENTS),
 });
+const browserEnvelopeSchema = z.object({ ...browserEnvelope, events: z.array(z.unknown()) });
+/**
+ * Validate a browser payload event by event (1.8.0). The envelope (version,
+ * service, environment) must be valid or the payload is refused; an event
+ * that fails validation — a type this ingester doesn't know, a bad
+ * timestamp, an oversized field — is dropped on its own and counted, so it
+ * no longer costs the errors sent in the same beacon. Events past
+ * MAX_BROWSER_EVENTS are dropped and counted unread.
+ */
+export function parseBrowserPayload(body) {
+    const envelope = browserEnvelopeSchema.safeParse(body);
+    if (!envelope.success)
+        return { success: false, issues: envelope.error.issues.slice(0, 5) };
+    const raw = envelope.data.events;
+    const considered = raw.slice(0, MAX_BROWSER_EVENTS);
+    const events = [];
+    const issues = [];
+    let invalidEvents = 0;
+    considered.forEach((candidate, index) => {
+        const parsed = browserEventSchema.safeParse(candidate);
+        if (parsed.success) {
+            events.push(parsed.data);
+            return;
+        }
+        invalidEvents += 1;
+        for (const issue of parsed.error.issues) {
+            if (issues.length >= 5)
+                break;
+            issues.push({ ...issue, path: ["events", index, ...issue.path] });
+        }
+    });
+    return {
+        success: true,
+        data: {
+            payload: { ...envelope.data, events },
+            invalidEvents,
+            overLimitEvents: raw.length - considered.length,
+            issues,
+        },
+    };
+}
 const TYPE_TO_ERROR_TYPE = {
     exception: "Error",
     unhandled_rejection: "UnhandledRejection",
@@ -154,14 +201,13 @@ export function normalizeBrowserPayload(payload) {
             continue;
         }
         const severity = asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error");
-        // The browser has no request stream, so error/message events are the
-        // only usage signal for many pages — count each as an event (and, for
-        // fatal/error severity, an error event). Without this, a page that
-        // only reports errors groups issues whose dashboards all read 0.
-        bumpRollup(event.route ? (event.route.split("?")[0] ?? "") : "", occurredAt, {
-            requestCount: 1,
-            errorCount: severity === "fatal" || severity === "error" ? 1 : 0,
-        });
+        // A browser error belongs to no server request: it counts in
+        // `error_count` only (fatal/error severity), never `request_count`,
+        // so a page that only reports errors still shows them without
+        // inventing requests. Warnings and info messages count nothing.
+        if (severity === "fatal" || severity === "error") {
+            bumpRollup(event.route ? (event.route.split("?")[0] ?? "") : "", occurredAt, { errorCount: 1 });
+        }
         occurrences.push({
             source: "browser",
             severity,

package/dist/normalize-otlp.d.ts

@@ -78,7 +78,13 @@ export interface OtlpMetricsRequest {
     }>;
 }
 export interface NormalizedTraces {
+    /** One per thrown error: duplicate recordings across a trace's spans are
+     * merged and redundant status-only occurrences dropped (trace-dedupe.ts). */
     occurrences: RuntimeOccurrenceInput[];
+    /** Each occurrence's position among every signal the batch recorded,
+     * before deduplication: the batch index its deterministic id is derived
+     * from, so a merge or a drop never shifts another occurrence's id. */
+    occurrenceBatchIndexes: number[];
     spans: RuntimeSpanRow[];
     metricPoints: RuntimeMetricPoint[];
     llmCalls: RuntimeLlmCall[];