@autter/otlp-ingester 1.3.1 → 1.3.3

package/README.md

@@ -1,6 +1,6 @@
 # @autter/otlp-ingester
 
-Self-hostable ingest service for Autter Runtime. Receives OTLP/HTTP (JSON)
+Self-hostable ingest service for Autter Runtime. Receives OTLP/HTTP (protobuf or JSON)
 traces and metrics plus compact browser error payloads, normalises them into
 one per-repo signal model, fingerprints errors, and writes ClickHouse.
 
@@ -9,7 +9,9 @@ one per-repo signal model, fingerprints errors, and writes ClickHouse.
 | Route | Payload | Purpose |
 | --- | --- | --- |
 | `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups; GenAI spans → `runtime_llm_calls` |
-| `POST /v1/metrics` | OTLP/JSON `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups |
+| `POST /v1/metrics` | OTLP `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups; portable process memory/GC metrics → per-instance memory samples |
+| `POST /v1/platform-events` | JSON, server key | ECS/Kubernetes OOM kills and restarts → memory incident correlation |
+| `POST /v1/profiles` | Symbolized pprof, server key | CPU/in-use heap profile samples |
 | `POST /v1/browser` | Browser payload `version: 1` | Errors/rejections → occurrences; session pings → rollups |
 | `GET /healthz` | — | Liveness + ClickHouse reachability |
 

package/dist/clickhouse.d.ts

@@ -1,5 +1,7 @@
 import type { IngesterConfig } from "./config.js";
 import { type LatencyHistogram } from "./latency.js";
+import { type ProfileSample } from "./profiles.js";
+import { type MemorySample, type PlatformEvent } from "./memory.js";
 import { type Migration } from "./migrations.js";
 import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
 export declare class ClickHouseStore {
@@ -23,6 +25,14 @@ export declare class ClickHouseStore {
     ping(): Promise<boolean>;
     insertOccurrences(ctx: IngestContext, occurrences: RuntimeOccurrence[]): Promise<void>;
     insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
+    insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise<void>;
+    insertMemorySamples(ctx: IngestContext, samples: MemorySample[]): Promise<void>;
+    insertPlatformEvent(ctx: IngestContext, event: PlatformEvent): Promise<void>;
+    insertSourceMap(ctx: IngestContext, sourceMap: {
+        release: string;
+        filename: string;
+        map: string;
+    }): Promise<void>;
     insertLlmCalls(ctx: IngestContext, calls: RuntimeLlmCall[]): Promise<void>;
     insertMetricPoints(ctx: IngestContext, points: RuntimeMetricPoint[]): Promise<void>;
     insertLatencyHistograms(ctx: IngestContext, points: LatencyHistogram[]): Promise<void>;

package/dist/clickhouse.js

@@ -1,5 +1,8 @@
 import { createClient, } from "@clickhouse/client";
 import { latencyTableDDL } from "./latency.js";
+import { profileTableDDL } from "./profiles.js";
+import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
+import { sourceMapTableDDL } from "./source-maps.js";
 import { MIGRATIONS, migrationsTableDDL, } from "./migrations.js";
 /**
  * ClickHouse persistence. The schema is idempotent and applied lazily once
@@ -46,6 +49,10 @@ export class ClickHouseStore {
         return [
             `CREATE DATABASE IF NOT EXISTS ${db}`,
             latencyTableDDL(db, metricsTtlDays),
+            profileTableDDL(db),
+            memoryTableDDL(db),
+            platformEventTableDDL(db),
+            sourceMapTableDDL(db),
             `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
 				org_id             String,
 				repository_id      String,
@@ -258,6 +265,57 @@ export class ClickHouseStore {
             })),
         });
     }
+    async insertProfileSamples(ctx, samples) {
+        if (!samples.length)
+            return;
+        if (!this.configured)
+            throw new Error("CLICKHOUSE_URL is not configured");
+        await this.ensureSchema();
+        await this.getClient().insert({
+            table: this.table("runtime_profile_samples"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: samples.map((sample) => ({
+                org_id: ctx.orgId, repository_id: ctx.repositoryId,
+                profile_id: sample.profileId, sample_index: sample.sampleIndex, service: sample.service,
+                environment: sample.environment, release: sample.release,
+                trace_id: sample.traceId, instance_id: sample.instanceId, observed_at: sample.observedAt.toISOString(),
+                sample_type: sample.sampleType, unit: sample.unit,
+                stack: sample.stack, value: sample.value,
+            })),
+        });
+    }
+    async insertMemorySamples(ctx, samples) {
+        if (!samples.length)
+            return;
+        if (!this.configured)
+            throw new Error("CLICKHOUSE_URL is not configured");
+        await this.ensureSchema();
+        await this.getClient().insert({ table: this.table("runtime_memory_samples"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: samples.map((s) => ({ org_id: ctx.orgId, repository_id: ctx.repositoryId,
+                service: s.service, environment: s.environment, release: s.release,
+                instance_id: s.instanceId, metric: s.metric, value: s.value, temporality: s.temporality,
+                observed_at: s.observedAt.toISOString() })) });
+    }
+    async insertPlatformEvent(ctx, event) {
+        if (!this.configured)
+            throw new Error("CLICKHOUSE_URL is not configured");
+        await this.ensureSchema();
+        await this.getClient().insert({ table: this.table("runtime_platform_events"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId, event_id: event.eventId,
+                    service: event.service, environment: event.environment, release: event.release,
+                    instance_id: event.instanceId, platform: event.platform, kind: event.kind,
+                    occurred_at: event.occurredAt }] });
+    }
+    async insertSourceMap(ctx, sourceMap) {
+        await this.ensureSchema();
+        await this.getClient().insert({
+            table: this.table("runtime_source_maps"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId, ...sourceMap }],
+        });
+    }
     async insertLlmCalls(ctx, calls) {
         if (calls.length === 0 || !this.configured)
             return;

package/dist/memory.d.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+import type { OtlpMetricsRequest } from "./normalize-otlp.js";
+export type MemoryMetric = "rss" | "heap_used" | "heap_limit" | "memory_limit" | "heap_after_gc" | "gc_count" | "gc_duration_ms";
+export interface MemorySample {
+    service: string;
+    environment: string;
+    release: string;
+    instanceId: string;
+    metric: MemoryMetric;
+    value: number;
+    observedAt: Date;
+    temporality: "gauge" | "delta" | "cumulative";
+}
+export declare function normalizeMemoryMetrics(request: OtlpMetricsRequest): MemorySample[];
+export declare const platformEventSchema: z.ZodObject<{
+    service: z.ZodString;
+    environment: z.ZodDefault<z.ZodString>;
+    release: z.ZodDefault<z.ZodString>;
+    instanceId: z.ZodString;
+    platform: z.ZodEnum<["ecs", "kubernetes", "other"]>;
+    kind: z.ZodEnum<["oom_kill", "restart"]>;
+    occurredAt: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    service: string;
+    environment: string;
+    release: string;
+    instanceId: string;
+    platform: "ecs" | "kubernetes" | "other";
+    kind: "oom_kill" | "restart";
+    occurredAt: string;
+}, {
+    service: string;
+    instanceId: string;
+    platform: "ecs" | "kubernetes" | "other";
+    kind: "oom_kill" | "restart";
+    occurredAt: string;
+    environment?: string | undefined;
+    release?: string | undefined;
+}>;
+export type PlatformEvent = z.infer<typeof platformEventSchema> & {
+    eventId: string;
+};
+export declare function normalizePlatformEvent(input: z.infer<typeof platformEventSchema>): PlatformEvent | null;
+export declare function memoryTableDDL(db: string): string;
+export declare function platformEventTableDDL(db: string): string;

package/dist/memory.js

@@ -0,0 +1,114 @@
+import { createHash } from "node:crypto";
+import { z } from "zod";
+function attrsOf(attributes) {
+    return new Map((attributes ?? []).flatMap((a) => {
+        const v = a.value?.stringValue ?? a.value?.intValue ?? a.value?.doubleValue;
+        return a.key && v !== undefined ? [[a.key, String(v)]] : [];
+    }));
+}
+function metricOf(name, attrs) {
+    switch (name) {
+        case "autter.process.memory.rss":
+        case "process.runtime.nodejs.memory.rss": return "rss";
+        case "autter.process.memory.heap.used":
+        case "process.runtime.nodejs.memory.heap.used": return "heap_used";
+        case "autter.process.memory.heap.limit":
+        case "process.runtime.nodejs.memory.heap.limit": return "heap_limit";
+        case "autter.process.memory.limit":
+        case "container.memory.limit": return "memory_limit";
+        case "autter.process.memory.heap.after_gc": return "heap_after_gc";
+        case "autter.process.gc.count":
+        case "process.runtime.nodejs.gc.count": return "gc_count";
+        case "autter.process.gc.duration":
+        case "process.runtime.nodejs.gc.duration": return "gc_duration_ms";
+        case "process.memory.usage": {
+            const type = attrs.get("type") ?? attrs.get("state");
+            return type === "rss" || type === "physical" ? "rss" : type === "heap_used" ? "heap_used" : null;
+        }
+        default: return null;
+    }
+}
+export function normalizeMemoryMetrics(request) {
+    const out = [];
+    const now = Date.now();
+    for (const rm of request.resourceMetrics ?? []) {
+        const resource = attrsOf(rm.resource?.attributes);
+        const instanceId = resource.get("service.instance.id")?.slice(0, 128) ?? "";
+        const service = resource.get("service.name")?.slice(0, 200) ?? "";
+        if (!instanceId || !service)
+            continue;
+        const environment = (resource.get("deployment.environment.name") ?? resource.get("deployment.environment") ?? "production").slice(0, 100);
+        const release = (resource.get("service.version") ?? "").slice(0, 200);
+        for (const scope of rm.scopeMetrics ?? [])
+            for (const metric of scope.metrics ?? []) {
+                const name = metric.name ?? "";
+                const points = metric.gauge?.dataPoints ?? metric.sum?.dataPoints ?? [];
+                for (const point of points) {
+                    const kind = metricOf(name, attrsOf(point.attributes));
+                    if (!kind)
+                        continue;
+                    const counter = kind === "gc_count" || kind === "gc_duration_ms";
+                    const aggregation = metric.sum?.aggregationTemporality;
+                    const temporality = counter && (aggregation === 2 || aggregation === "AGGREGATION_TEMPORALITY_CUMULATIVE")
+                        ? "cumulative" : counter ? "delta" : "gauge";
+                    const value = Number(point.asDouble ?? point.asInt);
+                    let observedMs = 0;
+                    try {
+                        observedMs = Number(BigInt(String(point.timeUnixNano ?? "0")) / 1000000n);
+                    }
+                    catch {
+                        continue;
+                    }
+                    if (!Number.isFinite(value) || value < 0 || value > 1e15 || observedMs < now - 30 * 86400_000 || observedMs > now + 300_000)
+                        continue;
+                    const isBytes = kind === "rss" || kind === "heap_used" || kind === "heap_limit" || kind === "memory_limit" || kind === "heap_after_gc";
+                    const unit = metric.unit ?? "By";
+                    if (isBytes && !["By", "byte", "bytes", "KiBy", "MiBy"].includes(unit))
+                        continue;
+                    // Standard OTel runtimes report GC duration in seconds; Autter uses ms.
+                    out.push({ service, environment, release, instanceId, metric: kind,
+                        value: isBytes ? value * (unit === "KiBy" ? 1024 : unit === "MiBy" ? 1048576 : 1)
+                            : kind === "gc_duration_ms" && unit === "s" ? value * 1000 : value,
+                        observedAt: new Date(observedMs), temporality });
+                    if (out.length >= 5000)
+                        return out;
+                }
+            }
+    }
+    return out;
+}
+export const platformEventSchema = z.object({
+    service: z.string().regex(/^[a-zA-Z0-9._-]{1,200}$/),
+    environment: z.string().min(1).max(100).default("production"),
+    release: z.string().max(200).default(""),
+    instanceId: z.string().min(1).max(128),
+    platform: z.enum(["ecs", "kubernetes", "other"]),
+    kind: z.enum(["oom_kill", "restart"]),
+    occurredAt: z.string().datetime(),
+});
+export function normalizePlatformEvent(input) {
+    const time = Date.parse(input.occurredAt);
+    if (time < Date.now() - 30 * 86400_000 || time > Date.now() + 300_000)
+        return null;
+    const eventId = createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 32);
+    return { ...input, eventId };
+}
+export function memoryTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_memory_samples (
+		org_id String, repository_id String, service LowCardinality(String),
+		environment LowCardinality(String), release String DEFAULT '', instance_id String,
+		metric LowCardinality(String), value Float64, temporality LowCardinality(String) DEFAULT 'gauge',
+		observed_at DateTime64(3, 'UTC')
+	) ENGINE = ReplacingMergeTree PARTITION BY toDate(observed_at)
+	ORDER BY (org_id, repository_id, service, environment, instance_id, metric, observed_at)
+	TTL toDateTime(observed_at) + INTERVAL 14 DAY`;
+}
+export function platformEventTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_platform_events (
+		org_id String, repository_id String, event_id String, service LowCardinality(String),
+		environment LowCardinality(String), release String DEFAULT '', instance_id String,
+		platform LowCardinality(String), kind LowCardinality(String), occurred_at DateTime64(3, 'UTC')
+	) ENGINE = ReplacingMergeTree PARTITION BY toDate(occurred_at)
+	ORDER BY (org_id, repository_id, event_id)
+	TTL toDateTime(occurred_at) + INTERVAL 30 DAY`;
+}

package/dist/migrations.js

@@ -33,6 +33,9 @@
  *   },
  */
 import { latencyTableDDL } from "./latency.js";
+import { profileTableDDL } from "./profiles.js";
+import { sourceMapTableDDL } from "./source-maps.js";
+import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -115,6 +118,15 @@ export const MIGRATIONS = [
         ],
     },
     { id: "0005-latency-histograms", statements: [latencyTableDDL("{db}")] },
+    { id: "0006-profile-samples", statements: [profileTableDDL("{db}")] },
+    { id: "0007-source-maps", statements: [sourceMapTableDDL("{db}")] },
+    { id: "0008-memory-signals", statements: [memoryTableDDL("{db}"), platformEventTableDDL("{db}")] },
+    { id: "0009-profile-instance", statements: [
+            `ALTER TABLE {db}.runtime_profile_samples ADD COLUMN IF NOT EXISTS instance_id String DEFAULT ''`,
+        ] },
+    { id: "0010-memory-temporality", statements: [
+            `ALTER TABLE {db}.runtime_memory_samples ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'gauge'`,
+        ] },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -8,13 +8,14 @@ export declare const browserPayloadSchema: z.ZodObject<{
     release: z.ZodOptional<z.ZodString>;
     events: z.ZodArray<z.ZodObject<{
         id: z.ZodOptional<z.ZodString>;
-        type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event"]>;
+        type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event", "outcome", "request_failure", "timing"]>;
         timestamp: z.ZodString;
         /** exception/message events: signal level. Defaults per type. */
         severity: z.ZodOptional<z.ZodEnum<["fatal", "error", "warning", "info"]>>;
         message: z.ZodDefault<z.ZodString>;
         /** track_event only: the event name (counted, never free-form PII). */
         name: z.ZodOptional<z.ZodString>;
+        durationMs: z.ZodOptional<z.ZodNumber>;
         stack: z.ZodOptional<z.ZodString>;
         errorType: z.ZodOptional<z.ZodString>;
         filename: z.ZodOptional<z.ZodString>;
@@ -24,29 +25,31 @@ export declare const browserPayloadSchema: z.ZodObject<{
         route: z.ZodOptional<z.ZodString>;
         context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     }, "strip", z.ZodTypeAny, {
-        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         message: string;
         timestamp: string;
+        durationMs?: number | undefined;
+        filename?: string | undefined;
         id?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;
         errorType?: string | undefined;
-        filename?: string | undefined;
         line?: number | undefined;
         column?: number | undefined;
         route?: string | undefined;
         context?: Record<string, unknown> | undefined;
     }, {
-        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         timestamp: string;
-        id?: string | undefined;
+        durationMs?: number | undefined;
         message?: string | undefined;
+        filename?: string | undefined;
+        id?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;
         errorType?: string | undefined;
-        filename?: string | undefined;
         line?: number | undefined;
         column?: number | undefined;
         route?: string | undefined;
@@ -57,15 +60,16 @@ export declare const browserPayloadSchema: z.ZodObject<{
     environment: string;
     version: 1;
     events: {
-        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         message: string;
         timestamp: string;
+        durationMs?: number | undefined;
+        filename?: string | undefined;
         id?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;
         errorType?: string | undefined;
-        filename?: string | undefined;
         line?: number | undefined;
         column?: number | undefined;
         route?: string | undefined;
@@ -78,15 +82,16 @@ export declare const browserPayloadSchema: z.ZodObject<{
     environment: string;
     version: 1;
     events: {
-        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         timestamp: string;
-        id?: string | undefined;
+        durationMs?: number | undefined;
         message?: string | undefined;
+        filename?: string | undefined;
+        id?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;
         errorType?: string | undefined;
-        filename?: string | undefined;
         line?: number | undefined;
         column?: number | undefined;
         route?: string | undefined;

package/dist/normalize-browser.js

@@ -1,4 +1,5 @@
 import { z } from "zod";
+import { normalizeRoute } from "./fingerprint.js";
 import { asSeverity, } from "./types.js";
 /**
  * The compact browser payload (`version: 1`) emitted by
@@ -14,6 +15,9 @@ const browserEventSchema = z.object({
         "message",
         "session_start",
         "track_event",
+        "outcome",
+        "request_failure",
+        "timing",
     ]),
     timestamp: z.string().datetime(),
     /** exception/message events: signal level. Defaults per type. */
@@ -21,6 +25,7 @@ const browserEventSchema = z.object({
     message: z.string().max(4000).default(""),
     /** track_event only: the event name (counted, never free-form PII). */
     name: z.string().max(200).optional(),
+    durationMs: z.number().nonnegative().max(120000).optional(),
     stack: z.string().max(32000).optional(),
     errorType: z.string().max(200).optional(),
     filename: z.string().max(1000).optional(),
@@ -42,6 +47,8 @@ const TYPE_TO_ERROR_TYPE = {
     exception: "Error",
     unhandled_rejection: "UnhandledRejection",
     message: "Message",
+    outcome: "OutcomeFailure",
+    request_failure: "HttpRequestError",
 };
 // Content-level gate for the free-form `context` bag. The schema whitelist
 // above is structural; this masks obvious PII/secrets inside whatever a
@@ -69,6 +76,8 @@ const TYPE_TO_SEVERITY = {
     exception: "error",
     unhandled_rejection: "error",
     message: "warning",
+    outcome: "error",
+    request_failure: "error",
 };
 export function normalizeBrowserPayload(payload) {
     const occurrences = [];
@@ -109,6 +118,15 @@ export function normalizeBrowserPayload(payload) {
                 bumpRollup(`event:${name}`, occurredAt, { requestCount: 1 });
             continue;
         }
+        if (event.type === "timing") {
+            const name = event.name === "browser.longtask" ? "browser.longtask"
+                : `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0].replace(EMAIL_VALUE_RE, REDACTED))}`;
+            bumpRollup(name, occurredAt, { requestCount: 1 });
+            const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`);
+            if (point)
+                point.durationSumMs += event.durationMs ?? 0;
+            continue;
+        }
         const severity = asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error");
         // The browser has no request stream, so error/message events are the
         // only usage signal for many pages — count each as an event (and, for
@@ -125,7 +143,9 @@ export function normalizeBrowserPayload(payload) {
             environment: payload.environment,
             release: payload.release ?? null,
             errorType: event.errorType ?? TYPE_TO_ERROR_TYPE[event.type] ?? "Error",
-            message: event.message || "Unknown error",
+            message: event.type === "outcome" ? `${(event.name ?? "outcome").replace(EMAIL_VALUE_RE, REDACTED)}: ${event.message.replace(EMAIL_VALUE_RE, REDACTED)}`
+                : event.type === "request_failure" ? `${normalizeRoute((event.name ?? "request").split("?")[0].replace(EMAIL_VALUE_RE, REDACTED))}: ${event.message}`
+                    : event.message || "Unknown error",
             stack: event.stack ?? null,
             route: event.route ? (event.route.split("?")[0] ?? null) : null,
             method: null,

package/dist/normalize-otlp.d.ts

@@ -66,12 +66,16 @@ export interface OtlpMetricsRequest {
             metrics?: Array<{
                 name?: string;
                 unit?: string;
+                gauge?: {
+                    dataPoints?: OtlpDataPoint[];
+                };
                 histogram?: {
                     dataPoints?: OtlpDataPoint[];
                     aggregationTemporality?: string | number;
                 };
                 sum?: {
                     dataPoints?: OtlpDataPoint[];
+                    aggregationTemporality?: string | number;
                 };
             }>;
         }>;

package/dist/normalize-otlp.js

@@ -1,6 +1,7 @@
 import { normalizeRoute } from "./fingerprint.js";
 import { extractLlmCall } from "./llm.js";
 import { asSeverity, } from "./types.js";
+const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
 function attrMap(attributes) {
     const map = new Map();
     for (const attr of attributes ?? []) {
@@ -112,8 +113,11 @@ export function normalizeTraces(request) {
                 const startedAt = nanosToDate(span.startTimeUnixNano);
                 const durationMs = spanDurationMs(span);
                 const kind = spanKind(span.kind);
-                const isError = isErrorStatus(span.status?.code) ||
-                    (statusCode !== null && statusCode >= 500);
+                const hasFailureEvent = (span.events ?? []).some((event) => event.name === "exception" || (event.name === "autter.outcome"
+                    && attrMap(event.attributes).get("autter.outcome.status") === "error"));
+                const isError = statusCode !== null && statusCode >= 400 && statusCode < 500
+                    ? hasFailureEvent
+                    : hasFailureEvent || isErrorStatus(span.status?.code) || (statusCode !== null && statusCode >= 500);
                 spans.push({
                     service: resource.service,
                     environment: resource.environment,
@@ -138,6 +142,7 @@ export function normalizeTraces(request) {
                 const exceptionEvents = (span.events ?? []).filter((event) => event.name === "exception");
                 for (const event of exceptionEvents) {
                     const eventAttrs = attrMap(event.attributes);
+                    const handled = eventAttrs.get("autter.handled") === "true";
                     occurrences.push({
                         source: "server",
                         severity: severityOf(eventAttrs, attrs),
@@ -155,19 +160,42 @@ export function normalizeTraces(request) {
                         statusCode,
                         traceId: span.traceId ?? null,
                         sessionId: null,
-                        attributes: null,
+                        attributes: handled ? {
+                            "autter.handled": true,
+                            "autter.sampled": eventAttrs.get("autter.sampled") === "true",
+                        } : null,
+                        occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
+                    });
+                }
+                const failedOutcomes = (span.events ?? []).filter((event) => {
+                    if (event.name !== "autter.outcome")
+                        return false;
+                    return attrMap(event.attributes).get("autter.outcome.status") === "error";
+                });
+                for (const event of failedOutcomes) {
+                    const outcome = attrMap(event.attributes);
+                    const name = (outcome.get("autter.outcome.name") ?? "operation").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 200);
+                    occurrences.push({
+                        source: "server", severity: "error", service: resource.service,
+                        environment: resource.environment, release: resource.release,
+                        errorType: "OutcomeFailure",
+                        message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`,
+                        stack: null, route, method: methodOf(attrs), statusCode,
+                        traceId: span.traceId ?? null, sessionId: null, attributes: null,
                         occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
                     });
                 }
-                if (exceptionEvents.length === 0 && isErrorStatus(span.status?.code)) {
+                if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) {
                     occurrences.push({
                         source: "server",
                         severity: severityOf(new Map(), attrs),
                         service: resource.service,
                         environment: resource.environment,
                         release: resource.release,
-                        errorType: "SpanError",
-                        message: span.status?.message || `${span.name ?? "span"} failed`,
+                        errorType: statusCode !== null && statusCode >= 500 ? "HttpServerError" : "SpanError",
+                        message: span.status?.message || (statusCode !== null && statusCode >= 500
+                            ? `${methodOf(attrs) ?? "HTTP"} ${route ?? span.name ?? "request"} returned ${statusCode}`
+                            : `${span.name ?? "span"} failed`),
                         stack: null,
                         route,
                         method: methodOf(attrs),

package/dist/otlp-proto.js

@@ -54,10 +54,12 @@ message ScopeMetrics { repeated Metric metrics = 2; }
 message Metric {
   string name = 1;
   string unit = 3;
+  Gauge gauge = 5;
   Sum sum = 7;
   Histogram histogram = 9;
 }
-message Sum { repeated NumberDataPoint data_points = 1; }
+message Gauge { repeated NumberDataPoint data_points = 1; }
+message Sum { repeated NumberDataPoint data_points = 1; int32 aggregation_temporality = 2; }
 message NumberDataPoint {
   fixed64 time_unix_nano = 3;
   double as_double = 4;

package/dist/profiles.d.ts

@@ -0,0 +1,22 @@
+export interface ProfileSample {
+    profileId: string;
+    sampleIndex: number;
+    service: string;
+    environment: string;
+    release: string;
+    traceId: string;
+    instanceId: string;
+    observedAt: Date;
+    sampleType: string;
+    unit: string;
+    stack: string[];
+    value: number;
+}
+export declare function profileTableDDL(db: string): string;
+export declare function decodeProfile(body: Buffer, meta: {
+    service: string;
+    environment: string;
+    release: string;
+    traceId: string;
+    instanceId: string;
+}): ProfileSample[];

package/dist/profiles.js

@@ -0,0 +1,63 @@
+import { createHash } from "node:crypto";
+import { gunzipSync } from "node:zlib";
+import protobuf from "protobufjs";
+// Only the pprof fields needed for a bounded, symbolized CPU sample index.
+const schema = protobuf.parse(`syntax = "proto3";
+message Profile {
+  repeated ValueType sample_type = 1;
+  repeated Sample sample = 2;
+  repeated Location location = 4;
+  repeated Function function = 5;
+  repeated string string_table = 6;
+  int64 time_nanos = 9;
+}
+message ValueType { int64 type = 1; int64 unit = 2; }
+message Sample { repeated uint64 location_id = 1 [packed=true]; repeated int64 value = 2 [packed=true]; }
+message Location { uint64 id = 1; repeated Line line = 4; }
+message Line { uint64 function_id = 1; int64 line = 2; }
+message Function { uint64 id = 1; int64 name = 2; int64 system_name = 3; int64 filename = 4; }
+`).root.lookupType("Profile");
+export function profileTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_profile_samples (
+		org_id String, repository_id String, profile_id String, sample_index UInt16,
+		service LowCardinality(String), environment LowCardinality(String), release String,
+		trace_id String DEFAULT '', instance_id String DEFAULT '', observed_at DateTime64(3, 'UTC'),
+		sample_type LowCardinality(String), unit LowCardinality(String),
+		stack Array(String), value UInt64
+	) ENGINE = MergeTree PARTITION BY toDate(observed_at)
+	ORDER BY (org_id, repository_id, service, environment, release, observed_at, profile_id)
+	TTL toDateTime(observed_at) + INTERVAL 7 DAY`;
+}
+export function decodeProfile(body, meta) {
+    if (body.length === 0 || body.length > 1024 * 1024)
+        throw new Error("invalid profile size");
+    const payload = body[0] === 0x1f && body[1] === 0x8b
+        ? gunzipSync(body, { maxOutputLength: 1024 * 1024 }) : body;
+    const profile = schema.toObject(schema.decode(payload), { longs: String, arrays: true });
+    const strings = profile.stringTable ?? [];
+    const locations = new Map((profile.location ?? []).map((location) => [location.id, location]));
+    const functions = new Map((profile.function ?? []).map((fn) => [fn.id, fn]));
+    const typeIndex = Math.max(0, (profile.sampleType ?? []).findIndex((t) => strings[Number(t.type ?? 0)] === "inuse_space"));
+    const type = profile.sampleType?.[typeIndex];
+    const observedMs = Number(BigInt(profile.timeNanos ?? "0") / 1000000n);
+    const observedAt = observedMs > 0 && observedMs <= Date.now() + 300_000 ? new Date(observedMs) : new Date();
+    const profileId = createHash("sha256").update(body).digest("hex").slice(0, 32);
+    const samples = [];
+    for (const [sampleIndex, sample] of (profile.sample ?? []).slice(0, 1000).entries()) {
+        const stack = (sample.locationId ?? []).slice(0, 64).flatMap((id) => {
+            const locLine = locations.get(id)?.line?.[0];
+            const fn = locLine?.functionId ? functions.get(locLine.functionId) : undefined;
+            const name = fn ? strings[Number(fn.name ?? 0)] : undefined;
+            const filename = fn ? strings[Number(fn.filename ?? 0)] : undefined;
+            return name ? [`${name.slice(0, 150)}${filename ? ` (${filename.slice(0, 300)}:${Number(locLine?.line ?? 0)})` : ""}`] : [];
+        });
+        const value = Number(sample.value?.[typeIndex] ?? 0);
+        if (!stack.length || !Number.isSafeInteger(value) || value <= 0)
+            continue;
+        samples.push({ profileId, sampleIndex, ...meta, observedAt,
+            sampleType: (strings[Number(type?.type ?? 0)] ?? "samples").slice(0, 80),
+            unit: (strings[Number(type?.unit ?? 0)] ?? "count").slice(0, 40),
+            stack, value });
+    }
+    return samples;
+}

package/dist/server.js

@@ -6,6 +6,9 @@ import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerp
 import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
 import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
+import { decodeProfile } from "./profiles.js";
+import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js";
+import { validateSourceMap } from "./source-maps.js";
 import { SinkForwarder } from "./sink.js";
 export function createIngesterApp(config) {
     const store = new ClickHouseStore(config);
@@ -18,6 +21,8 @@ export function createIngesterApp(config) {
     const clientRateLimiter = new RateLimiter(config.clientRateLimitPerMinute);
     const app = express();
     app.disable("x-powered-by");
+    app.use(express.raw({ limit: "1mb", type: ["application/x-pprof"] }));
+    app.use("/v1/sourcemaps", express.json({ limit: "5mb" }));
     app.use(express.json({
         limit: config.maxBodyBytes,
         type: ["application/json"],
@@ -140,6 +145,81 @@ export function createIngesterApp(config) {
         }
         res.status(200).json({ partialSuccess: {} });
     }
+    app.post("/v1/profiles", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        if (!req.is("application/x-pprof") || !Buffer.isBuffer(req.body)) {
+            res.status(415).json({ error: "expected application/x-pprof" });
+            return;
+        }
+        const service = req.header("x-autter-service")?.trim() ?? "";
+        const environment = req.header("x-autter-environment")?.trim() || "production";
+        const release = req.header("x-autter-release")?.trim() ?? "";
+        const traceId = req.header("x-autter-trace-id")?.trim() ?? "";
+        const instanceId = req.header("x-autter-instance-id")?.trim() ?? "";
+        if (!/^[a-zA-Z0-9._-]{1,200}$/.test(service) || environment.length > 100 || release.length > 200 || instanceId.length > 128 || (traceId && !/^[a-f0-9]{32}$/.test(traceId))) {
+            res.status(400).json({ error: "invalid profile metadata" });
+            return;
+        }
+        let samples;
+        try {
+            samples = decodeProfile(req.body, { service, environment, release, traceId, instanceId });
+            if (!samples.length)
+                throw new Error("empty profile");
+        }
+        catch {
+            res.status(400).json({ error: "invalid or unsymbolized profile" });
+            return;
+        }
+        try {
+            await store.insertProfileSamples(ctx, samples);
+            res.status(202).json({ profileId: samples[0].profileId, samples: samples.length });
+        }
+        catch (err) {
+            storageError(res, err);
+        }
+    });
+    app.post("/v1/sourcemaps", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        const sourceMap = validateSourceMap(req.body);
+        if (!sourceMap) {
+            res.status(400).json({ error: "invalid source map" });
+            return;
+        }
+        try {
+            await store.insertSourceMap(ctx, sourceMap);
+            res.status(202).json({ accepted: true });
+        }
+        catch (err) {
+            storageError(res, err);
+        }
+    });
+    /** ECS/Kubernetes event forwarders use the same server key as OTLP. */
+    app.post("/v1/platform-events", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        const parsed = platformEventSchema.safeParse(req.body);
+        if (!parsed.success) {
+            res.status(400).json({ error: "invalid platform event" });
+            return;
+        }
+        const event = normalizePlatformEvent(parsed.data);
+        if (!event) {
+            res.status(400).json({ error: "event timestamp outside retention window" });
+            return;
+        }
+        try {
+            await store.insertPlatformEvent(ctx, event);
+            res.status(202).json({ accepted: true });
+        }
+        catch (err) {
+            storageError(res, err);
+        }
+    });
     app.post("/v1/traces", async (req, res) => {
         const ctx = await authenticate(req, res, "otlp");
         if (!ctx)
@@ -200,9 +280,11 @@ export function createIngesterApp(config) {
             request = req.body;
         }
         const metricPoints = normalizeMetrics(request);
+        const memorySamples = normalizeMemoryMetrics(request);
         try {
             await store.insertLatencyHistograms(ctx, normalizeLatencyHistograms(request));
             await store.insertMetricPoints(ctx, metricPoints);
+            await store.insertMemorySamples(ctx, memorySamples);
         }
         catch (err) {
             storageError(res, err);

package/dist/source-maps.d.ts

@@ -0,0 +1,6 @@
+export declare function sourceMapTableDDL(db: string): string;
+export declare function validateSourceMap(body: unknown): {
+    release: string;
+    filename: string;
+    map: string;
+} | null;

package/dist/source-maps.js

@@ -0,0 +1,41 @@
+export function sourceMapTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_source_maps (
+		org_id String, repository_id String, release String, filename String,
+		map String CODEC(ZSTD(3)), uploaded_at DateTime64(3, 'UTC') DEFAULT now64(3)
+	) ENGINE = ReplacingMergeTree(uploaded_at)
+	ORDER BY (org_id, repository_id, release, filename)
+	TTL toDateTime(uploaded_at) + INTERVAL 30 DAY`;
+}
+export function validateSourceMap(body) {
+    if (!body || typeof body !== "object")
+        return null;
+    const input = body;
+    if (typeof input.release !== "string" || !/^[a-zA-Z0-9._-]{1,200}$/.test(input.release))
+        return null;
+    if (typeof input.filename !== "string" || input.filename.length > 1000)
+        return null;
+    let filename;
+    try {
+        filename = new URL(input.filename, "https://autter.invalid").pathname;
+    }
+    catch {
+        return null;
+    }
+    if (!filename.endsWith(".js"))
+        return null;
+    let parsed;
+    try {
+        parsed = typeof input.map === "string" ? JSON.parse(input.map) : input.map;
+    }
+    catch {
+        return null;
+    }
+    if (!parsed || parsed.version !== 3 || typeof parsed.mappings !== "string" || !Array.isArray(parsed.sources))
+        return null;
+    // Source contents are unnecessary for position lookup and may contain secrets.
+    const { sourcesContent: _omitted, ...safe } = parsed;
+    const map = JSON.stringify(safe);
+    if (map.length > 5 * 1024 * 1024)
+        return null;
+    return { release: input.release, filename, map };
+}

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.3.1",
+	"version": "1.3.3",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",