@autter/otlp-ingester 1.3.2 → 1.3.3

package/README.md

@@ -1,6 +1,6 @@
 # @autter/otlp-ingester
 
-Self-hostable ingest service for Autter Runtime. Receives OTLP/HTTP (JSON)
+Self-hostable ingest service for Autter Runtime. Receives OTLP/HTTP (protobuf or JSON)
 traces and metrics plus compact browser error payloads, normalises them into
 one per-repo signal model, fingerprints errors, and writes ClickHouse.
 
@@ -9,7 +9,9 @@ one per-repo signal model, fingerprints errors, and writes ClickHouse.
 | Route | Payload | Purpose |
 | --- | --- | --- |
 | `POST /v1/traces` | OTLP/JSON `ExportTraceServiceRequest` | Error spans → occurrences; all spans → `runtime_spans`; server spans → usage rollups; GenAI spans → `runtime_llm_calls` |
-| `POST /v1/metrics` | OTLP/JSON `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups |
+| `POST /v1/metrics` | OTLP `ExportMetricsServiceRequest` | HTTP-server duration histograms → usage rollups; portable process memory/GC metrics → per-instance memory samples |
+| `POST /v1/platform-events` | JSON, server key | ECS/Kubernetes OOM kills and restarts → memory incident correlation |
+| `POST /v1/profiles` | Symbolized pprof, server key | CPU/in-use heap profile samples |
 | `POST /v1/browser` | Browser payload `version: 1` | Errors/rejections → occurrences; session pings → rollups |
 | `GET /healthz` | — | Liveness + ClickHouse reachability |
 

package/dist/clickhouse.d.ts

@@ -1,6 +1,7 @@
 import type { IngesterConfig } from "./config.js";
 import { type LatencyHistogram } from "./latency.js";
 import { type ProfileSample } from "./profiles.js";
+import { type MemorySample, type PlatformEvent } from "./memory.js";
 import { type Migration } from "./migrations.js";
 import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
 export declare class ClickHouseStore {
@@ -25,6 +26,8 @@ export declare class ClickHouseStore {
     insertOccurrences(ctx: IngestContext, occurrences: RuntimeOccurrence[]): Promise<void>;
     insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
     insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise<void>;
+    insertMemorySamples(ctx: IngestContext, samples: MemorySample[]): Promise<void>;
+    insertPlatformEvent(ctx: IngestContext, event: PlatformEvent): Promise<void>;
     insertSourceMap(ctx: IngestContext, sourceMap: {
         release: string;
         filename: string;

package/dist/clickhouse.js

@@ -1,6 +1,7 @@
 import { createClient, } from "@clickhouse/client";
 import { latencyTableDDL } from "./latency.js";
 import { profileTableDDL } from "./profiles.js";
+import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
 import { sourceMapTableDDL } from "./source-maps.js";
 import { MIGRATIONS, migrationsTableDDL, } from "./migrations.js";
 /**
@@ -49,6 +50,8 @@ export class ClickHouseStore {
             `CREATE DATABASE IF NOT EXISTS ${db}`,
             latencyTableDDL(db, metricsTtlDays),
             profileTableDDL(db),
+            memoryTableDDL(db),
+            platformEventTableDDL(db),
             sourceMapTableDDL(db),
             `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
 				org_id             String,
@@ -275,12 +278,36 @@ export class ClickHouseStore {
                 org_id: ctx.orgId, repository_id: ctx.repositoryId,
                 profile_id: sample.profileId, sample_index: sample.sampleIndex, service: sample.service,
                 environment: sample.environment, release: sample.release,
-                trace_id: sample.traceId, observed_at: sample.observedAt.toISOString(),
+                trace_id: sample.traceId, instance_id: sample.instanceId, observed_at: sample.observedAt.toISOString(),
                 sample_type: sample.sampleType, unit: sample.unit,
                 stack: sample.stack, value: sample.value,
             })),
         });
     }
+    async insertMemorySamples(ctx, samples) {
+        if (!samples.length)
+            return;
+        if (!this.configured)
+            throw new Error("CLICKHOUSE_URL is not configured");
+        await this.ensureSchema();
+        await this.getClient().insert({ table: this.table("runtime_memory_samples"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: samples.map((s) => ({ org_id: ctx.orgId, repository_id: ctx.repositoryId,
+                service: s.service, environment: s.environment, release: s.release,
+                instance_id: s.instanceId, metric: s.metric, value: s.value, temporality: s.temporality,
+                observed_at: s.observedAt.toISOString() })) });
+    }
+    async insertPlatformEvent(ctx, event) {
+        if (!this.configured)
+            throw new Error("CLICKHOUSE_URL is not configured");
+        await this.ensureSchema();
+        await this.getClient().insert({ table: this.table("runtime_platform_events"), format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId, event_id: event.eventId,
+                    service: event.service, environment: event.environment, release: event.release,
+                    instance_id: event.instanceId, platform: event.platform, kind: event.kind,
+                    occurred_at: event.occurredAt }] });
+    }
     async insertSourceMap(ctx, sourceMap) {
         await this.ensureSchema();
         await this.getClient().insert({

package/dist/memory.d.ts

@@ -0,0 +1,45 @@
+import { z } from "zod";
+import type { OtlpMetricsRequest } from "./normalize-otlp.js";
+export type MemoryMetric = "rss" | "heap_used" | "heap_limit" | "memory_limit" | "heap_after_gc" | "gc_count" | "gc_duration_ms";
+export interface MemorySample {
+    service: string;
+    environment: string;
+    release: string;
+    instanceId: string;
+    metric: MemoryMetric;
+    value: number;
+    observedAt: Date;
+    temporality: "gauge" | "delta" | "cumulative";
+}
+export declare function normalizeMemoryMetrics(request: OtlpMetricsRequest): MemorySample[];
+export declare const platformEventSchema: z.ZodObject<{
+    service: z.ZodString;
+    environment: z.ZodDefault<z.ZodString>;
+    release: z.ZodDefault<z.ZodString>;
+    instanceId: z.ZodString;
+    platform: z.ZodEnum<["ecs", "kubernetes", "other"]>;
+    kind: z.ZodEnum<["oom_kill", "restart"]>;
+    occurredAt: z.ZodString;
+}, "strip", z.ZodTypeAny, {
+    service: string;
+    environment: string;
+    release: string;
+    instanceId: string;
+    platform: "ecs" | "kubernetes" | "other";
+    kind: "oom_kill" | "restart";
+    occurredAt: string;
+}, {
+    service: string;
+    instanceId: string;
+    platform: "ecs" | "kubernetes" | "other";
+    kind: "oom_kill" | "restart";
+    occurredAt: string;
+    environment?: string | undefined;
+    release?: string | undefined;
+}>;
+export type PlatformEvent = z.infer<typeof platformEventSchema> & {
+    eventId: string;
+};
+export declare function normalizePlatformEvent(input: z.infer<typeof platformEventSchema>): PlatformEvent | null;
+export declare function memoryTableDDL(db: string): string;
+export declare function platformEventTableDDL(db: string): string;

package/dist/memory.js

@@ -0,0 +1,114 @@
+import { createHash } from "node:crypto";
+import { z } from "zod";
+function attrsOf(attributes) {
+    return new Map((attributes ?? []).flatMap((a) => {
+        const v = a.value?.stringValue ?? a.value?.intValue ?? a.value?.doubleValue;
+        return a.key && v !== undefined ? [[a.key, String(v)]] : [];
+    }));
+}
+function metricOf(name, attrs) {
+    switch (name) {
+        case "autter.process.memory.rss":
+        case "process.runtime.nodejs.memory.rss": return "rss";
+        case "autter.process.memory.heap.used":
+        case "process.runtime.nodejs.memory.heap.used": return "heap_used";
+        case "autter.process.memory.heap.limit":
+        case "process.runtime.nodejs.memory.heap.limit": return "heap_limit";
+        case "autter.process.memory.limit":
+        case "container.memory.limit": return "memory_limit";
+        case "autter.process.memory.heap.after_gc": return "heap_after_gc";
+        case "autter.process.gc.count":
+        case "process.runtime.nodejs.gc.count": return "gc_count";
+        case "autter.process.gc.duration":
+        case "process.runtime.nodejs.gc.duration": return "gc_duration_ms";
+        case "process.memory.usage": {
+            const type = attrs.get("type") ?? attrs.get("state");
+            return type === "rss" || type === "physical" ? "rss" : type === "heap_used" ? "heap_used" : null;
+        }
+        default: return null;
+    }
+}
+export function normalizeMemoryMetrics(request) {
+    const out = [];
+    const now = Date.now();
+    for (const rm of request.resourceMetrics ?? []) {
+        const resource = attrsOf(rm.resource?.attributes);
+        const instanceId = resource.get("service.instance.id")?.slice(0, 128) ?? "";
+        const service = resource.get("service.name")?.slice(0, 200) ?? "";
+        if (!instanceId || !service)
+            continue;
+        const environment = (resource.get("deployment.environment.name") ?? resource.get("deployment.environment") ?? "production").slice(0, 100);
+        const release = (resource.get("service.version") ?? "").slice(0, 200);
+        for (const scope of rm.scopeMetrics ?? [])
+            for (const metric of scope.metrics ?? []) {
+                const name = metric.name ?? "";
+                const points = metric.gauge?.dataPoints ?? metric.sum?.dataPoints ?? [];
+                for (const point of points) {
+                    const kind = metricOf(name, attrsOf(point.attributes));
+                    if (!kind)
+                        continue;
+                    const counter = kind === "gc_count" || kind === "gc_duration_ms";
+                    const aggregation = metric.sum?.aggregationTemporality;
+                    const temporality = counter && (aggregation === 2 || aggregation === "AGGREGATION_TEMPORALITY_CUMULATIVE")
+                        ? "cumulative" : counter ? "delta" : "gauge";
+                    const value = Number(point.asDouble ?? point.asInt);
+                    let observedMs = 0;
+                    try {
+                        observedMs = Number(BigInt(String(point.timeUnixNano ?? "0")) / 1000000n);
+                    }
+                    catch {
+                        continue;
+                    }
+                    if (!Number.isFinite(value) || value < 0 || value > 1e15 || observedMs < now - 30 * 86400_000 || observedMs > now + 300_000)
+                        continue;
+                    const isBytes = kind === "rss" || kind === "heap_used" || kind === "heap_limit" || kind === "memory_limit" || kind === "heap_after_gc";
+                    const unit = metric.unit ?? "By";
+                    if (isBytes && !["By", "byte", "bytes", "KiBy", "MiBy"].includes(unit))
+                        continue;
+                    // Standard OTel runtimes report GC duration in seconds; Autter uses ms.
+                    out.push({ service, environment, release, instanceId, metric: kind,
+                        value: isBytes ? value * (unit === "KiBy" ? 1024 : unit === "MiBy" ? 1048576 : 1)
+                            : kind === "gc_duration_ms" && unit === "s" ? value * 1000 : value,
+                        observedAt: new Date(observedMs), temporality });
+                    if (out.length >= 5000)
+                        return out;
+                }
+            }
+    }
+    return out;
+}
+export const platformEventSchema = z.object({
+    service: z.string().regex(/^[a-zA-Z0-9._-]{1,200}$/),
+    environment: z.string().min(1).max(100).default("production"),
+    release: z.string().max(200).default(""),
+    instanceId: z.string().min(1).max(128),
+    platform: z.enum(["ecs", "kubernetes", "other"]),
+    kind: z.enum(["oom_kill", "restart"]),
+    occurredAt: z.string().datetime(),
+});
+export function normalizePlatformEvent(input) {
+    const time = Date.parse(input.occurredAt);
+    if (time < Date.now() - 30 * 86400_000 || time > Date.now() + 300_000)
+        return null;
+    const eventId = createHash("sha256").update(JSON.stringify(input)).digest("hex").slice(0, 32);
+    return { ...input, eventId };
+}
+export function memoryTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_memory_samples (
+		org_id String, repository_id String, service LowCardinality(String),
+		environment LowCardinality(String), release String DEFAULT '', instance_id String,
+		metric LowCardinality(String), value Float64, temporality LowCardinality(String) DEFAULT 'gauge',
+		observed_at DateTime64(3, 'UTC')
+	) ENGINE = ReplacingMergeTree PARTITION BY toDate(observed_at)
+	ORDER BY (org_id, repository_id, service, environment, instance_id, metric, observed_at)
+	TTL toDateTime(observed_at) + INTERVAL 14 DAY`;
+}
+export function platformEventTableDDL(db) {
+    return `CREATE TABLE IF NOT EXISTS ${db}.runtime_platform_events (
+		org_id String, repository_id String, event_id String, service LowCardinality(String),
+		environment LowCardinality(String), release String DEFAULT '', instance_id String,
+		platform LowCardinality(String), kind LowCardinality(String), occurred_at DateTime64(3, 'UTC')
+	) ENGINE = ReplacingMergeTree PARTITION BY toDate(occurred_at)
+	ORDER BY (org_id, repository_id, event_id)
+	TTL toDateTime(occurred_at) + INTERVAL 30 DAY`;
+}

package/dist/migrations.js

@@ -35,6 +35,7 @@
 import { latencyTableDDL } from "./latency.js";
 import { profileTableDDL } from "./profiles.js";
 import { sourceMapTableDDL } from "./source-maps.js";
+import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -119,6 +120,13 @@ export const MIGRATIONS = [
     { id: "0005-latency-histograms", statements: [latencyTableDDL("{db}")] },
     { id: "0006-profile-samples", statements: [profileTableDDL("{db}")] },
     { id: "0007-source-maps", statements: [sourceMapTableDDL("{db}")] },
+    { id: "0008-memory-signals", statements: [memoryTableDDL("{db}"), platformEventTableDDL("{db}")] },
+    { id: "0009-profile-instance", statements: [
+            `ALTER TABLE {db}.runtime_profile_samples ADD COLUMN IF NOT EXISTS instance_id String DEFAULT ''`,
+        ] },
+    { id: "0010-memory-temporality", statements: [
+            `ALTER TABLE {db}.runtime_memory_samples ADD COLUMN IF NOT EXISTS temporality LowCardinality(String) DEFAULT 'gauge'`,
+        ] },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -43,9 +43,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         timestamp: string;
         durationMs?: number | undefined;
+        message?: string | undefined;
         filename?: string | undefined;
         id?: string | undefined;
-        message?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;
@@ -85,9 +85,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "timing";
         timestamp: string;
         durationMs?: number | undefined;
+        message?: string | undefined;
         filename?: string | undefined;
         id?: string | undefined;
-        message?: string | undefined;
         severity?: "fatal" | "error" | "warning" | "info" | undefined;
         name?: string | undefined;
         stack?: string | undefined;

package/dist/normalize-otlp.d.ts

@@ -66,12 +66,16 @@ export interface OtlpMetricsRequest {
             metrics?: Array<{
                 name?: string;
                 unit?: string;
+                gauge?: {
+                    dataPoints?: OtlpDataPoint[];
+                };
                 histogram?: {
                     dataPoints?: OtlpDataPoint[];
                     aggregationTemporality?: string | number;
                 };
                 sum?: {
                     dataPoints?: OtlpDataPoint[];
+                    aggregationTemporality?: string | number;
                 };
             }>;
         }>;

package/dist/otlp-proto.js

@@ -54,10 +54,12 @@ message ScopeMetrics { repeated Metric metrics = 2; }
 message Metric {
   string name = 1;
   string unit = 3;
+  Gauge gauge = 5;
   Sum sum = 7;
   Histogram histogram = 9;
 }
-message Sum { repeated NumberDataPoint data_points = 1; }
+message Gauge { repeated NumberDataPoint data_points = 1; }
+message Sum { repeated NumberDataPoint data_points = 1; int32 aggregation_temporality = 2; }
 message NumberDataPoint {
   fixed64 time_unix_nano = 3;
   double as_double = 4;

package/dist/profiles.d.ts

@@ -5,6 +5,7 @@ export interface ProfileSample {
     environment: string;
     release: string;
     traceId: string;
+    instanceId: string;
     observedAt: Date;
     sampleType: string;
     unit: string;
@@ -17,4 +18,5 @@ export declare function decodeProfile(body: Buffer, meta: {
     environment: string;
     release: string;
     traceId: string;
+    instanceId: string;
 }): ProfileSample[];

package/dist/profiles.js

@@ -21,7 +21,7 @@ export function profileTableDDL(db) {
     return `CREATE TABLE IF NOT EXISTS ${db}.runtime_profile_samples (
 		org_id String, repository_id String, profile_id String, sample_index UInt16,
 		service LowCardinality(String), environment LowCardinality(String), release String,
-		trace_id String DEFAULT '', observed_at DateTime64(3, 'UTC'),
+		trace_id String DEFAULT '', instance_id String DEFAULT '', observed_at DateTime64(3, 'UTC'),
 		sample_type LowCardinality(String), unit LowCardinality(String),
 		stack Array(String), value UInt64
 	) ENGINE = MergeTree PARTITION BY toDate(observed_at)
@@ -37,18 +37,21 @@ export function decodeProfile(body, meta) {
     const strings = profile.stringTable ?? [];
     const locations = new Map((profile.location ?? []).map((location) => [location.id, location]));
     const functions = new Map((profile.function ?? []).map((fn) => [fn.id, fn]));
-    const type = profile.sampleType?.[0];
+    const typeIndex = Math.max(0, (profile.sampleType ?? []).findIndex((t) => strings[Number(t.type ?? 0)] === "inuse_space"));
+    const type = profile.sampleType?.[typeIndex];
     const observedMs = Number(BigInt(profile.timeNanos ?? "0") / 1000000n);
     const observedAt = observedMs > 0 && observedMs <= Date.now() + 300_000 ? new Date(observedMs) : new Date();
     const profileId = createHash("sha256").update(body).digest("hex").slice(0, 32);
     const samples = [];
     for (const [sampleIndex, sample] of (profile.sample ?? []).slice(0, 1000).entries()) {
         const stack = (sample.locationId ?? []).slice(0, 64).flatMap((id) => {
-            const fnId = locations.get(id)?.line?.[0]?.functionId;
-            const name = fnId ? strings[Number(functions.get(fnId)?.name ?? 0)] : undefined;
-            return name ? [name.slice(0, 200)] : [];
+            const locLine = locations.get(id)?.line?.[0];
+            const fn = locLine?.functionId ? functions.get(locLine.functionId) : undefined;
+            const name = fn ? strings[Number(fn.name ?? 0)] : undefined;
+            const filename = fn ? strings[Number(fn.filename ?? 0)] : undefined;
+            return name ? [`${name.slice(0, 150)}${filename ? ` (${filename.slice(0, 300)}:${Number(locLine?.line ?? 0)})` : ""}`] : [];
         });
-        const value = Number(sample.value?.[0] ?? 0);
+        const value = Number(sample.value?.[typeIndex] ?? 0);
         if (!stack.length || !Number.isSafeInteger(value) || value <= 0)
             continue;
         samples.push({ profileId, sampleIndex, ...meta, observedAt,

package/dist/server.js

@@ -7,6 +7,7 @@ import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-brow
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
 import { decodeMetricsRequest, decodeTraceRequest } from "./otlp-proto.js";
 import { decodeProfile } from "./profiles.js";
+import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js";
 import { validateSourceMap } from "./source-maps.js";
 import { SinkForwarder } from "./sink.js";
 export function createIngesterApp(config) {
@@ -156,13 +157,14 @@ export function createIngesterApp(config) {
         const environment = req.header("x-autter-environment")?.trim() || "production";
         const release = req.header("x-autter-release")?.trim() ?? "";
         const traceId = req.header("x-autter-trace-id")?.trim() ?? "";
-        if (!/^[a-zA-Z0-9._-]{1,200}$/.test(service) || environment.length > 100 || release.length > 200 || (traceId && !/^[a-f0-9]{32}$/.test(traceId))) {
+        const instanceId = req.header("x-autter-instance-id")?.trim() ?? "";
+        if (!/^[a-zA-Z0-9._-]{1,200}$/.test(service) || environment.length > 100 || release.length > 200 || instanceId.length > 128 || (traceId && !/^[a-f0-9]{32}$/.test(traceId))) {
             res.status(400).json({ error: "invalid profile metadata" });
             return;
         }
         let samples;
         try {
-            samples = decodeProfile(req.body, { service, environment, release, traceId });
+            samples = decodeProfile(req.body, { service, environment, release, traceId, instanceId });
             if (!samples.length)
                 throw new Error("empty profile");
         }
@@ -195,6 +197,29 @@ export function createIngesterApp(config) {
             storageError(res, err);
         }
     });
+    /** ECS/Kubernetes event forwarders use the same server key as OTLP. */
+    app.post("/v1/platform-events", async (req, res) => {
+        const ctx = await authenticate(req, res, "otlp");
+        if (!ctx)
+            return;
+        const parsed = platformEventSchema.safeParse(req.body);
+        if (!parsed.success) {
+            res.status(400).json({ error: "invalid platform event" });
+            return;
+        }
+        const event = normalizePlatformEvent(parsed.data);
+        if (!event) {
+            res.status(400).json({ error: "event timestamp outside retention window" });
+            return;
+        }
+        try {
+            await store.insertPlatformEvent(ctx, event);
+            res.status(202).json({ accepted: true });
+        }
+        catch (err) {
+            storageError(res, err);
+        }
+    });
     app.post("/v1/traces", async (req, res) => {
         const ctx = await authenticate(req, res, "otlp");
         if (!ctx)
@@ -255,9 +280,11 @@ export function createIngesterApp(config) {
             request = req.body;
         }
         const metricPoints = normalizeMetrics(request);
+        const memorySamples = normalizeMemoryMetrics(request);
         try {
             await store.insertLatencyHistograms(ctx, normalizeLatencyHistograms(request));
             await store.insertMetricPoints(ctx, metricPoints);
+            await store.insertMemorySamples(ctx, memorySamples);
         }
         catch (err) {
             storageError(res, err);

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.3.2",
+	"version": "1.3.3",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",