@autter/otlp-ingester 1.2.0 → 1.3.0

package/dist/clickhouse.d.ts

@@ -1,4 +1,5 @@
 import type { IngesterConfig } from "./config.js";
+import { type LatencyHistogram } from "./latency.js";
 import { type Migration } from "./migrations.js";
 import type { IngestContext, RuntimeLlmCall, RuntimeMetricPoint, RuntimeOccurrence, RuntimeSpanRow } from "./types.js";
 export declare class ClickHouseStore {
@@ -24,5 +25,6 @@ export declare class ClickHouseStore {
     insertSpans(ctx: IngestContext, spans: RuntimeSpanRow[]): Promise<void>;
     insertLlmCalls(ctx: IngestContext, calls: RuntimeLlmCall[]): Promise<void>;
     insertMetricPoints(ctx: IngestContext, points: RuntimeMetricPoint[]): Promise<void>;
+    insertLatencyHistograms(ctx: IngestContext, points: LatencyHistogram[]): Promise<void>;
     close(): Promise<void>;
 }

package/dist/clickhouse.js

@@ -1,4 +1,5 @@
 import { createClient, } from "@clickhouse/client";
+import { latencyTableDDL } from "./latency.js";
 import { MIGRATIONS, migrationsTableDDL, } from "./migrations.js";
 /**
  * ClickHouse persistence. The schema is idempotent and applied lazily once
@@ -44,6 +45,7 @@ export class ClickHouseStore {
         const { occurrenceTtlDays, spanTtlDays, metricsTtlDays, llmCallTtlDays } = this.config;
         return [
             `CREATE DATABASE IF NOT EXISTS ${db}`,
+            latencyTableDDL(db, metricsTtlDays),
             `CREATE TABLE IF NOT EXISTS ${db}.runtime_error_occurrences (
 				org_id             String,
 				repository_id      String,
@@ -312,6 +314,23 @@ export class ClickHouseStore {
             })),
         });
     }
+    async insertLatencyHistograms(ctx, points) {
+        if (points.length === 0 || !this.configured)
+            return;
+        await this.ensureSchema();
+        await this.getClient().insert({
+            table: this.table("runtime_latency_histograms"),
+            format: "JSONEachRow",
+            clickhouse_settings: INSERT_SETTINGS,
+            values: points.map((point) => ({
+                org_id: ctx.orgId, repository_id: ctx.repositoryId, point_id: point.pointId,
+                service: point.service, environment: point.environment, release: point.release,
+                method: point.method, route: point.route, bucket_at: point.bucketAt.toISOString(),
+                request_count: point.requestCount, error_count: point.errorCount,
+                duration_sum_ms: point.durationSumMs, bounds_ms: point.boundsMs, counts: point.counts,
+            })),
+        });
+    }
     async close() {
         if (!this.client)
             return;

package/dist/latency.d.ts

@@ -0,0 +1,17 @@
+import type { OtlpMetricsRequest } from "./normalize-otlp.js";
+export interface LatencyHistogram {
+    pointId: string;
+    service: string;
+    environment: string;
+    release: string;
+    method: string;
+    route: string;
+    bucketAt: Date;
+    requestCount: number;
+    errorCount: number;
+    durationSumMs: number;
+    boundsMs: number[];
+    counts: number[];
+}
+export declare function latencyTableDDL(database: string, ttlDays?: number): string;
+export declare function normalizeLatencyHistograms(request: OtlpMetricsRequest): LatencyHistogram[];

package/dist/latency.js

@@ -0,0 +1,68 @@
+import { createHash } from "node:crypto";
+import { normalizeRoute } from "./fingerprint.js";
+export function latencyTableDDL(database, ttlDays = 30) {
+    return `CREATE TABLE IF NOT EXISTS ${database}.runtime_latency_histograms (
+		org_id String, repository_id String, point_id String,
+		service LowCardinality(String), environment LowCardinality(String),
+		release String, method LowCardinality(String), route String,
+		bucket_at DateTime('UTC'), request_count UInt64, error_count UInt64,
+		duration_sum_ms Float64, bounds_ms Array(Float64), counts Array(UInt64)
+	) ENGINE = ReplacingMergeTree
+	PARTITION BY toDate(bucket_at)
+	ORDER BY (org_id, repository_id, service, environment, bucket_at, point_id)
+	TTL bucket_at + INTERVAL ${Math.max(1, Math.round(ttlDays))} DAY`;
+}
+export function normalizeLatencyHistograms(request) {
+    const points = [];
+    for (const resource of request.resourceMetrics ?? []) {
+        const hasStableDuration = resource.scopeMetrics?.some((scope) => scope.metrics?.some((metric) => metric.name === "http.server.request.duration"));
+        const resourceAttributes = Object.fromEntries((resource.resource?.attributes ?? [])
+            .map((attribute) => [attribute.key ?? "", attribute.value?.stringValue ?? ""]));
+        for (const scope of resource.scopeMetrics ?? []) {
+            for (const metric of scope.metrics ?? []) {
+                if (hasStableDuration && metric.name === "http.server.duration")
+                    continue;
+                const multiplier = metric.name === "http.server.request.duration" ? 1000
+                    : metric.name === "http.server.duration" ? 1 : null;
+                if (multiplier === null || ![2, "2", "AGGREGATION_TEMPORALITY_DELTA"].includes(metric.histogram?.aggregationTemporality ?? ""))
+                    continue;
+                for (const point of metric.histogram?.dataPoints ?? []) {
+                    const bounds = point.explicitBounds ?? [];
+                    const counts = (point.bucketCounts ?? []).map(Number);
+                    const count = Number(point.count);
+                    const endMs = Number(point.timeUnixNano) / 1e6;
+                    const startMs = Number(point.startTimeUnixNano) / 1e6;
+                    if (!Number.isSafeInteger(count) || count <= 0 || !Number.isFinite(endMs)
+                        || !Number.isFinite(startMs) || endMs <= startMs || endMs - startMs > 120_000
+                        || !Number.isFinite(point.sum) || Number(point.sum) < 0
+                        || bounds.length === 0 || bounds.length > 256 || counts.length !== bounds.length + 1
+                        || bounds.some((bound, index) => !Number.isFinite(bound) || bound < 0 || (index > 0 && bound <= bounds[index - 1]))
+                        || counts.some((value) => !Number.isSafeInteger(value) || value < 0)
+                        || counts.reduce((total, value) => total + value, 0) !== count)
+                        continue;
+                    const attrs = Object.fromEntries((point.attributes ?? []).map((attribute) => [attribute.key ?? "",
+                        String(attribute.value?.stringValue ?? attribute.value?.intValue ?? "")]));
+                    const route = normalizeRoute(attrs["http.route"] ?? "");
+                    const method = attrs["http.request.method"] ?? attrs["http.method"] ?? "";
+                    if (!route || !method)
+                        continue;
+                    const status = Number(attrs["http.response.status_code"] ?? attrs["http.status_code"]);
+                    points.push({
+                        pointId: createHash("sha256").update(JSON.stringify([metric.name,
+                            Object.entries(resourceAttributes).sort(), Object.entries(attrs).sort(),
+                            String(point.startTimeUnixNano), String(point.timeUnixNano)])).digest("hex"),
+                        service: resourceAttributes["service.name"] || "unknown",
+                        environment: resourceAttributes["deployment.environment.name"] || resourceAttributes["deployment.environment"] || "production",
+                        release: resourceAttributes["service.version"] || "",
+                        method: method.slice(0, 20), route,
+                        bucketAt: new Date(Math.floor(endMs / 60_000) * 60_000),
+                        requestCount: count, errorCount: status >= 500 ? count : 0,
+                        durationSumMs: Number(point.sum) * multiplier,
+                        boundsMs: bounds.map((bound) => bound * multiplier), counts,
+                    });
+                }
+            }
+        }
+    }
+    return points;
+}

package/dist/migrations.js

@@ -32,6 +32,7 @@
  *     ],
  *   },
  */
+import { latencyTableDDL } from "./latency.js";
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -113,6 +114,7 @@ export const MIGRATIONS = [
 			TTL toDateTime(started_at) + INTERVAL 90 DAY`,
         ],
     },
+    { id: "0005-latency-histograms", statements: [latencyTableDDL("{db}")] },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -24,7 +24,7 @@ export declare const browserPayloadSchema: z.ZodObject<{
         route: z.ZodOptional<z.ZodString>;
         context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
     }, "strip", z.ZodTypeAny, {
-        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
         timestamp: string;
         id?: string | undefined;
@@ -38,7 +38,7 @@ export declare const browserPayloadSchema: z.ZodObject<{
         route?: string | undefined;
         context?: Record<string, unknown> | undefined;
     }, {
-        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
         id?: string | undefined;
         message?: string | undefined;
@@ -53,11 +53,11 @@ export declare const browserPayloadSchema: z.ZodObject<{
         context?: Record<string, unknown> | undefined;
     }>, "many">;
 }, "strip", z.ZodTypeAny, {
-    version: 1;
     service: string;
     environment: string;
+    version: 1;
     events: {
-        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
         message: string;
         timestamp: string;
         id?: string | undefined;
@@ -71,14 +71,14 @@ export declare const browserPayloadSchema: z.ZodObject<{
         route?: string | undefined;
         context?: Record<string, unknown> | undefined;
     }[];
-    sessionId?: string | undefined;
     release?: string | undefined;
+    sessionId?: string | undefined;
 }, {
-    version: 1;
     service: string;
     environment: string;
+    version: 1;
     events: {
-        type: "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
+        type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event";
         timestamp: string;
         id?: string | undefined;
         message?: string | undefined;
@@ -92,8 +92,8 @@ export declare const browserPayloadSchema: z.ZodObject<{
         route?: string | undefined;
         context?: Record<string, unknown> | undefined;
     }[];
-    sessionId?: string | undefined;
     release?: string | undefined;
+    sessionId?: string | undefined;
 }>;
 export type BrowserPayload = z.infer<typeof browserPayloadSchema>;
 export interface NormalizedBrowser {

package/dist/normalize-otlp.d.ts

@@ -50,7 +50,10 @@ export interface OtlpTraceRequest {
 }
 interface OtlpDataPoint {
     attributes?: OtlpKeyValue[];
+    startTimeUnixNano?: string | number;
     timeUnixNano?: string | number;
+    explicitBounds?: number[];
+    bucketCounts?: Array<string | number>;
     count?: string | number;
     sum?: number;
     asInt?: string | number;

package/dist/normalize-otlp.js

@@ -124,10 +124,12 @@ export function normalizeTraces(request) {
                     name: span.name ?? "unnamed",
                     kind,
                     status: isError ? "error" : "ok",
-                    route,
+                    route: route ? normalizeRoute(route) : null,
                     statusCode,
                     durationMs,
-                    attributes: null,
+                    attributes: {
+                        "http.request.method": (attrs.get("http.request.method") ?? attrs.get("http.method") ?? "").slice(0, 20),
+                    },
                     startedAt,
                 });
                 // Error occurrences: one per exception event; if the span is

package/dist/otlp-proto.js

@@ -66,9 +66,12 @@ message NumberDataPoint {
 }
 message Histogram { repeated HistogramDataPoint data_points = 1; int32 aggregation_temporality = 2; }
 message HistogramDataPoint {
+  fixed64 start_time_unix_nano = 2;
   fixed64 time_unix_nano = 3;
   fixed64 count = 4;
   optional double sum = 5;
+  repeated fixed64 bucket_counts = 6;
+  repeated double explicit_bounds = 7;
   repeated KeyValue attributes = 9;
 }
 `;

package/dist/server.js

@@ -1,6 +1,7 @@
 import express from "express";
 import { KeyResolver, RateLimiter } from "./auth.js";
 import { ClickHouseStore } from "./clickhouse.js";
+import { normalizeLatencyHistograms } from "./latency.js";
 import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerprint.js";
 import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
@@ -200,6 +201,7 @@ export function createIngesterApp(config) {
         }
         const metricPoints = normalizeMetrics(request);
         try {
+            await store.insertLatencyHistograms(ctx, normalizeLatencyHistograms(request));
             await store.insertMetricPoints(ctx, metricPoints);
         }
         catch (err) {

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.2.0",
+	"version": "1.3.0",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",