@autter/otlp-ingester 1.4.0 → 1.5.0

package/dist/server.js

@@ -2,7 +2,8 @@ import express from "express";
 import { KeyResolver, RateLimiter } from "./auth.js";
 import { ClickHouseStore } from "./clickhouse.js";
 import { normalizeLatencyHistograms } from "./latency.js";
-import { deriveFields, fingerprintOccurrence, occurrenceIdFor, } from "./fingerprint.js";
+import { deriveFields, occurrenceFingerprint, occurrenceIdFor, } from "./fingerprint.js";
+import { logPromotionCandidates, promotionLookups, promotionRollups, } from "./log-promotion.js";
 import { browserPayloadSchema, normalizeBrowserPayload, } from "./normalize-browser.js";
 import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
 import { decodeMetricsRequest, decodeTraceRequest, decodeLogsRequest } from "./otlp-proto.js";
@@ -11,7 +12,15 @@ import { decodeProfile } from "./profiles.js";
 import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js";
 import { validateSourceMap } from "./source-maps.js";
 import { SinkForwarder } from "./sink.js";
+import { configureRedaction, scrubOccurrence } from "./redact.js";
+import { buildCompatReport, evaluateCompat, INGESTER_VERSION_HEADER, } from "./compat.js";
+import { MIGRATIONS } from "./migrations.js";
+import { BROWSER_SDK_NAME, ingesterVersion, SdkVersionTracker, sdkSightingsFromOtlp, } from "./sdk-versions.js";
+/** Re-report the ingester version at most this often (keeps the row inside
+ * its TTL for long-running ingesters). */
+const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000;
 export function createIngesterApp(config) {
+    configureRedaction(config);
     const store = new ClickHouseStore(config);
     // Fingerprinted occurrences feed the consumer's issue grouping, metric
     // points feed the request/error-rate rollups, LLM calls feed spend
@@ -20,8 +29,54 @@ export function createIngesterApp(config) {
     const keys = new KeyResolver(config);
     const serverRateLimiter = new RateLimiter(config.rateLimitPerMinute);
     const clientRateLimiter = new RateLimiter(config.clientRateLimitPerMinute);
+    // ClickHouse dedupe lookups made by /v1/logs promotion, per tenant. Far
+    // tighter than the request limit: each lookup is a query, not an insert.
+    const promotionLookupLimiter = new RateLimiter(config.promotionLookupsPerMinute ?? 30);
+    const version = ingesterVersion();
+    const sdkVersions = new SdkVersionTracker((ctx, rows) => store.insertSdkVersions(ctx, rows));
+    const allMigrationIds = MIGRATIONS.map((migration) => migration.id);
+    const compatReport = () => {
+        const schema = store.schemaStatus();
+        return buildCompatReport({
+            version,
+            schemaStatus: schema.status,
+            applied: schema.applied,
+            allMigrations: allMigrationIds,
+        });
+    };
+    let lastInfoReport = 0;
+    async function reportIngesterInfo() {
+        if (!store.configured)
+            return;
+        if (Date.now() - lastInfoReport < INGESTER_INFO_INTERVAL_MS)
+            return;
+        lastInfoReport = Date.now();
+        try {
+            // Only reached from boot and authenticated ingest, which run the
+            // (memoized) schema bootstrap anyway.
+            await store.ensureSchema();
+            const report = compatReport();
+            await store.insertIngesterInfo({ version, schemaLevel: report.schema.level, report });
+        }
+        catch (err) {
+            // Retry in a minute, not on every request.
+            lastInfoReport = Date.now() - INGESTER_INFO_INTERVAL_MS + 60_000;
+            console.warn("ingester info report failed:", err?.message ?? err);
+        }
+    }
+    /** Fire-and-forget after a successful ingest: never delays or fails it. */
+    function recordSdks(ctx, request) {
+        void sdkVersions.observe(ctx, sdkSightingsFromOtlp(request));
+        void reportIngesterInfo();
+    }
     const app = express();
     app.disable("x-powered-by");
+    // Every response names the ingester version, so SDKs (and the browser
+    // relay) learn it for free from responses they already receive.
+    app.use((_req, res, next) => {
+        res.setHeader(INGESTER_VERSION_HEADER, version);
+        next();
+    });
     app.use(express.raw({ limit: "1mb", type: ["application/x-pprof"] }));
     app.use("/v1/sourcemaps", express.json({ limit: "5mb" }));
     app.use(express.json({
@@ -49,6 +104,7 @@ export function createIngesterApp(config) {
         res.setHeader("access-control-allow-origin", "*");
         res.setHeader("access-control-allow-methods", "POST, OPTIONS");
         res.setHeader("access-control-allow-headers", "content-type, authorization, x-autter-key");
+        res.setHeader("access-control-expose-headers", INGESTER_VERSION_HEADER);
         res.setHeader("access-control-max-age", "86400");
         if (req.method === "OPTIONS") {
             res.status(204).end();
@@ -72,6 +128,36 @@ export function createIngesterApp(config) {
             res.status(503).json({ ok: false, clickhouse: "down", ...sinkStats });
         }
     });
+    /**
+     * Version compatibility (public, like /healthz; no tenant data): this
+     * ingester's version, its ClickHouse schema level, and every feature in
+     * the compat manifest with whether it is available here. Optional
+     * `?features=a,b&sdk=<name>@<version>` adds the evaluated `issues`.
+     * Never triggers schema DDL.
+     */
+    app.get("/v1/compat", (req, res) => {
+        const report = compatReport();
+        const features = typeof req.query.features === "string"
+            ? req.query.features.split(",").map((f) => f.trim()).filter(Boolean).slice(0, 50)
+            : [];
+        const sdkParam = typeof req.query.sdk === "string" ? req.query.sdk.slice(0, 120) : "";
+        const at = sdkParam.lastIndexOf("@");
+        const sdk = at > 0 ? { name: sdkParam.slice(0, at), version: sdkParam.slice(at + 1) } : null;
+        const body = features.length || sdk
+            ? {
+                ...report,
+                issues: evaluateCompat({
+                    features: features.length
+                        ? features
+                        : report.features.filter((f) => sdk && f.sdks[sdk.name]).map((f) => f.id),
+                    ingester: { version, schema: { status: report.schema.status, applied: report.schema.applied } },
+                    sdk,
+                }),
+            }
+            : report;
+        res.setHeader("cache-control", "no-store");
+        res.status(200).json(body);
+    });
     /** Auth + scope + rate limit; returns null (response sent) on failure. */
     async function authenticate(req, res, surface) {
         // A storage-less ingester must refuse, not accept-and-drop: exporters
@@ -123,16 +209,48 @@ export function createIngesterApp(config) {
      * the sink consumer's per-occurrence dedupe holds across transport
      * retries and duplicated ClickHouse rows stay identifiable. */
     function fingerprintAll(ctx, inputs) {
-        return inputs.map((input, index) => {
-            const fingerprint = fingerprintOccurrence(input);
+        return inputs.map((raw, index) => {
+            // Second line of defence, before fingerprinting, storage, and the
+            // sink (which feeds LLM fix generation): scrub secrets from the
+            // free-text fields old SDKs and third-party OTLP senders pass
+            // through verbatim. Attributes were already sanitised by the
+            // normalisers (sanitizeRuntimeContext).
+            const input = scrubOccurrence(raw);
+            const { fingerprint, scheme } = occurrenceFingerprint(input);
             return {
                 ...input,
                 occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index),
                 fingerprint,
+                fingerprintScheme: scheme,
                 ...deriveFields(input),
             };
         });
     }
+    /** Fingerprint log-promotion candidates and drop the ones whose trace
+     * already produced an occurrence (in this batch, or within ±60 s). Ids
+     * are assigned BEFORE the lookup so they stay stable across retries
+     * whatever the lookup returns. */
+    async function promoteLogs(ctx, logs) {
+        const candidates = logPromotionCandidates(logs);
+        if (candidates.length === 0)
+            return [];
+        const fingerprinted = fingerprintAll(ctx, candidates);
+        const seen = new Set();
+        const ownIds = fingerprinted.map((o) => o.occurrenceId);
+        for (const lookup of promotionLookups(candidates)) {
+            if (!promotionLookupLimiter.allow(`${ctx.orgId}:${ctx.repositoryId}`))
+                break;
+            try {
+                const found = await store.recentOccurrenceTraceIds(ctx, lookup.traceIds, lookup.from, lookup.to, ownIds);
+                for (const traceId of found)
+                    seen.add(traceId);
+            }
+            catch (err) {
+                console.warn("log promotion dedupe lookup failed — promoting without it:", err instanceof Error ? err.message : err);
+            }
+        }
+        return fingerprinted.filter((o) => !o.traceId || !seen.has(o.traceId));
+    }
     function storageError(res, err) {
         console.error("clickhouse write failed:", err);
         res.status(503).json({ error: "storage unavailable, retry later" });
@@ -261,6 +379,7 @@ export function createIngesterApp(config) {
             return;
         }
         sink?.enqueue(ctx, fingerprinted, metricPoints, llmCalls);
+        recordSdks(ctx, request);
         otlpSuccess(req, res);
     });
     app.post("/v1/logs", async (req, res) => {
@@ -268,22 +387,34 @@ export function createIngesterApp(config) {
         if (!ctx)
             return;
         let logs;
+        let logsRequest;
         try {
-            logs = normalizeLogs(req.is("application/x-protobuf") ? decodeLogsRequest(req.body) : req.body);
+            logsRequest = req.is("application/x-protobuf") ? decodeLogsRequest(req.body) : req.body;
+            logs = normalizeLogs(logsRequest);
         }
         catch {
             res.status(400).json({ error: "invalid OTLP logs payload" });
             return;
         }
+        // Logs are diagnostic evidence; exceptions and failed outcomes normally
+        // reach grouping through the trace path. The exception: logger-only /
+        // edge SDKs mark span-less error records `autter.capture.mode = "log"`,
+        // and those are promoted to occurrences here — see log-promotion.ts.
+        const promoted = await promoteLogs(ctx, logs);
+        const metricPoints = promotionRollups(promoted);
         try {
-            await store.insertLogs(ctx, logs);
+            await Promise.all([
+                store.insertLogs(ctx, logs),
+                store.insertOccurrences(ctx, promoted),
+                store.insertMetricPoints(ctx, metricPoints),
+            ]);
         }
         catch (err) {
             storageError(res, err);
             return;
         }
-        // Logs are diagnostic evidence. Exceptions and failed outcomes use the trace sink,
-        // so one operation does not create duplicate issues through two export paths.
+        sink?.enqueue(ctx, promoted, metricPoints);
+        recordSdks(ctx, logsRequest);
         otlpSuccess(req, res);
     });
     app.post("/v1/metrics", async (req, res) => {
@@ -315,6 +446,7 @@ export function createIngesterApp(config) {
             return;
         }
         sink?.enqueue(ctx, [], metricPoints);
+        recordSdks(ctx, request);
         otlpSuccess(req, res);
     });
     app.post("/v1/browser", async (req, res) => {
@@ -353,6 +485,16 @@ export function createIngesterApp(config) {
             return;
         }
         sink?.enqueue(ctx, fingerprinted, metricPoints);
+        if (parsed.data.sdk) {
+            void sdkVersions.observe(ctx, [{
+                    service: parsed.data.service,
+                    environment: parsed.data.environment,
+                    sdkName: BROWSER_SDK_NAME,
+                    sdkVersion: parsed.data.sdk,
+                    sdkLanguage: "webjs",
+                }]);
+        }
+        void reportIngesterInfo();
         res.status(202).json({ accepted: fingerprinted.length });
     });
     // Body-parser errors (oversized/malformed JSON) → clean 4xx, not a stack.
@@ -370,5 +512,5 @@ export function createIngesterApp(config) {
         console.error("unhandled error:", err);
         res.status(500).json({ error: "internal error" });
     });
-    return { app, store, sink };
+    return { app, store, sink, reportIngesterInfo };
 }

package/dist/sink.d.ts

@@ -101,3 +101,12 @@ export declare class SinkForwarder {
     private scheduleWake;
     private send;
 }
+/**
+ * Wire shape of one occurrence (payload `version: 1`). 1.5.0 adds, all
+ * optional and omitted when absent: errorCode, why, fix, link, expected,
+ * requestId, traceId, route, method, statusCode, fingerprintScheme. The
+ * request-context four were previously spread as `null` when unknown and
+ * are now simply left out, matching the documented optional-string shape.
+ * Older consumers ignore unknown keys, so the version stays 1.
+ */
+export declare function sinkOccurrence(o: RuntimeOccurrence): Record<string, unknown>;

package/dist/sink.js

@@ -108,10 +108,7 @@ export class SinkForwarder {
             batchId,
             orgId: ctx.orgId,
             repositoryId: ctx.repositoryId,
-            occurrences: occurrences.map((o) => ({
-                ...o,
-                occurredAt: o.occurredAt.toISOString(),
-            })),
+            occurrences: occurrences.map(sinkOccurrence),
             metrics: metricPoints.map((p) => ({
                 ...p,
                 bucketAt: p.bucketAt.toISOString(),
@@ -298,6 +295,31 @@ export class SinkForwarder {
         this.scheduleWake();
     }
 }
+/**
+ * Wire shape of one occurrence (payload `version: 1`). 1.5.0 adds, all
+ * optional and omitted when absent: errorCode, why, fix, link, expected,
+ * requestId, traceId, route, method, statusCode, fingerprintScheme. The
+ * request-context four were previously spread as `null` when unknown and
+ * are now simply left out, matching the documented optional-string shape.
+ * Older consumers ignore unknown keys, so the version stays 1.
+ */
+export function sinkOccurrence(o) {
+    const { traceId, route, method, statusCode, errorCode, why, fix, link, expected, requestId, ...rest } = o;
+    return {
+        ...rest,
+        occurredAt: o.occurredAt.toISOString(),
+        ...(traceId ? { traceId } : {}),
+        ...(route ? { route } : {}),
+        ...(method ? { method } : {}),
+        ...(statusCode != null ? { statusCode } : {}),
+        ...(errorCode ? { errorCode } : {}),
+        ...(why ? { why } : {}),
+        ...(fix ? { fix } : {}),
+        ...(link ? { link } : {}),
+        ...(expected ? { expected: true } : {}),
+        ...(requestId ? { requestId } : {}),
+    };
+}
 /** ISO range across every signal in the batch — the replay hint on drops. */
 function signalRange(occurrences, metricPoints, llmCalls) {
     const timestamps = [

package/dist/types.d.ts

@@ -28,6 +28,22 @@ export interface RuntimeOccurrenceInput {
     sessionId: string | null;
     attributes: Record<string, unknown> | null;
     occurredAt: Date;
+    /**
+     * Declared error metadata (`autter.error.*`, see error-fields.ts) —
+     * optional and omitted when absent. A valid `errorCode` switches the
+     * fingerprint to the source-independent "code-v1" scheme.
+     */
+    errorCode?: string;
+    /** Declared cause (≤1000 chars). */
+    why?: string;
+    /** Declared remedy (≤1000 chars). */
+    fix?: string;
+    /** Docs link — http(s) only, ≤500 chars. */
+    link?: string;
+    /** Expected business failure (declines, validation): recorded, never paged. */
+    expected?: boolean;
+    /** `autter.request.id` — joins the occurrence to its request summary. */
+    requestId?: string;
 }
 /**
  * Derived, aggregation-ready fields — computed once at ingest (from the
@@ -37,6 +53,8 @@ export interface RuntimeOccurrenceInput {
 export interface RuntimeOccurrence extends RuntimeOccurrenceInput {
     occurrenceId: string;
     fingerprint: string;
+    /** "code-v1" when grouped by a valid error code, else "message-v1". */
+    fingerprintScheme: "message-v1" | "code-v1";
     /** normalizeRoute(route) — "/users/:id/orders/:id"; GROUP BY-safe. */
     routeNormalized: string;
     /** normalizeMessage(message) — ids/numbers/strings templated out. */

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "1.4.0",
+	"version": "1.5.0",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",