@autter/otlp-ingester 0.1.0 → 1.1.0

package/dist/sink.js

@@ -0,0 +1,314 @@
+import { randomUUID } from "node:crypto";
+const DEFAULT_TUNING = {
+    retryBaseMs: 1000,
+    retryCapMs: 60_000,
+    requestTimeoutMs: 10_000,
+    healthyConcurrency: 4,
+};
+/** Fixed failure category — safe for the unauthenticated health response. */
+function failureReason(err) {
+    if (err instanceof Error) {
+        if (err.name === "TimeoutError" || err.name === "AbortError") {
+            return "timeout";
+        }
+        // fetch surfaces DNS/TLS/socket failures as TypeError.
+        if (err instanceof TypeError)
+            return "connection_error";
+    }
+    return "error";
+}
+export class SinkForwarder {
+    config;
+    fetchImpl;
+    /** Always sorted by `seq`: queue[0] is the oldest batch. */
+    queue = [];
+    queuedBytes = 0;
+    /** Per-org share of queuedBytes — overflow evicts from the heaviest org. */
+    queuedBytesByOrg = new Map();
+    inFlight = 0;
+    timer = null;
+    stopped = false;
+    seqCounter = 0;
+    delivered = 0;
+    retried = 0;
+    droppedOverflow = 0;
+    droppedPermanent = 0;
+    consecutiveFailures = 0;
+    /**
+     * Bumped on every failure. A success only clears consecutiveFailures if
+     * no failure happened after that request STARTED — a concurrent success
+     * that overlapped a failure proves nothing about current sink health and
+     * must not reopen full concurrency mid-outage.
+     */
+    failureEpoch = 0;
+    lastFailureAt = null;
+    lastFailureReason = null;
+    tuning;
+    constructor(config, fetchImpl = fetch, tuning = {}) {
+        this.config = config;
+        this.fetchImpl = fetchImpl;
+        this.tuning = { ...DEFAULT_TUNING, ...tuning };
+    }
+    /** Queue a batch for delivery. No-op when there is nothing to send. */
+    enqueue(ctx, occurrences, metricPoints = [], llmCalls = []) {
+        if (!this.config.sinkUrl || this.stopped)
+            return;
+        if (occurrences.length === 0 &&
+            metricPoints.length === 0 &&
+            llmCalls.length === 0) {
+            return;
+        }
+        const batch = this.buildBatch(ctx, occurrences, metricPoints, llmCalls);
+        // A batch bigger than the whole buffer could never be admitted without
+        // evicting everyone else — and the consumer's body cap would reject it
+        // anyway. Drop it alone instead of letting it flush the queue.
+        if (batch.bytes > this.maxBufferedBytes()) {
+            this.droppedOverflow += 1;
+            console.warn(`sink batch ${batch.batchId} (org ${batch.orgId}) exceeds the buffer cap (${batch.bytes} bytes) — dropped; replay signals ${batch.signalsFrom ?? "?"} .. ${batch.signalsTo ?? "?"} from ClickHouse`);
+            return;
+        }
+        this.queue.push(batch);
+        this.track(batch);
+        this.enforceBounds();
+        this.pump();
+    }
+    stats() {
+        const oldest = this.queue[0]?.enqueuedAt ?? null;
+        return {
+            queued: this.queue.length,
+            queuedBytes: this.queuedBytes,
+            inFlight: this.inFlight,
+            delivered: this.delivered,
+            retried: this.retried,
+            droppedOverflow: this.droppedOverflow,
+            droppedPermanent: this.droppedPermanent,
+            consecutiveFailures: this.consecutiveFailures,
+            lastFailureAt: this.lastFailureAt,
+            lastFailureReason: this.lastFailureReason,
+            oldestQueuedSince: oldest ? new Date(oldest).toISOString() : null,
+        };
+    }
+    pendingCount() {
+        return this.queue.length + this.inFlight;
+    }
+    /** Stop scheduling new deliveries (shutdown). Queued batches are logged
+     * by the caller — they are recoverable from ClickHouse, not from here. */
+    stop() {
+        this.stopped = true;
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+    }
+    /** Serialize one delivery payload; `seq` pins its age for ordering. */
+    buildBatch(ctx, occurrences, metricPoints, llmCalls) {
+        const batchId = randomUUID();
+        const body = JSON.stringify({
+            version: 1,
+            batchId,
+            orgId: ctx.orgId,
+            repositoryId: ctx.repositoryId,
+            occurrences: occurrences.map((o) => ({
+                ...o,
+                occurredAt: o.occurredAt.toISOString(),
+            })),
+            metrics: metricPoints.map((p) => ({
+                ...p,
+                bucketAt: p.bucketAt.toISOString(),
+            })),
+            llmCalls: llmCalls.map((c) => ({
+                ...c,
+                startedAt: c.startedAt.toISOString(),
+            })),
+        });
+        const range = signalRange(occurrences, metricPoints, llmCalls);
+        return {
+            batchId,
+            orgId: ctx.orgId,
+            body,
+            bytes: Buffer.byteLength(body),
+            seq: ++this.seqCounter,
+            attempts: 0,
+            nextAttemptAt: Date.now(),
+            signalsFrom: range.from,
+            signalsTo: range.to,
+            enqueuedAt: Date.now(),
+        };
+    }
+    maxBufferedBytes() {
+        return this.config.sinkMaxBufferedMb * 1024 * 1024;
+    }
+    track(batch) {
+        this.queuedBytes += batch.bytes;
+        this.queuedBytesByOrg.set(batch.orgId, (this.queuedBytesByOrg.get(batch.orgId) ?? 0) + batch.bytes);
+    }
+    untrack(batch) {
+        this.queuedBytes -= batch.bytes;
+        const left = (this.queuedBytesByOrg.get(batch.orgId) ?? 0) - batch.bytes;
+        if (left > 0)
+            this.queuedBytesByOrg.set(batch.orgId, left);
+        else
+            this.queuedBytesByOrg.delete(batch.orgId);
+    }
+    /** Re-insert a retrying batch at its age position (queue is seq-sorted),
+     * so overflow eviction still drops the genuinely oldest signals first. */
+    insertBySeq(batch) {
+        const at = this.queue.findIndex((b) => b.seq > batch.seq);
+        if (at === -1)
+            this.queue.push(batch);
+        else
+            this.queue.splice(at, 0, batch);
+    }
+    /**
+     * Oldest-first eviction, charged to the heaviest tenant: the freshest
+     * signals matter most to grouping, and one org flooding the shared
+     * buffer must not evict everyone else's batches.
+     */
+    enforceBounds() {
+        const maxBytes = this.maxBufferedBytes();
+        while (this.queue.length > this.config.sinkMaxBufferedBatches ||
+            (this.queuedBytes > maxBytes && this.queue.length > 1)) {
+            const dropped = this.evictOne();
+            if (!dropped)
+                break;
+            this.droppedOverflow += 1;
+            console.warn(`sink buffer overflow: dropped batch ${dropped.batchId} (org ${dropped.orgId}, signals ${dropped.signalsFrom ?? "?"} .. ${dropped.signalsTo ?? "?"}) — replay this range from ClickHouse`);
+        }
+    }
+    /** The oldest batch of the org holding the most buffered bytes. */
+    evictOne() {
+        let heaviest = null;
+        let heaviestBytes = -1;
+        for (const [orgId, bytes] of this.queuedBytesByOrg) {
+            if (bytes > heaviestBytes) {
+                heaviest = orgId;
+                heaviestBytes = bytes;
+            }
+        }
+        const at = heaviest
+            ? this.queue.findIndex((b) => b.orgId === heaviest)
+            : 0;
+        const [dropped] = this.queue.splice(at === -1 ? 0 : at, 1);
+        if (!dropped)
+            return null;
+        this.untrack(dropped);
+        return dropped;
+    }
+    currentLimit() {
+        return this.consecutiveFailures > 0 ? 1 : this.tuning.healthyConcurrency;
+    }
+    pump() {
+        if (this.stopped || !this.config.sinkUrl)
+            return;
+        const limit = this.currentLimit();
+        const now = Date.now();
+        while (this.inFlight < limit) {
+            const index = this.queue.findIndex((b) => b.nextAttemptAt <= now);
+            if (index === -1)
+                break;
+            const [batch] = this.queue.splice(index, 1);
+            if (!batch)
+                break;
+            this.untrack(batch);
+            this.inFlight += 1;
+            void this.send(batch).finally(() => {
+                this.inFlight -= 1;
+                this.pump();
+            });
+        }
+        this.scheduleWake();
+    }
+    scheduleWake() {
+        if (this.timer) {
+            clearTimeout(this.timer);
+            this.timer = null;
+        }
+        if (this.stopped || this.queue.length === 0)
+            return;
+        // At capacity there is nothing to wake for: every completion pumps
+        // again anyway, and a 0 ms timer here would spin the event loop while
+        // a ready batch waits on a slow in-flight request.
+        if (this.inFlight >= this.currentLimit())
+            return;
+        const next = Math.min(...this.queue.map((b) => b.nextAttemptAt));
+        this.timer = setTimeout(() => {
+            this.timer = null;
+            this.pump();
+        }, Math.max(0, next - Date.now()));
+        this.timer.unref();
+    }
+    async send(batch) {
+        batch.attempts += 1;
+        const epochAtStart = this.failureEpoch;
+        let reason = null;
+        /** Raw failure text — logged server-side, never surfaced in stats. */
+        let detail = null;
+        let permanent = false;
+        try {
+            const res = await this.fetchImpl(this.config.sinkUrl, {
+                method: "POST",
+                headers: {
+                    "content-type": "application/json",
+                    ...(this.config.sinkToken
+                        ? { authorization: `Bearer ${this.config.sinkToken}` }
+                        : {}),
+                },
+                body: batch.body,
+                signal: AbortSignal.timeout(this.tuning.requestTimeoutMs),
+            });
+            // Drain the body so keep-alive sockets are reusable; its content is
+            // irrelevant to delivery.
+            await res.text().catch(() => { });
+            if (res.ok) {
+                this.delivered += 1;
+                if (epochAtStart === this.failureEpoch) {
+                    this.consecutiveFailures = 0;
+                }
+                return;
+            }
+            reason = `http_${res.status}`;
+            detail = `sink responded ${res.status}`;
+            // 4xx (except timeout/rate-limit) means the consumer rejected the
+            // batch — retrying the same body cannot succeed.
+            permanent = res.status < 500 && res.status !== 408 && res.status !== 429;
+        }
+        catch (err) {
+            reason = failureReason(err);
+            detail = err instanceof Error ? err.message : String(err);
+        }
+        this.failureEpoch += 1;
+        this.consecutiveFailures += 1;
+        this.lastFailureAt = new Date().toISOString();
+        this.lastFailureReason = reason;
+        if (permanent || batch.attempts >= this.config.sinkMaxAttempts) {
+            this.droppedPermanent += 1;
+            console.error(`sink delivery gave up after ${batch.attempts} attempt(s) (${detail}): batch ${batch.batchId} (signals ${batch.signalsFrom ?? "?"} .. ${batch.signalsTo ?? "?"}) — replay this range from ClickHouse`);
+            return;
+        }
+        this.retried += 1;
+        // Full jitter avoids retry stampedes when the consumer comes back.
+        const backoff = Math.min(this.tuning.retryCapMs, this.tuning.retryBaseMs * 2 ** (batch.attempts - 1));
+        batch.nextAttemptAt = Date.now() + backoff / 2 + Math.random() * (backoff / 2);
+        this.insertBySeq(batch);
+        this.track(batch);
+        if (batch.attempts === 1 || batch.attempts % 5 === 0) {
+            console.warn(`sink delivery failed (attempt ${batch.attempts}/${this.config.sinkMaxAttempts}, will retry): ${detail}`);
+        }
+        this.enforceBounds();
+        this.scheduleWake();
+    }
+}
+/** ISO range across every signal in the batch — the replay hint on drops. */
+function signalRange(occurrences, metricPoints, llmCalls) {
+    const timestamps = [
+        ...occurrences.map((o) => o.occurredAt),
+        ...metricPoints.map((p) => p.bucketAt),
+        ...llmCalls.map((c) => c.startedAt),
+    ].map((d) => d.getTime());
+    if (timestamps.length === 0)
+        return { from: null, to: null };
+    return {
+        from: new Date(Math.min(...timestamps)).toISOString(),
+        to: new Date(Math.max(...timestamps)).toISOString(),
+    };
+}

package/dist/types.d.ts

@@ -62,6 +62,39 @@ export interface RuntimeSpanRow {
     attributes: Record<string, unknown> | null;
     startedAt: Date;
 }
+/**
+ * One LLM provider call (chat, completion, embedding, …) extracted from a
+ * span carrying GenAI/Vercel-AI/Autter attributes — see llm.ts. Stored per
+ * call, not rolled up: LLM traffic is orders of magnitude smaller than
+ * HTTP, and spend analysis needs every call. Tokens and cost are resolved
+ * at ingest so usage/spend queries are plain aggregates.
+ */
+export interface RuntimeLlmCall {
+    service: string;
+    environment: string;
+    release: string | null;
+    traceId: string;
+    spanId: string;
+    /** gen_ai.provider.name / gen_ai.system / ai.model.provider — "openai", … */
+    provider: string;
+    model: string;
+    /** gen_ai.operation.name — "chat", "embeddings", "generateText", … */
+    operation: string;
+    inputTokens: number;
+    outputTokens: number;
+    costUsd: number;
+    /** reported = SDK sent the exact cost; estimated = built-in pricing table. */
+    costSource: "reported" | "estimated" | "none";
+    durationMs: number;
+    status: "ok" | "error";
+    /** Provider exception type for failed calls ("RateLimitError", …); "" when ok. */
+    errorType: string;
+    /** Opaque end-user id for per-user usage/cost attribution. */
+    userId: string;
+    sessionId: string;
+    attributes: Record<string, string> | null;
+    startedAt: Date;
+}
 /** One 60-second usage rollup bucket. */
 export interface RuntimeMetricPoint {
     service: string;

package/package.json

@@ -1,6 +1,6 @@
 {
 	"name": "@autter/otlp-ingester",
-	"version": "0.1.0",
+	"version": "1.1.0",
 	"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
 	"license": "MIT",
 	"type": "module",
@@ -14,7 +14,8 @@
 	"scripts": {
 		"dev": "tsx watch src/index.ts",
 		"start": "node dist/index.js",
-		"build": "tsc -p tsconfig.json"
+		"build": "tsc -p tsconfig.json",
+		"test": "tsx --test src/*.test.ts"
 	},
 	"dependencies": {
 		"@clickhouse/client": "^1.12.0",