@autter/otlp-ingester 1.8.0 → 1.9.0

package/dist/loop-collapse.js

@@ -0,0 +1,326 @@
+import { createHash } from "node:crypto";
+const MINUTE_MS = 60_000;
+/**
+ * The identity two occurrences must share to fold together (see above),
+ * hashed: the raw parts can run to kilobytes (frames, messages).
+ */
+export function collapseIdentity(o) {
+    const parts = [
+        o.fingerprint,
+        (o.fingerprints ?? []).map((candidate) => `${candidate.scheme}:${candidate.fingerprint}`).join(","),
+        o.service,
+        o.environment,
+        o.release ?? "",
+        o.source,
+        o.severity,
+        o.expected ? "1" : "0",
+        o.errorType,
+        o.errorCode ?? "",
+        o.messageNormalized,
+        o.topFrames.join("\n"),
+        o.sessionId ?? "",
+    ];
+    return createHash("sha256").update(parts.join("\u0000")).digest("base64url").slice(0, 32);
+}
+const tenantKey = (ctx) => `${ctx.orgId}\u0000${ctx.repositoryId}`;
+export class LoopCollapser {
+    flushSink;
+    minutes = new Map();
+    /** Folds committed after their window was flushed, and failed flushes. */
+    carry = [];
+    heldSamples = 0;
+    trackedIds = new Map();
+    timer = null;
+    flushing = null;
+    lastFailureLog = Number.NEGATIVE_INFINITY;
+    stats_ = { folded: 0, samplesFlushed: 0, overflow: 0, failedFlushes: 0 };
+    after;
+    maxWindows;
+    maxSamples;
+    maxTrackedIds;
+    flushIntervalMs;
+    forceAfterMs;
+    now;
+    log;
+    constructor(flushSink, options) {
+        this.flushSink = flushSink;
+        this.after = Math.max(0, Math.floor(options.after));
+        this.maxWindows = options.maxWindows ?? 100_000;
+        this.maxSamples = options.maxSamples ?? 5_000;
+        this.maxTrackedIds = options.maxTrackedIds ?? 500_000;
+        this.flushIntervalMs = options.flushIntervalMs ?? 5_000;
+        this.forceAfterMs = options.forceAfterMs ?? 4 * MINUTE_MS;
+        this.now = options.now ?? Date.now;
+        this.log = options.log ?? ((message) => console.warn(message));
+    }
+    get enabled() {
+        return this.after > 0;
+    }
+    /**
+     * Split a request's occurrences into those stored now and those folded
+     * into their window's sample. Synchronous; never throws (on an internal
+     * error every occurrence is kept).
+     */
+    plan(ctx, route, occurrences) {
+        if (!this.enabled || occurrences.length === 0)
+            return passThrough(occurrences);
+        try {
+            return this.planWindows(ctx, route, occurrences);
+        }
+        catch (err) {
+            this.log(`loop collapse skipped for a batch: ${err?.message ?? err}`);
+            return passThrough(occurrences);
+        }
+    }
+    planWindows(ctx, route, occurrences) {
+        const minute = Math.floor(this.now() / MINUTE_MS);
+        let windows = this.minutes.get(minute);
+        if (!windows) {
+            windows = new Map();
+            this.minutes.set(minute, windows);
+        }
+        const kept = [];
+        const keptReservations = [];
+        const folds = [];
+        const touched = new Set();
+        let repeated = 0;
+        const tenant = tenantKey(ctx);
+        for (const o of occurrences) {
+            const key = `${tenant}\u0000${route}\u0000${collapseIdentity(o)}`;
+            let window = windows.get(key);
+            if (!window) {
+                if (windows.size >= this.maxWindows) {
+                    this.stats_.overflow += 1;
+                    kept.push(o);
+                    continue;
+                }
+                window = {
+                    ctx, route, minute, admitted: 0, kept: new Set(), foldedIds: new Set(),
+                    count: 0, sample: null, reserved: 0, pending: 0, holdsSlot: false, flushed: false, late: null,
+                };
+                windows.set(key, window);
+            }
+            if (!touched.has(window)) {
+                touched.add(window);
+                window.pending += 1;
+            }
+            const id = o.occurrenceId;
+            if (window.kept.has(id)) {
+                // A retried batch: stored again with the same id (deduped downstream).
+                kept.push(o);
+                continue;
+            }
+            if (window.foldedIds?.has(id)) {
+                repeated += 1;
+                continue;
+            }
+            if (window.admitted < this.after) {
+                window.admitted += 1;
+                window.kept.add(id);
+                keptReservations.push([window, id]);
+                kept.push(o);
+                continue;
+            }
+            if (!window.holdsSlot) {
+                if (this.heldSamples >= this.maxSamples) {
+                    this.stats_.overflow += 1;
+                    kept.push(o);
+                    continue;
+                }
+                window.holdsSlot = true;
+                this.heldSamples += 1;
+            }
+            window.admitted += 1;
+            window.reserved += 1;
+            if (window.foldedIds) {
+                const tracked = this.trackedIds.get(minute) ?? 0;
+                if (tracked < this.maxTrackedIds) {
+                    window.foldedIds.add(id);
+                    this.trackedIds.set(minute, tracked + 1);
+                }
+                else {
+                    window.foldedIds = null;
+                }
+            }
+            folds.push([window, o]);
+        }
+        let settled = false;
+        const release = () => {
+            for (const window of touched)
+                window.pending -= 1;
+        };
+        return {
+            kept,
+            folded: folds.length,
+            repeated,
+            commit: () => {
+                if (settled)
+                    return;
+                settled = true;
+                for (const [window, o] of folds) {
+                    window.reserved -= 1;
+                    if (window.flushed) {
+                        // Its minute was flushed while this request ran: carry the fold
+                        // (with every other late one of the window) into the next flush.
+                        if (!window.late) {
+                            window.late = { ctx: window.ctx, route: window.route, sample: o, count: 0, window: null, lateOf: window };
+                            this.carry.push(window.late);
+                        }
+                        window.late.count += 1;
+                        if (o.occurredAt.getTime() >= window.late.sample.occurredAt.getTime())
+                            window.late.sample = o;
+                        continue;
+                    }
+                    window.count += 1;
+                    if (!window.sample || o.occurredAt.getTime() >= window.sample.occurredAt.getTime())
+                        window.sample = o;
+                }
+                this.stats_.folded += folds.length;
+                release();
+            },
+            rollback: () => {
+                if (settled)
+                    return;
+                settled = true;
+                for (const [window, id] of keptReservations) {
+                    window.admitted -= 1;
+                    window.kept.delete(id);
+                }
+                for (const [window, o] of folds) {
+                    window.admitted -= 1;
+                    window.reserved -= 1;
+                    window.foldedIds?.delete(o.occurrenceId);
+                    // Nothing folded after all: free the sample slot. (A flushed
+                    // window's slot was freed by the flush, or is by its sample.)
+                    if (!window.flushed && window.count === 0 && window.reserved === 0)
+                        this.releaseSlot(window);
+                }
+                release();
+            },
+        };
+    }
+    /** Flush due windows every flushIntervalMs (unref'd). */
+    start() {
+        if (!this.enabled || this.timer)
+            return;
+        this.timer = setInterval(() => void this.flush(), this.flushIntervalMs);
+        this.timer.unref?.();
+    }
+    /** Stop the timer and flush every window, the current minute's too (shutdown). */
+    async stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        await this.flush(true);
+    }
+    /**
+     * Store the samples of every window whose minute is over and that no
+     * request still holds, plus carried folds and earlier failed writes.
+     * `all`: every window, current minute included. One flush at a time;
+     * never rejects.
+     */
+    flush(all = false) {
+        if (this.flushing) {
+            if (!all)
+                return this.flushing;
+            return this.flushing.then(() => this.flush(true));
+        }
+        this.flushing = this.flushDue(all).finally(() => {
+            this.flushing = null;
+        });
+        return this.flushing;
+    }
+    collectDue(all) {
+        const due = this.carry;
+        this.carry = [];
+        // Late folds committed from now on start a new carried sample.
+        for (const item of due)
+            if (item.lateOf?.late === item)
+                item.lateOf.late = null;
+        const nowMs = this.now();
+        const current = Math.floor(nowMs / MINUTE_MS);
+        for (const [minute, windows] of this.minutes) {
+            if (!all && minute >= current)
+                continue;
+            const forced = all || nowMs - (minute + 1) * MINUTE_MS >= this.forceAfterMs;
+            let open = 0;
+            for (const window of windows.values()) {
+                if (window.flushed)
+                    continue;
+                if (window.pending > 0 && !forced) {
+                    open += 1;
+                    continue;
+                }
+                window.flushed = true;
+                if (window.count > 0 && window.sample) {
+                    due.push({ ctx: window.ctx, route: window.route, sample: window.sample, count: window.count, window, lateOf: null });
+                }
+                else {
+                    // Nothing to store; folds still in flight are carried without a slot.
+                    this.releaseSlot(window);
+                }
+            }
+            if (open === 0) {
+                this.minutes.delete(minute);
+                this.trackedIds.delete(minute);
+            }
+        }
+        return due;
+    }
+    async flushDue(all) {
+        let due;
+        try {
+            due = this.collectDue(all);
+        }
+        catch (err) {
+            this.log(`loop collapse flush failed: ${err?.message ?? err}`);
+            return;
+        }
+        if (due.length === 0)
+            return;
+        const byTenant = new Map();
+        for (const item of due) {
+            const key = tenantKey(item.ctx);
+            byTenant.set(key, [...(byTenant.get(key) ?? []), item]);
+        }
+        for (const items of byTenant.values()) {
+            const samples = items.map((item) => ({
+                route: item.route,
+                sample: { ...item.sample, collapsedCount: item.count - 1 },
+                folded: item.count,
+            }));
+            try {
+                await this.flushSink(items[0].ctx, samples);
+                this.stats_.samplesFlushed += samples.length;
+                for (const item of items)
+                    if (item.window)
+                        this.releaseSlot(item.window);
+            }
+            catch (err) {
+                this.stats_.failedFlushes += 1;
+                // Kept (with their slots) and written again next tick.
+                this.carry.push(...items);
+                if (this.now() - this.lastFailureLog >= MINUTE_MS) {
+                    this.lastFailureLog = this.now();
+                    this.log(`loop collapse: ${samples.length} collapsed sample(s) not stored yet, retrying: ${err?.message ?? err}`);
+                }
+            }
+        }
+    }
+    releaseSlot(window) {
+        if (!window.holdsSlot)
+            return;
+        window.holdsSlot = false;
+        this.heldSamples -= 1;
+    }
+    stats() {
+        let windows = 0;
+        for (const minute of this.minutes.values())
+            windows += minute.size;
+        return { windows, heldSamples: this.heldSamples, ...this.stats_ };
+    }
+}
+function passThrough(occurrences) {
+    return { kept: occurrences, folded: 0, repeated: 0, commit() { }, rollback() { } };
+}

package/dist/migrations.d.ts

@@ -53,6 +53,18 @@ export declare const OCCURRENCE_GROUPING_COLUMNS: string[];
  * before 0020.
  */
 export declare const OCCURRENCE_FRAME_RESOLUTION_COLUMNS: string[];
+/**
+ * Migration 0021 column — shared with the baseline DDL in clickhouse.ts.
+ * Loop collapse (1.9.0, loop-collapse.ts): the identical occurrences a
+ * minute's collapsed sample stands for besides itself, whose payloads were
+ * not stored; 0 on every other row. An exact occurrence count is, per
+ * distinct `occurrence_id` (an exporter retry can store a row twice),
+ * 1 + collapsed_count: `SELECT sum(1 + c) FROM (SELECT occurrence_id,
+ * max(collapsed_count) AS c … GROUP BY occurrence_id)`. A metadata-only
+ * ALTER with a constant default: rows written before it, and by older
+ * replicas during a rolling deploy, read 0.
+ */
+export declare const OCCURRENCE_COLLAPSE_COLUMNS: string[];
 export interface Migration {
     /** Unique, ordered id: "<serial>-<slug>". Never reuse or reorder. */
     id: string;

package/dist/migrations.js

@@ -74,6 +74,18 @@ export const OCCURRENCE_FRAME_RESOLUTION_COLUMNS = [
     "in_app_frames_resolved UInt16 DEFAULT 0",
     "in_app_frames_unresolved UInt16 DEFAULT 0",
 ];
+/**
+ * Migration 0021 column — shared with the baseline DDL in clickhouse.ts.
+ * Loop collapse (1.9.0, loop-collapse.ts): the identical occurrences a
+ * minute's collapsed sample stands for besides itself, whose payloads were
+ * not stored; 0 on every other row. An exact occurrence count is, per
+ * distinct `occurrence_id` (an exporter retry can store a row twice),
+ * 1 + collapsed_count: `SELECT sum(1 + c) FROM (SELECT occurrence_id,
+ * max(collapsed_count) AS c … GROUP BY occurrence_id)`. A metadata-only
+ * ALTER with a constant default: rows written before it, and by older
+ * replicas during a rolling deploy, read 0.
+ */
+export const OCCURRENCE_COLLAPSE_COLUMNS = ["collapsed_count UInt32 DEFAULT 0"];
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -230,6 +242,11 @@ export const MIGRATIONS = [
         id: "0020-occurrence-frame-resolution",
         statements: OCCURRENCE_FRAME_RESOLUTION_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
     },
+    // Loop collapse (1.9.0): the count a minute's collapsed sample carries.
+    {
+        id: "0021-occurrence-collapsed-count",
+        statements: OCCURRENCE_COLLAPSE_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
+    },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {

package/dist/normalize-browser.d.ts

@@ -53,8 +53,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         timestamp: string;
         breadcrumbs?: unknown[] | undefined;
         route?: string | undefined;
-        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
+        id?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         stack?: string | undefined;
         traceId?: string | undefined;
         debugIds?: Record<string, unknown> | undefined;
@@ -63,7 +64,6 @@ export declare const browserPayloadSchema: z.ZodObject<{
         name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
     }, {
@@ -72,8 +72,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         breadcrumbs?: unknown;
         route?: string | undefined;
         message?: string | undefined;
-        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
+        id?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         stack?: string | undefined;
         traceId?: unknown;
         debugIds?: unknown;
@@ -82,7 +83,6 @@ export declare const browserPayloadSchema: z.ZodObject<{
         name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
     }>, "many">;
@@ -103,8 +103,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         timestamp: string;
         breadcrumbs?: unknown[] | undefined;
         route?: string | undefined;
-        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
+        id?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         stack?: string | undefined;
         traceId?: string | undefined;
         debugIds?: Record<string, unknown> | undefined;
@@ -113,7 +114,6 @@ export declare const browserPayloadSchema: z.ZodObject<{
         name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: string | string[] | undefined;
     }[];
@@ -130,8 +130,9 @@ export declare const browserPayloadSchema: z.ZodObject<{
         breadcrumbs?: unknown;
         route?: string | undefined;
         message?: string | undefined;
-        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         errorType?: string | undefined;
+        id?: string | undefined;
+        severity?: "info" | "warning" | "error" | "fatal" | undefined;
         stack?: string | undefined;
         traceId?: unknown;
         debugIds?: unknown;
@@ -140,7 +141,6 @@ export declare const browserPayloadSchema: z.ZodObject<{
         name?: string | undefined;
         column?: number | undefined;
         line?: number | undefined;
-        id?: string | undefined;
         context?: Record<string, unknown> | undefined;
         fingerprint?: unknown;
     }[];

package/dist/server.d.ts

@@ -3,6 +3,7 @@ import { IngestDropCounter } from "./ingest-drops.js";
 import { ClickHouseStore } from "./clickhouse.js";
 import type { IngesterConfig } from "./config.js";
 import { SinkForwarder } from "./sink.js";
+import { LoopCollapser } from "./loop-collapse.js";
 export interface IngesterApp {
     app: Express;
     store: ClickHouseStore;
@@ -13,6 +14,8 @@ export interface IngesterApp {
     reportIngesterInfo: () => Promise<void>;
     /** Rejection and drop counts per tenant (runtime_ingest_drops); `stop()` writes what is buffered. */
     drops: IngestDropCounter;
+    /** Loop collapse (1.9.0): `stop()` stores every window's sample (call it before `drops.stop()`). */
+    collapse: LoopCollapser;
 }
 /** The only routes an upload key (`autter_rtu_…`, scope "upload") may call. */
 export declare const UPLOAD_KEY_ROUTES: readonly ["/v1/sourcemaps", "/v1/sourcemaps/check"];

package/dist/server.js

@@ -17,6 +17,8 @@ import { decodeProfile } from "./profiles.js";
 import { MAX_MEMORY_SAMPLES_PER_REQUEST, normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema, } from "./memory.js";
 import { SOURCE_MAP_MAX_BODY_BYTES, validateSourceMap, validateSourceMapCheck } from "./source-maps.js";
 import { SinkForwarder } from "./sink.js";
+import { LoopCollapser } from "./loop-collapse.js";
+import { applySuppression } from "./suppression.js";
 import { configureRedaction, scrubOccurrence } from "./redact.js";
 import { buildCompatReport, evaluateCompat, INGESTER_VERSION_HEADER, } from "./compat.js";
 import { MIGRATIONS } from "./migrations.js";
@@ -71,6 +73,46 @@ export function createIngesterApp(config) {
             records: drop.occurrences,
         });
     }
+    // Loop collapse (1.9.0, loop-collapse.ts): a looping error past its first
+    // LOOP_COLLAPSE_AFTER occurrences of the minute is stored as one sample a
+    // minute with an exact count. The minute's samples are written here, then
+    // forwarded; a failed write is kept and retried by the collapser.
+    const collapse = new LoopCollapser(async (ctx, samples) => {
+        const occurrences = samples.map((entry) => entry.sample);
+        await store.insertOccurrences(ctx, occurrences);
+        sink?.enqueue(ctx, occurrences);
+        for (const entry of samples) {
+            // Payloads not stored: every folded occurrence but the sample itself.
+            if (entry.folded > 1) {
+                drops.record({ tenant: ctx, route: entry.route, reason: "collapsed", requests: 0, records: entry.folded - 1 });
+            }
+        }
+    }, { after: config.loopCollapseAfter ?? 10 });
+    if (store.configured)
+        collapse.start();
+    /**
+     * Volume control on a request's fingerprinted occurrences (1.9.0): the
+     * tenant's suppression rules first, then loop collapse. `kept` is what to
+     * store and forward now. `commit()` once the request's writes succeeded
+     * (folds and suppressions are counted then), `rollback()` when they
+     * failed (the exporter retries, and must find the same windows).
+     */
+    function admit(ctx, req, occurrences) {
+        const route = dropRoute(req.path);
+        const { kept, suppressed } = applySuppression(occurrences, ctx.suppression);
+        const plan = collapse.plan(ctx, route, kept);
+        return {
+            kept: plan.kept,
+            commit() {
+                plan.commit();
+                if (suppressed > 0)
+                    drops.record({ tenant: ctx, route, reason: "suppressed", requests: 0, records: suppressed });
+            },
+            rollback() {
+                plan.rollback();
+            },
+        };
+    }
     const allMigrationIds = MIGRATIONS.map((migration) => migration.id);
     const compatReport = () => {
         const schema = store.schemaStatus();
@@ -231,7 +273,11 @@ export function createIngesterApp(config) {
         next();
     });
     app.get("/healthz", async (_req, res) => {
-        const sinkStats = sink ? { sink: sink.stats() } : {};
+        const sinkStats = {
+            ...(sink ? { sink: sink.stats() } : {}),
+            // Counters only (no tenant data): /healthz is public.
+            ...(collapse.enabled ? { collapse: collapse.stats() } : {}),
+        };
         if (!store.configured) {
             res.status(200).json({ ok: true, clickhouse: "unconfigured", ...sinkStats });
             return;
@@ -593,6 +639,7 @@ export function createIngesterApp(config) {
         }
         const { occurrences, occurrenceBatchIndexes, spans, metricPoints, llmCalls } = normalized;
         const fingerprinted = await fingerprintAll(ctx, occurrences, occurrenceBatchIndexes);
+        const admitted = admit(ctx, req, fingerprinted);
         // ClickHouse has no cross-table transaction, so these four inserts can
         // partially commit. Recovery boundary: any failure → 503 → the exporter
         // retries the whole batch. Deterministic occurrence ids make the retry
@@ -603,17 +650,19 @@ export function createIngesterApp(config) {
         // ClickHouse reconciliation instead.
         try {
             await Promise.all([
-                store.insertOccurrences(ctx, fingerprinted),
+                store.insertOccurrences(ctx, admitted.kept),
                 store.insertSpans(ctx, spans),
                 store.insertMetricPoints(ctx, metricPoints),
                 store.insertLlmCalls(ctx, llmCalls),
             ]);
         }
         catch (err) {
+            admitted.rollback();
             storageError(req, res, ctx, err, totalSpans);
             return;
         }
-        sink?.enqueue(ctx, fingerprinted, metricPoints, llmCalls);
+        admitted.commit();
+        sink?.enqueue(ctx, admitted.kept, metricPoints, llmCalls);
         recordSdks(ctx, request, req);
         // normalizeTraces stops at SPANS_PER_REQUEST; the rest were not read.
         const overLimit = Math.max(0, totalSpans - normalized.spanCount);
@@ -650,19 +699,23 @@ export function createIngesterApp(config) {
         // edge SDKs mark span-less error records `autter.capture.mode = "log"`,
         // and those are promoted to occurrences here — see log-promotion.ts.
         const promoted = await promoteLogs(ctx, logs);
+        // Rollups count every promoted error, collapsed or suppressed ones too.
         const metricPoints = promotionRollups(promoted);
+        const admitted = admit(ctx, req, promoted);
         try {
             await Promise.all([
                 store.insertLogs(ctx, logs),
-                store.insertOccurrences(ctx, promoted),
+                store.insertOccurrences(ctx, admitted.kept),
                 store.insertMetricPoints(ctx, metricPoints),
             ]);
         }
         catch (err) {
+            admitted.rollback();
             storageError(req, res, ctx, err, logs.length + stats.overLimit);
             return;
         }
-        sink?.enqueue(ctx, promoted, metricPoints);
+        admitted.commit();
+        sink?.enqueue(ctx, admitted.kept, metricPoints);
         recordSdks(ctx, logsRequest, req);
         otlpSuccess(req, res, "logs", partialSuccess([
             { rejected: stats.overLimit, message: PARTIAL_MESSAGES.logRecords(stats.overLimit, MAX_LOG_RECORDS_PER_REQUEST) },
@@ -770,17 +823,20 @@ export function createIngesterApp(config) {
         }
         const { occurrences, metricPoints } = normalized;
         const fingerprinted = await fingerprintAll(ctx, occurrences);
+        const admitted = admit(ctx, req, fingerprinted);
         try {
             await Promise.all([
-                store.insertOccurrences(ctx, fingerprinted),
+                store.insertOccurrences(ctx, admitted.kept),
                 store.insertMetricPoints(ctx, metricPoints),
             ]);
         }
         catch (err) {
+            admitted.rollback();
             storageError(req, res, ctx, err, payload.events.length + invalidEvents + overLimitEvents);
             return;
         }
-        sink?.enqueue(ctx, fingerprinted, metricPoints);
+        admitted.commit();
+        sink?.enqueue(ctx, admitted.kept, metricPoints);
         if (payload.sdk) {
             void sdkVersions.observe(ctx, [{
                     service: payload.service,
@@ -828,5 +884,5 @@ export function createIngesterApp(config) {
         res.status(500).json({ error: "internal error" });
         countUnrouted(req, "internal_error", 500);
     });
-    return { app, store, sink, reportIngesterInfo, drops };
+    return { app, store, sink, reportIngesterInfo, drops, collapse };
 }

package/dist/sink.d.ts

@@ -124,7 +124,10 @@ export declare class SinkForwarder {
  * 1.6.0 adds `frames` (innermost first, ≤ 50), `fingerprints` (every
  * identity candidate, highest precedence first, ending with the legacy
  * `fingerprint`) and `fingerprintRecord`; `fingerprint`/`fingerprintScheme`
- * keep their legacy meaning. Older consumers ignore unknown keys, so the
- * version stays 1.
+ * keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
+ * record stands for, sent only when above 1 (a loop-collapse sample:
+ * 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
+ * their event counts. Older consumers ignore unknown keys, so the version
+ * stays 1.
  */
 export declare function sinkOccurrence(o: RuntimeOccurrence): Record<string, unknown>;

package/dist/sink.js

@@ -319,13 +319,16 @@ export class SinkForwarder {
  * 1.6.0 adds `frames` (innermost first, ≤ 50), `fingerprints` (every
  * identity candidate, highest precedence first, ending with the legacy
  * `fingerprint`) and `fingerprintRecord`; `fingerprint`/`fingerprintScheme`
- * keep their legacy meaning. Older consumers ignore unknown keys, so the
- * version stays 1.
+ * keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
+ * record stands for, sent only when above 1 (a loop-collapse sample:
+ * 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
+ * their event counts. Older consumers ignore unknown keys, so the version
+ * stays 1.
  */
 export function sinkOccurrence(o) {
     const { traceId, route, method, statusCode, errorCode, why, fix, link, expected, requestId, breadcrumbs, frames, fingerprints, fingerprintRecord, 
     // Input-only: locates source maps, never leaves the ingester.
-    debugIds: _debugIds, ...rest } = o;
+    debugIds: _debugIds, collapsedCount, ...rest } = o;
     return {
         ...rest,
         occurredAt: o.occurredAt.toISOString(),
@@ -344,6 +347,8 @@ export function sinkOccurrence(o) {
         ...(frames?.length ? { frames } : {}),
         ...(fingerprints?.length ? { fingerprints } : {}),
         ...(fingerprintRecord ? { fingerprintRecord } : {}),
+        // 1.9.0: a collapsed sample stands for 1 + collapsedCount occurrences.
+        ...(collapsedCount && collapsedCount > 0 ? { count: 1 + collapsedCount } : {}),
     };
 }
 /** ISO range across every signal in the batch — the replay hint on drops. */