@autter/otlp-ingester 1.7.0 → 1.9.0

package/dist/loop-collapse.d.ts

@@ -0,0 +1,144 @@
+import type { DropRoute } from "./ingest-drops.js";
+import type { IngestContext, RuntimeOccurrence } from "./types.js";
+/**
+ * Loop collapse (1.9.0, EI-38): an error that repeats in a loop is stored
+ * as its first N occurrences of the minute plus ONE sample that carries the
+ * exact count of the rest, so a loop of 10,000 events a minute costs N + 1
+ * rows a minute, not 10,000, and only duplicate payloads are lost.
+ *
+ * Identical: same tenant, ingest route and arrival minute, and the same
+ * - identity: legacy `fingerprint` and every `fingerprints` candidate;
+ * - service, environment, release, source, severity, `expected`;
+ * - error type and declared error code;
+ * - normalised message (`messageNormalized`: ids, numbers and quoted
+ *   strings templated out) and normalised top frames (`topFrames`);
+ * - session id: an error hitting many sessions is volume, not a loop, and
+ *   each session must still count as a user.
+ * Trace, request and occurrence ids, route, timestamps, attributes and
+ * breadcrumbs may differ: they are what a loop varies.
+ *
+ * Per window (that identity in one arrival minute): the first `after`
+ * occurrences (LOOP_COLLAPSE_AFTER, default 10) are stored and forwarded as
+ * they arrive. Every later one is folded: not stored on its own, counted.
+ * When the minute is over, the window's newest folded occurrence is stored
+ * and forwarded as the sample with `collapsedCount` = folded − 1, so the
+ * sample stands for every folded occurrence: stored rows + Σ collapsed_count
+ * is exactly the number of occurrences accepted. Its time is the newest
+ * folded one, so an issue's last-seen stays exact; its weight lands in the
+ * minute it arrived in.
+ *
+ * Exactness under retries: a request's folds are reserved when it is planned
+ * and only counted once its ClickHouse writes succeed (`commit`); a request
+ * that fails (503, the exporter retries) gives them back (`rollback`). Ids
+ * already seen in the window are recognised, so a batch retried after a lost
+ * 2xx is not counted twice. A window is flushed only once no request still
+ * holds it (or, as a safety net, five minutes after its minute); a fold that
+ * commits after that is carried into the next flush. A failed flush write is
+ * kept and retried every tick. Shutdown (SIGTERM) flushes every window, so a
+ * deploy loses nothing; only a crash loses the folded counts of the minute in
+ * progress.
+ *
+ * Memory is bounded (safety, not a work cap): windows live for their minute
+ * and are cleared on the next flush; past `maxWindows` windows a minute, or
+ * `maxSamples` held samples, occurrences of new identities pass through
+ * uncollapsed (stored as usual, never lost), and past `maxTrackedIds` folded
+ * ids a minute, retry recognition stops for new folds.
+ */
+export interface LoopCollapseOptions {
+    /** Occurrences stored per window before folding starts; 0 turns collapse off. */
+    after: number;
+    /** Windows (identities) per minute. Default 100,000. */
+    maxWindows?: number;
+    /** Samples held across all windows until they are flushed. Default 5,000. */
+    maxSamples?: number;
+    /** Folded occurrence ids remembered per minute, for retry recognition. Default 500,000. */
+    maxTrackedIds?: number;
+    /** How often due windows are flushed. Default 5 s. */
+    flushIntervalMs?: number;
+    /** A window still held by a request is flushed anyway this long after its minute. Default 4 min. */
+    forceAfterMs?: number;
+    now?: () => number;
+    log?: (message: string) => void;
+}
+/** One window's sample, ready to store: `sample.collapsedCount` = folded − 1. */
+export interface CollapsedSample {
+    route: DropRoute;
+    sample: RuntimeOccurrence;
+    /** Occurrences the sample stands for (itself included). */
+    folded: number;
+}
+/** Store and forward one tenant's samples. A rejection keeps them for the next tick. */
+export type CollapseFlush = (ctx: IngestContext, samples: CollapsedSample[]) => Promise<void>;
+export interface CollapsePlan {
+    /** Occurrences to store and forward now. */
+    kept: RuntimeOccurrence[];
+    /** Occurrences folded into a window's sample by this request. */
+    folded: number;
+    /** Occurrences recognised as already counted (a retried batch); not stored again. */
+    repeated: number;
+    /** The request's writes succeeded: count its folds. */
+    commit(): void;
+    /** The request failed: give its reservations back. */
+    rollback(): void;
+}
+export interface LoopCollapseStats {
+    windows: number;
+    heldSamples: number;
+    /** Occurrences folded (committed) since start. */
+    folded: number;
+    /** Samples stored since start. */
+    samplesFlushed: number;
+    /** Occurrences that passed through uncollapsed because a bound was reached. */
+    overflow: number;
+    /** Flush writes that failed (their samples were kept and retried). */
+    failedFlushes: number;
+}
+/**
+ * The identity two occurrences must share to fold together (see above),
+ * hashed: the raw parts can run to kilobytes (frames, messages).
+ */
+export declare function collapseIdentity(o: RuntimeOccurrence): string;
+export declare class LoopCollapser {
+    private readonly flushSink;
+    private readonly minutes;
+    /** Folds committed after their window was flushed, and failed flushes. */
+    private carry;
+    private heldSamples;
+    private trackedIds;
+    private timer;
+    private flushing;
+    private lastFailureLog;
+    private stats_;
+    private readonly after;
+    private readonly maxWindows;
+    private readonly maxSamples;
+    private readonly maxTrackedIds;
+    private readonly flushIntervalMs;
+    private readonly forceAfterMs;
+    private readonly now;
+    private readonly log;
+    constructor(flushSink: CollapseFlush, options: LoopCollapseOptions);
+    get enabled(): boolean;
+    /**
+     * Split a request's occurrences into those stored now and those folded
+     * into their window's sample. Synchronous; never throws (on an internal
+     * error every occurrence is kept).
+     */
+    plan(ctx: IngestContext, route: DropRoute, occurrences: RuntimeOccurrence[]): CollapsePlan;
+    private planWindows;
+    /** Flush due windows every flushIntervalMs (unref'd). */
+    start(): void;
+    /** Stop the timer and flush every window, the current minute's too (shutdown). */
+    stop(): Promise<void>;
+    /**
+     * Store the samples of every window whose minute is over and that no
+     * request still holds, plus carried folds and earlier failed writes.
+     * `all`: every window, current minute included. One flush at a time;
+     * never rejects.
+     */
+    flush(all?: boolean): Promise<void>;
+    private collectDue;
+    private flushDue;
+    private releaseSlot;
+    stats(): LoopCollapseStats;
+}

package/dist/loop-collapse.js

@@ -0,0 +1,326 @@
+import { createHash } from "node:crypto";
+const MINUTE_MS = 60_000;
+/**
+ * The identity two occurrences must share to fold together (see above),
+ * hashed: the raw parts can run to kilobytes (frames, messages).
+ */
+export function collapseIdentity(o) {
+    const parts = [
+        o.fingerprint,
+        (o.fingerprints ?? []).map((candidate) => `${candidate.scheme}:${candidate.fingerprint}`).join(","),
+        o.service,
+        o.environment,
+        o.release ?? "",
+        o.source,
+        o.severity,
+        o.expected ? "1" : "0",
+        o.errorType,
+        o.errorCode ?? "",
+        o.messageNormalized,
+        o.topFrames.join("\n"),
+        o.sessionId ?? "",
+    ];
+    return createHash("sha256").update(parts.join("\u0000")).digest("base64url").slice(0, 32);
+}
+const tenantKey = (ctx) => `${ctx.orgId}\u0000${ctx.repositoryId}`;
+export class LoopCollapser {
+    flushSink;
+    minutes = new Map();
+    /** Folds committed after their window was flushed, and failed flushes. */
+    carry = [];
+    heldSamples = 0;
+    trackedIds = new Map();
+    timer = null;
+    flushing = null;
+    lastFailureLog = Number.NEGATIVE_INFINITY;
+    stats_ = { folded: 0, samplesFlushed: 0, overflow: 0, failedFlushes: 0 };
+    after;
+    maxWindows;
+    maxSamples;
+    maxTrackedIds;
+    flushIntervalMs;
+    forceAfterMs;
+    now;
+    log;
+    constructor(flushSink, options) {
+        this.flushSink = flushSink;
+        this.after = Math.max(0, Math.floor(options.after));
+        this.maxWindows = options.maxWindows ?? 100_000;
+        this.maxSamples = options.maxSamples ?? 5_000;
+        this.maxTrackedIds = options.maxTrackedIds ?? 500_000;
+        this.flushIntervalMs = options.flushIntervalMs ?? 5_000;
+        this.forceAfterMs = options.forceAfterMs ?? 4 * MINUTE_MS;
+        this.now = options.now ?? Date.now;
+        this.log = options.log ?? ((message) => console.warn(message));
+    }
+    get enabled() {
+        return this.after > 0;
+    }
+    /**
+     * Split a request's occurrences into those stored now and those folded
+     * into their window's sample. Synchronous; never throws (on an internal
+     * error every occurrence is kept).
+     */
+    plan(ctx, route, occurrences) {
+        if (!this.enabled || occurrences.length === 0)
+            return passThrough(occurrences);
+        try {
+            return this.planWindows(ctx, route, occurrences);
+        }
+        catch (err) {
+            this.log(`loop collapse skipped for a batch: ${err?.message ?? err}`);
+            return passThrough(occurrences);
+        }
+    }
+    planWindows(ctx, route, occurrences) {
+        const minute = Math.floor(this.now() / MINUTE_MS);
+        let windows = this.minutes.get(minute);
+        if (!windows) {
+            windows = new Map();
+            this.minutes.set(minute, windows);
+        }
+        const kept = [];
+        const keptReservations = [];
+        const folds = [];
+        const touched = new Set();
+        let repeated = 0;
+        const tenant = tenantKey(ctx);
+        for (const o of occurrences) {
+            const key = `${tenant}\u0000${route}\u0000${collapseIdentity(o)}`;
+            let window = windows.get(key);
+            if (!window) {
+                if (windows.size >= this.maxWindows) {
+                    this.stats_.overflow += 1;
+                    kept.push(o);
+                    continue;
+                }
+                window = {
+                    ctx, route, minute, admitted: 0, kept: new Set(), foldedIds: new Set(),
+                    count: 0, sample: null, reserved: 0, pending: 0, holdsSlot: false, flushed: false, late: null,
+                };
+                windows.set(key, window);
+            }
+            if (!touched.has(window)) {
+                touched.add(window);
+                window.pending += 1;
+            }
+            const id = o.occurrenceId;
+            if (window.kept.has(id)) {
+                // A retried batch: stored again with the same id (deduped downstream).
+                kept.push(o);
+                continue;
+            }
+            if (window.foldedIds?.has(id)) {
+                repeated += 1;
+                continue;
+            }
+            if (window.admitted < this.after) {
+                window.admitted += 1;
+                window.kept.add(id);
+                keptReservations.push([window, id]);
+                kept.push(o);
+                continue;
+            }
+            if (!window.holdsSlot) {
+                if (this.heldSamples >= this.maxSamples) {
+                    this.stats_.overflow += 1;
+                    kept.push(o);
+                    continue;
+                }
+                window.holdsSlot = true;
+                this.heldSamples += 1;
+            }
+            window.admitted += 1;
+            window.reserved += 1;
+            if (window.foldedIds) {
+                const tracked = this.trackedIds.get(minute) ?? 0;
+                if (tracked < this.maxTrackedIds) {
+                    window.foldedIds.add(id);
+                    this.trackedIds.set(minute, tracked + 1);
+                }
+                else {
+                    window.foldedIds = null;
+                }
+            }
+            folds.push([window, o]);
+        }
+        let settled = false;
+        const release = () => {
+            for (const window of touched)
+                window.pending -= 1;
+        };
+        return {
+            kept,
+            folded: folds.length,
+            repeated,
+            commit: () => {
+                if (settled)
+                    return;
+                settled = true;
+                for (const [window, o] of folds) {
+                    window.reserved -= 1;
+                    if (window.flushed) {
+                        // Its minute was flushed while this request ran: carry the fold
+                        // (with every other late one of the window) into the next flush.
+                        if (!window.late) {
+                            window.late = { ctx: window.ctx, route: window.route, sample: o, count: 0, window: null, lateOf: window };
+                            this.carry.push(window.late);
+                        }
+                        window.late.count += 1;
+                        if (o.occurredAt.getTime() >= window.late.sample.occurredAt.getTime())
+                            window.late.sample = o;
+                        continue;
+                    }
+                    window.count += 1;
+                    if (!window.sample || o.occurredAt.getTime() >= window.sample.occurredAt.getTime())
+                        window.sample = o;
+                }
+                this.stats_.folded += folds.length;
+                release();
+            },
+            rollback: () => {
+                if (settled)
+                    return;
+                settled = true;
+                for (const [window, id] of keptReservations) {
+                    window.admitted -= 1;
+                    window.kept.delete(id);
+                }
+                for (const [window, o] of folds) {
+                    window.admitted -= 1;
+                    window.reserved -= 1;
+                    window.foldedIds?.delete(o.occurrenceId);
+                    // Nothing folded after all: free the sample slot. (A flushed
+                    // window's slot was freed by the flush, or is by its sample.)
+                    if (!window.flushed && window.count === 0 && window.reserved === 0)
+                        this.releaseSlot(window);
+                }
+                release();
+            },
+        };
+    }
+    /** Flush due windows every flushIntervalMs (unref'd). */
+    start() {
+        if (!this.enabled || this.timer)
+            return;
+        this.timer = setInterval(() => void this.flush(), this.flushIntervalMs);
+        this.timer.unref?.();
+    }
+    /** Stop the timer and flush every window, the current minute's too (shutdown). */
+    async stop() {
+        if (this.timer) {
+            clearInterval(this.timer);
+            this.timer = null;
+        }
+        await this.flush(true);
+    }
+    /**
+     * Store the samples of every window whose minute is over and that no
+     * request still holds, plus carried folds and earlier failed writes.
+     * `all`: every window, current minute included. One flush at a time;
+     * never rejects.
+     */
+    flush(all = false) {
+        if (this.flushing) {
+            if (!all)
+                return this.flushing;
+            return this.flushing.then(() => this.flush(true));
+        }
+        this.flushing = this.flushDue(all).finally(() => {
+            this.flushing = null;
+        });
+        return this.flushing;
+    }
+    collectDue(all) {
+        const due = this.carry;
+        this.carry = [];
+        // Late folds committed from now on start a new carried sample.
+        for (const item of due)
+            if (item.lateOf?.late === item)
+                item.lateOf.late = null;
+        const nowMs = this.now();
+        const current = Math.floor(nowMs / MINUTE_MS);
+        for (const [minute, windows] of this.minutes) {
+            if (!all && minute >= current)
+                continue;
+            const forced = all || nowMs - (minute + 1) * MINUTE_MS >= this.forceAfterMs;
+            let open = 0;
+            for (const window of windows.values()) {
+                if (window.flushed)
+                    continue;
+                if (window.pending > 0 && !forced) {
+                    open += 1;
+                    continue;
+                }
+                window.flushed = true;
+                if (window.count > 0 && window.sample) {
+                    due.push({ ctx: window.ctx, route: window.route, sample: window.sample, count: window.count, window, lateOf: null });
+                }
+                else {
+                    // Nothing to store; folds still in flight are carried without a slot.
+                    this.releaseSlot(window);
+                }
+            }
+            if (open === 0) {
+                this.minutes.delete(minute);
+                this.trackedIds.delete(minute);
+            }
+        }
+        return due;
+    }
+    async flushDue(all) {
+        let due;
+        try {
+            due = this.collectDue(all);
+        }
+        catch (err) {
+            this.log(`loop collapse flush failed: ${err?.message ?? err}`);
+            return;
+        }
+        if (due.length === 0)
+            return;
+        const byTenant = new Map();
+        for (const item of due) {
+            const key = tenantKey(item.ctx);
+            byTenant.set(key, [...(byTenant.get(key) ?? []), item]);
+        }
+        for (const items of byTenant.values()) {
+            const samples = items.map((item) => ({
+                route: item.route,
+                sample: { ...item.sample, collapsedCount: item.count - 1 },
+                folded: item.count,
+            }));
+            try {
+                await this.flushSink(items[0].ctx, samples);
+                this.stats_.samplesFlushed += samples.length;
+                for (const item of items)
+                    if (item.window)
+                        this.releaseSlot(item.window);
+            }
+            catch (err) {
+                this.stats_.failedFlushes += 1;
+                // Kept (with their slots) and written again next tick.
+                this.carry.push(...items);
+                if (this.now() - this.lastFailureLog >= MINUTE_MS) {
+                    this.lastFailureLog = this.now();
+                    this.log(`loop collapse: ${samples.length} collapsed sample(s) not stored yet, retrying: ${err?.message ?? err}`);
+                }
+            }
+        }
+    }
+    releaseSlot(window) {
+        if (!window.holdsSlot)
+            return;
+        window.holdsSlot = false;
+        this.heldSamples -= 1;
+    }
+    stats() {
+        let windows = 0;
+        for (const minute of this.minutes.values())
+            windows += minute.size;
+        return { windows, heldSamples: this.heldSamples, ...this.stats_ };
+    }
+}
+function passThrough(occurrences) {
+    return { kept: occurrences, folded: 0, repeated: 0, commit() { }, rollback() { } };
+}

package/dist/memory.d.ts

@@ -11,7 +11,15 @@ export interface MemorySample {
     observedAt: Date;
     temporality: "gauge" | "delta" | "cumulative";
 }
-export declare function normalizeMemoryMetrics(request: OtlpMetricsRequest): MemorySample[];
+/** Memory/GC samples one metrics request may produce; past it, points are dropped and counted. */
+export declare const MAX_MEMORY_SAMPLES_PER_REQUEST = 5000;
+export interface MemoryNormalizeStats {
+    /** Memory/GC points with a value, timestamp or byte unit out of range, or on a resource without service.name and service.instance.id. */
+    invalid: number;
+    /** Memory/GC points past MAX_MEMORY_SAMPLES_PER_REQUEST. */
+    overLimit: number;
+}
+export declare function normalizeMemoryMetrics(request: OtlpMetricsRequest, stats?: MemoryNormalizeStats): MemorySample[];
 export declare const platformEventSchema: z.ZodObject<{
     service: z.ZodString;
     environment: z.ZodDefault<z.ZodString>;

package/dist/memory.js

@@ -28,15 +28,29 @@ function metricOf(name, attrs) {
         default: return null;
     }
 }
-export function normalizeMemoryMetrics(request) {
+/** Memory/GC samples one metrics request may produce; past it, points are dropped and counted. */
+export const MAX_MEMORY_SAMPLES_PER_REQUEST = 5000;
+export function normalizeMemoryMetrics(request, stats) {
     const out = [];
     const now = Date.now();
     for (const rm of request.resourceMetrics ?? []) {
         const resource = attrsOf(rm.resource?.attributes);
         const instanceId = resource.get("service.instance.id")?.slice(0, 128) ?? "";
         const service = resource.get("service.name")?.slice(0, 200) ?? "";
-        if (!instanceId || !service)
+        if (!instanceId || !service) {
+            // Samples are per instance: memory points of a resource without
+            // service.name and service.instance.id can't be stored.
+            if (stats) {
+                for (const scope of rm.scopeMetrics ?? [])
+                    for (const metric of scope.metrics ?? []) {
+                        for (const point of metric.gauge?.dataPoints ?? metric.sum?.dataPoints ?? []) {
+                            if (metricOf(metric.name ?? "", attrsOf(point.attributes)))
+                                stats.invalid += 1;
+                        }
+                    }
+            }
             continue;
+        }
         const environment = (resource.get("deployment.environment.name") ?? resource.get("deployment.environment") ?? "production").slice(0, 100);
         const release = (resource.get("service.version") ?? "").slice(0, 200);
         for (const scope of rm.scopeMetrics ?? [])
@@ -47,6 +61,11 @@ export function normalizeMemoryMetrics(request) {
                     const kind = metricOf(name, attrsOf(point.attributes));
                     if (!kind)
                         continue;
+                    if (out.length >= MAX_MEMORY_SAMPLES_PER_REQUEST) {
+                        if (stats)
+                            stats.overLimit += 1;
+                        continue;
+                    }
                     const counter = kind === "gc_count" || kind === "gc_duration_ms";
                     const aggregation = metric.sum?.aggregationTemporality;
                     const temporality = counter && (aggregation === 2 || aggregation === "AGGREGATION_TEMPORALITY_CUMULATIVE")
@@ -57,21 +76,23 @@ export function normalizeMemoryMetrics(request) {
                         observedMs = Number(BigInt(String(point.timeUnixNano ?? "0")) / 1000000n);
                     }
                     catch {
+                        if (stats)
+                            stats.invalid += 1;
                         continue;
                     }
-                    if (!Number.isFinite(value) || value < 0 || value > 1e15 || observedMs < now - 30 * 86400_000 || observedMs > now + 300_000)
-                        continue;
                     const isBytes = kind === "rss" || kind === "heap_used" || kind === "heap_limit" || kind === "memory_limit" || kind === "heap_after_gc";
                     const unit = metric.unit ?? "By";
-                    if (isBytes && !["By", "byte", "bytes", "KiBy", "MiBy"].includes(unit))
+                    if (!Number.isFinite(value) || value < 0 || value > 1e15 || observedMs < now - 30 * 86400_000 || observedMs > now + 300_000
+                        || (isBytes && !["By", "byte", "bytes", "KiBy", "MiBy"].includes(unit))) {
+                        if (stats)
+                            stats.invalid += 1;
                         continue;
+                    }
                     // Standard OTel runtimes report GC duration in seconds; Autter uses ms.
                     out.push({ service, environment, release, instanceId, metric: kind,
                         value: isBytes ? value * (unit === "KiBy" ? 1024 : unit === "MiBy" ? 1048576 : 1)
                             : kind === "gc_duration_ms" && unit === "s" ? value * 1000 : value,
                         observedAt: new Date(observedMs), temporality });
-                    if (out.length >= 5000)
-                        return out;
                 }
             }
     }

package/dist/migrations.d.ts

@@ -45,6 +45,26 @@ export declare const OCCURRENCE_REQUEST_ID_INDEX = "INDEX idx_occ_request_id req
  * read `[]` and still match on the legacy `fingerprint`.
  */
 export declare const OCCURRENCE_GROUPING_COLUMNS: string[];
+/**
+ * Migration 0020 columns — shared with the baseline DDL in clickhouse.ts.
+ * In-app browser JS frames a source map resolved / left unresolved at
+ * ingest (symbolicate.ts inAppFrameResolution); 0 and 0 for server
+ * occurrences, browser ones with no in-app JS frame, and rows written
+ * before 0020.
+ */
+export declare const OCCURRENCE_FRAME_RESOLUTION_COLUMNS: string[];
+/**
+ * Migration 0021 column — shared with the baseline DDL in clickhouse.ts.
+ * Loop collapse (1.9.0, loop-collapse.ts): the identical occurrences a
+ * minute's collapsed sample stands for besides itself, whose payloads were
+ * not stored; 0 on every other row. An exact occurrence count is, per
+ * distinct `occurrence_id` (an exporter retry can store a row twice),
+ * 1 + collapsed_count: `SELECT sum(1 + c) FROM (SELECT occurrence_id,
+ * max(collapsed_count) AS c … GROUP BY occurrence_id)`. A metadata-only
+ * ALTER with a constant default: rows written before it, and by older
+ * replicas during a rolling deploy, read 0.
+ */
+export declare const OCCURRENCE_COLLAPSE_COLUMNS: string[];
 export interface Migration {
     /** Unique, ordered id: "<serial>-<slug>". Never reuse or reorder. */
     id: string;

package/dist/migrations.js

@@ -37,7 +37,8 @@ import { profileTableDDL } from "./profiles.js";
 import { SOURCE_MAP_DEBUG_COLUMNS, SOURCE_MAP_DEBUG_ID_INDEX, sourceMapTableDDL, } from "./source-maps.js";
 import { LOG_REQUEST_COLUMNS, LOG_REQUEST_ID_INDEX, logTableDDL, requestRollupTableDDL, requestRollupViewDDL, } from "./logs.js";
 import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
-import { ingesterInfoTableDDL, sdkVersionTableDDL } from "./sdk-versions.js";
+import { ingesterInfoTableDDL, SDK_VERSION_VIA_COLUMN, sdkVersionTableDDL } from "./sdk-versions.js";
+import { ingestDropsTableDDL } from "./ingest-drops.js";
 /** Migration 0013 columns — shared with the baseline DDL in clickhouse.ts. */
 export const OCCURRENCE_CODE_COLUMNS = [
     "error_code String DEFAULT ''",
@@ -62,6 +63,29 @@ export const OCCURRENCE_GROUPING_COLUMNS = [
     "frames String DEFAULT '' CODEC(ZSTD(3))",
     "fingerprints Array(String) DEFAULT []",
 ];
+/**
+ * Migration 0020 columns — shared with the baseline DDL in clickhouse.ts.
+ * In-app browser JS frames a source map resolved / left unresolved at
+ * ingest (symbolicate.ts inAppFrameResolution); 0 and 0 for server
+ * occurrences, browser ones with no in-app JS frame, and rows written
+ * before 0020.
+ */
+export const OCCURRENCE_FRAME_RESOLUTION_COLUMNS = [
+    "in_app_frames_resolved UInt16 DEFAULT 0",
+    "in_app_frames_unresolved UInt16 DEFAULT 0",
+];
+/**
+ * Migration 0021 column — shared with the baseline DDL in clickhouse.ts.
+ * Loop collapse (1.9.0, loop-collapse.ts): the identical occurrences a
+ * minute's collapsed sample stands for besides itself, whose payloads were
+ * not stored; 0 on every other row. An exact occurrence count is, per
+ * distinct `occurrence_id` (an exporter retry can store a row twice),
+ * 1 + collapsed_count: `SELECT sum(1 + c) FROM (SELECT occurrence_id,
+ * max(collapsed_count) AS c … GROUP BY occurrence_id)`. A metadata-only
+ * ALTER with a constant default: rows written before it, and by older
+ * replicas during a rolling deploy, read 0.
+ */
+export const OCCURRENCE_COLLAPSE_COLUMNS = ["collapsed_count UInt32 DEFAULT 0"];
 export const MIGRATIONS = [
     // 0001 intentionally reserved as the baseline marker: databases created
     // before the migration runner existed record it as applied without
@@ -202,6 +226,27 @@ export const MIGRATIONS = [
             `ALTER TABLE {db}.runtime_source_maps ADD ${SOURCE_MAP_DEBUG_ID_INDEX.replace("INDEX ", "INDEX IF NOT EXISTS ")}`,
         ],
     },
+    // Rejections and drops per tenant, minute, route and reason (EI-27):
+    // what the ingester refused or did not store, so a repository's missing
+    // telemetry has a reason (ingest-drops.ts).
+    { id: "0018-runtime-ingest-drops", statements: [ingestDropsTableDDL("{db}")] },
+    // How each service's telemetry arrived: an OpenTelemetry Collector, an
+    // SDK exporter directly, or the browser relay (sdk-versions.ts).
+    {
+        id: "0019-sdk-versions-via",
+        statements: [`ALTER TABLE {db}.runtime_sdk_versions ADD COLUMN IF NOT EXISTS ${SDK_VERSION_VIA_COLUMN}`],
+    },
+    // Resolved and unresolved in-app browser frames per occurrence, after
+    // symbolication — the share the EI-14 source map rule reads.
+    {
+        id: "0020-occurrence-frame-resolution",
+        statements: OCCURRENCE_FRAME_RESOLUTION_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
+    },
+    // Loop collapse (1.9.0): the count a minute's collapsed sample carries.
+    {
+        id: "0021-occurrence-collapsed-count",
+        statements: OCCURRENCE_COLLAPSE_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
+    },
 ];
 /** The tracking table itself — created by the runner before anything else. */
 export function migrationsTableDDL(db) {