@autter/otlp-ingester 1.8.0 → 1.9.0

package/README.md

@@ -57,8 +57,28 @@ AUTTER_INGEST_KEYS='[
 ```
 
 The validator webhook may return the same extra fields:
-`{ orgId, repositoryId, scope?, allowedOrigins? }`, with `scope` one of
-`server`, `client`, `upload`.
+`{ orgId, repositoryId, scope?, allowedOrigins?, suppression? }`, with
+`scope` one of `server`, `client`, `upload`, and `suppression` the
+repository's suppression rules (1.9.0, below); a static entry takes
+`"suppression": [ … ]` too.
+
+### Volume control (1.9.0)
+
+- **Loop collapse.** Per tenant and identity (fingerprints, service,
+  environment, release, source, severity, error type and code, normalised
+  message and top frames, session), the first `LOOP_COLLAPSE_AFTER`
+  occurrences of a minute are stored as usual; the rest of the minute is
+  stored as ONE sample whose `collapsed_count` makes the count exact, and
+  forwarded to the sink with `count`. A loop of 10,000 a minute is 11 rows,
+  not 10,000. Shutdown stores the minute in progress.
+- **Suppression rules** set per repository in Autter: `drop` or `sample`
+  (keep `sampleRate`, never a fatal one) the occurrences whose error type,
+  message, service, environment, route, release or source match. Delivered
+  with the key, applied before anything is stored.
+
+Folded and suppressed occurrences are counted in `runtime_ingest_drops`
+(`collapsed`, `suppressed`); request and error-rate rollups still count
+every error. Details: [docs/ARCHITECTURE.md § Volume control](../../docs/ARCHITECTURE.md#volume-control).
 
 ### Rejections and drops (1.8.0)
 
@@ -93,8 +113,8 @@ Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILIT
 | `CLICKHOUSE_URL` | — | e.g. `http://localhost:8123`; unset = ingest returns 503 |
 | `CLICKHOUSE_USER` / `CLICKHOUSE_PASSWORD` | `default` / empty | |
 | `CLICKHOUSE_DATABASE` | `autter_runtime` | Created automatically |
-| `AUTTER_INGEST_KEYS` | — | JSON: `[{"key":"...","orgId":"...","repositoryId":"..."}]` |
-| `AUTTER_KEY_VALIDATOR_URL` | — | Webhook: `POST {key}` → `{orgId, repositoryId}` (60 s cache) |
+| `AUTTER_INGEST_KEYS` | — | JSON: `[{"key":"...","orgId":"...","repositoryId":"..."}]` (optional `scope`, `allowedOrigins`, `suppression`) |
+| `AUTTER_KEY_VALIDATOR_URL` | — | Webhook: `POST {key}` → `{orgId, repositoryId, scope?, allowedOrigins?, suppression?}` (60 s cache) |
 | `AUTTER_KEY_VALIDATOR_TOKEN` | — | Bearer token sent to the validator |
 | `AUTTER_SINK_URL` | — | Issue-grouping webhook; at-least-once (`docs/ARCHITECTURE.md`) |
 | `AUTTER_SINK_TOKEN` | — | Bearer token sent to the sink |
@@ -103,6 +123,7 @@ Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIBILIT
 | `SINK_MAX_BUFFERED_MB` | `64` | Sink retry buffer cap (memory) |
 | `SOURCE_MAP_CACHE_MB` | `64` | Memory budget (decoded size, function scope tables included) for parsed source maps used to symbolicate browser frames at ingest; least recently used maps are evicted, and a map that needs more than half of it is used without being kept |
 | `MAX_BODY_BYTES` | `1048576` | Request body cap |
+| `LOOP_COLLAPSE_AFTER` | `10` | Identical occurrences stored per minute before the rest of the minute is collapsed into one sample with an exact count; `0` turns loop collapse off |
 | `RATE_LIMIT_PER_MINUTE` | `300` | Per-key fixed window (server keys) |
 | `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) |
 | `PROMOTION_LOOKUPS_PER_MINUTE` | `30` | Per-tenant ClickHouse dedupe lookups for `/v1/logs` error promotion; over budget, records are promoted with in-batch dedupe only |

package/dist/auth.js

@@ -1,6 +1,7 @@
 import { INGESTER_VERSION_HEADER } from "./compat.js";
 import { defaultScopeFor, isIngestKeyScope } from "./config.js";
 import { ingesterVersion } from "./sdk-versions.js";
+import { parseSuppressionRules } from "./suppression.js";
 const VALIDATOR_CACHE_TTL_MS = 60_000;
 export class KeyResolver {
     config;
@@ -12,11 +13,16 @@ export class KeyResolver {
             const scope = entry.scope ?? defaultScopeFor(entry.key);
             if (!isIngestKeyScope(scope))
                 continue;
+            const suppression = parseSuppressionRules(entry.suppression);
+            if (Array.isArray(entry.suppression) && suppression.length < entry.suppression.length) {
+                console.error(`AUTTER_INGEST_KEYS: ${entry.suppression.length - suppression.length} invalid suppression rule(s) for repository ${entry.repositoryId} ignored`);
+            }
             this.staticKeys.set(entry.key, {
                 orgId: entry.orgId,
                 repositoryId: entry.repositoryId,
                 scope,
                 allowedOrigins: entry.allowedOrigins ?? [],
+                ...(suppression.length ? { suppression } : {}),
             });
         }
     }
@@ -71,11 +77,13 @@ export class KeyResolver {
                 const body = (await response.json());
                 const scope = body.scope === undefined || body.scope === null ? defaultScopeFor(key) : body.scope;
                 if (body.orgId && body.repositoryId && isIngestKeyScope(scope)) {
+                    const suppression = parseSuppressionRules(body.suppression);
                     ctx = {
                         orgId: body.orgId,
                         repositoryId: body.repositoryId,
                         scope,
                         allowedOrigins: body.allowedOrigins ?? [],
+                        ...(suppression.length ? { suppression } : {}),
                     };
                 }
             }

package/dist/clickhouse.js

@@ -9,7 +9,7 @@ import { logTableDDL, requestRollupTableDDL, } from "./logs.js";
 import { ingesterInfoTableDDL, sdkVersionTableDDL, } from "./sdk-versions.js";
 import { ingestDropsTableDDL } from "./ingest-drops.js";
 import { inAppFrameResolution } from "./symbolicate.js";
-import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_FRAME_RESOLUTION_COLUMNS, OCCURRENCE_GROUPING_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
+import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_COLLAPSE_COLUMNS, OCCURRENCE_FRAME_RESOLUTION_COLUMNS, OCCURRENCE_GROUPING_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
 /**
  * ClickHouse persistence. The schema is idempotent and applied lazily once
  * per process. Every row is keyed by (org_id, repository_id) — tenant
@@ -113,6 +113,7 @@ export class ClickHouseStore {
 				${OCCURRENCE_CODE_COLUMNS.join(",\n\t\t\t\t")},
 				${OCCURRENCE_GROUPING_COLUMNS.join(",\n\t\t\t\t")},
 				${OCCURRENCE_FRAME_RESOLUTION_COLUMNS.join(",\n\t\t\t\t")},
+				${OCCURRENCE_COLLAPSE_COLUMNS.join(",\n\t\t\t\t")},
 				${OCCURRENCE_REQUEST_ID_INDEX}
 			)
 			ENGINE = MergeTree
@@ -421,6 +422,8 @@ export class ClickHouseStore {
                 fingerprint_scheme: o.fingerprintScheme,
                 frames: o.frames?.length ? JSON.stringify(o.frames) : "",
                 fingerprints: (o.fingerprints ?? []).map((candidate) => candidate.fingerprint),
+                // Migration 0021: only collapsed samples name the column.
+                ...(o.collapsedCount ? { collapsed_count: Math.min(0xffff_ffff, Math.max(0, Math.round(o.collapsedCount))) } : {}),
             })),
         });
     }

package/dist/compat-manifest.json

@@ -155,6 +155,20 @@
             "ingester": "1.8.0",
             "migrations": ["0020-occurrence-frame-resolution"],
             "sdks": {}
+        },
+        {
+            "id": "ingest_suppression",
+            "title": "Suppression rules: drop or sample matching errors at ingest",
+            "ingester": "1.9.0",
+            "migrations": [],
+            "sdks": {}
+        },
+        {
+            "id": "loop_collapse",
+            "title": "Loop collapse: a looping error stored as one sample a minute with an exact count",
+            "ingester": "1.9.0",
+            "migrations": ["0021-occurrence-collapsed-count"],
+            "sdks": {}
         }
     ]
 }

package/dist/config.d.ts

@@ -23,6 +23,13 @@ export interface StaticIngestKey {
     scope?: IngestKeyScope;
     /** client keys: exact origins allowed to send (e.g. https://app.example.com). */
     allowedOrigins?: string[];
+    /**
+     * Suppression rules for this key's repository (1.9.0, suppression.ts):
+     * `[{ "id": "…", "action": "drop" | "sample", "sampleRate"?: 0.1,
+     * "filters": [{ "key": "errorType", "op": "equals", "value": "…" }] }]`.
+     * Invalid rules are left out (logged once at startup).
+     */
+    suppression?: unknown[];
 }
 export interface IngesterConfig {
     port: number;
@@ -77,7 +84,15 @@ export interface IngesterConfig {
      * A map that needs more than half of it is used without being kept.
      */
     sourceMapCacheMb?: number;
+    /**
+     * Loop collapse (1.9.0, loop-collapse.ts): identical occurrences stored
+     * per minute before the rest are folded into one sample with an exact
+     * count (LOOP_COLLAPSE_AFTER, default 10; 0 turns it off).
+     */
+    loopCollapseAfter?: number;
 }
+/** LOOP_COLLAPSE_AFTER: a whole number ≥ 0 (0 = off); anything else is the default, 10. */
+export declare function loopCollapseAfterEnv(raw: string | undefined): number;
 /** Parse AUTTER_INGEST_KEYS. Entries without a string key, orgId and
  * repositoryId, or with an unknown scope, are dropped (a typo in `scope`
  * must not turn a restricted key into a server key). */

package/dist/config.js

@@ -12,6 +12,13 @@ export function isIngestKeyScope(value) {
 export function defaultScopeFor(key) {
     return key.startsWith(UPLOAD_KEY_PREFIX) ? "upload" : "server";
 }
+/** LOOP_COLLAPSE_AFTER: a whole number ≥ 0 (0 = off); anything else is the default, 10. */
+export function loopCollapseAfterEnv(raw) {
+    if (raw === undefined || raw.trim() === "")
+        return 10;
+    const value = Number(raw);
+    return Number.isInteger(value) && value >= 0 ? value : 10;
+}
 function intEnv(name, fallback) {
     const raw = process.env[name];
     if (!raw)
@@ -92,6 +99,7 @@ export function loadConfig() {
         redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"),
         redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"),
         sourceMapCacheMb: intEnv("SOURCE_MAP_CACHE_MB", 64),
+        loopCollapseAfter: loopCollapseAfterEnv(process.env.LOOP_COLLAPSE_AFTER),
     };
     if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) {
         throw new Error(`Invalid CLICKHOUSE_DATABASE name: ${config.clickhouseDatabase}`);

package/dist/index.d.ts

@@ -2,5 +2,7 @@ export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
 export * from "./compat.js";
 export { SinkForwarder, type SinkDrop, type SinkStats, type SinkTuning } from "./sink.js";
+export { collapseIdentity, LoopCollapser, type CollapsedSample, type CollapsePlan, type LoopCollapseOptions, type LoopCollapseStats, } from "./loop-collapse.js";
+export { applySuppression, globMatch, parseSuppressionRules, SUPPRESSION_OPERATORS, SUPPRESSION_PROPERTIES, type SuppressionAction, type SuppressionFilter, type SuppressionOperator, type SuppressionProperty, type SuppressionRule, } from "./suppression.js";
 export { DROP_REASONS, IngestDropCounter, type DropKind, type DropReason, type DropRoute, type IngestDrop, type IngestDropRow, } from "./ingest-drops.js";
 export * from "./types.js";

package/dist/index.js

@@ -2,7 +2,7 @@ import { loadConfig } from "./config.js";
 import { createIngesterApp } from "./server.js";
 import { ingesterVersion } from "./sdk-versions.js";
 const config = loadConfig();
-const { app, store, sink, reportIngesterInfo, drops } = createIngesterApp(config);
+const { app, store, sink, reportIngesterInfo, drops, collapse } = createIngesterApp(config);
 const server = app.listen(config.port, () => {
     console.log(`autter otlp-ingester ${ingesterVersion()} listening on :${config.port} ` +
         `(clickhouse: ${config.clickhouseUrl ? "configured" : "NOT configured"})`);
@@ -25,10 +25,14 @@ async function shutdown(signal) {
         }
     }
     server.close(() => {
-        // Write the rejection/drop counts still buffered (best effort: a
-        // failed write is logged, never retried).
-        void drops
+        // Store every loop-collapse window's sample (the minute in progress
+        // too), so a deploy loses no folded count; the sink is stopped, so
+        // the consumer picks them up from ClickHouse. Then write the
+        // rejection/drop counts still buffered (best effort: a failed write
+        // is logged, never retried).
+        void collapse
             .stop()
+            .finally(() => drops.stop())
             .finally(() => store.close())
             .finally(() => process.exit(0));
     });
@@ -40,5 +44,7 @@ export { createIngesterApp } from "./server.js";
 export { loadConfig } from "./config.js";
 export * from "./compat.js";
 export { SinkForwarder } from "./sink.js";
+export { collapseIdentity, LoopCollapser, } from "./loop-collapse.js";
+export { applySuppression, globMatch, parseSuppressionRules, SUPPRESSION_OPERATORS, SUPPRESSION_PROPERTIES, } from "./suppression.js";
 export { DROP_REASONS, IngestDropCounter, } from "./ingest-drops.js";
 export * from "./types.js";

package/dist/ingest-drops.d.ts

@@ -19,11 +19,24 @@ import type { IngestContext } from "./types.js";
  * - `unforwarded`: stored in ClickHouse, but the sink webhook gave up on
  *   the batch; the consumer recovers it by replaying ClickHouse.
  *
+ * And two kinds of records left out on purpose (1.9.0, EI-38), counted so
+ * the dashboard can explain why stored rows are fewer than records sent:
+ * - `collapsed`: a looping error past its first occurrences of the minute
+ *   (loop-collapse.ts); its payload was not stored, its count is, on the
+ *   minute's collapsed sample (`collapsed_count`). `records` counts payloads
+ *   not stored.
+ * - `suppressed`: occurrences a suppression rule set by the user dropped or
+ *   sampled out (suppression.ts).
+ *
+ * Both are recorded with `requests` 0: no request was refused, and a
+ * dashboard before 1.9.0 (which reads an unknown kind as a rejection)
+ * finds nothing refused in the last hour.
+ *
  * Counting never slows or fails ingest: `record()` is a synchronous map
  * update, rows are written in batches every few seconds, a failed write is
  * logged and dropped (never retried), and the buffer is bounded.
  */
-export type DropKind = "rejected" | "dropped" | "truncated" | "unforwarded";
+export type DropKind = "rejected" | "dropped" | "truncated" | "unforwarded" | "collapsed" | "suppressed";
 /** Every reason, its kind, and (for rejections) the HTTP status it answers with. */
 export declare const DROP_REASONS: {
     /** No key in `Authorization`, `x-autter-key` or `?key=`. */
@@ -146,6 +159,14 @@ export declare const DROP_REASONS: {
     readonly sink_gave_up: {
         readonly kind: "unforwarded";
     };
+    /** Occurrences of a looping error past the first LOOP_COLLAPSE_AFTER of the minute. */
+    readonly collapsed: {
+        readonly kind: "collapsed";
+    };
+    /** Occurrences a `drop` rule matched, or a `sample` rule sampled out. */
+    readonly suppressed: {
+        readonly kind: "suppressed";
+    };
 };
 export type DropReason = keyof typeof DROP_REASONS;
 /** Routes counted by name; anything else is "other". `sink` is the webhook. */

package/dist/ingest-drops.js

@@ -56,6 +56,12 @@ export const DROP_REASONS = {
     sink_rejected: { kind: "unforwarded" },
     /** SINK_MAX_ATTEMPTS deliveries failed. */
     sink_gave_up: { kind: "unforwarded" },
+    // ── collapsed: counted on the minute's sample, payload not stored (1.9.0) ──
+    /** Occurrences of a looping error past the first LOOP_COLLAPSE_AFTER of the minute. */
+    collapsed: { kind: "collapsed" },
+    // ── suppressed: a suppression rule set by the user left them out (1.9.0) ──
+    /** Occurrences a `drop` rule matched, or a `sample` rule sampled out. */
+    suppressed: { kind: "suppressed" },
 };
 /** Routes counted by name; anything else is "other". `sink` is the webhook. */
 export const DROP_ROUTES = [

package/dist/loop-collapse.d.ts

@@ -0,0 +1,144 @@
+import type { DropRoute } from "./ingest-drops.js";
+import type { IngestContext, RuntimeOccurrence } from "./types.js";
+/**
+ * Loop collapse (1.9.0, EI-38): an error that repeats in a loop is stored
+ * as its first N occurrences of the minute plus ONE sample that carries the
+ * exact count of the rest, so a loop of 10,000 events a minute costs N + 1
+ * rows a minute, not 10,000, and only duplicate payloads are lost.
+ *
+ * Identical: same tenant, ingest route and arrival minute, and the same
+ * - identity: legacy `fingerprint` and every `fingerprints` candidate;
+ * - service, environment, release, source, severity, `expected`;
+ * - error type and declared error code;
+ * - normalised message (`messageNormalized`: ids, numbers and quoted
+ *   strings templated out) and normalised top frames (`topFrames`);
+ * - session id: an error hitting many sessions is volume, not a loop, and
+ *   each session must still count as a user.
+ * Trace, request and occurrence ids, route, timestamps, attributes and
+ * breadcrumbs may differ: they are what a loop varies.
+ *
+ * Per window (that identity in one arrival minute): the first `after`
+ * occurrences (LOOP_COLLAPSE_AFTER, default 10) are stored and forwarded as
+ * they arrive. Every later one is folded: not stored on its own, counted.
+ * When the minute is over, the window's newest folded occurrence is stored
+ * and forwarded as the sample with `collapsedCount` = folded − 1, so the
+ * sample stands for every folded occurrence: stored rows + Σ collapsed_count
+ * is exactly the number of occurrences accepted. Its time is the newest
+ * folded one, so an issue's last-seen stays exact; its weight lands in the
+ * minute it arrived in.
+ *
+ * Exactness under retries: a request's folds are reserved when it is planned
+ * and only counted once its ClickHouse writes succeed (`commit`); a request
+ * that fails (503, the exporter retries) gives them back (`rollback`). Ids
+ * already seen in the window are recognised, so a batch retried after a lost
+ * 2xx is not counted twice. A window is flushed only once no request still
+ * holds it (or, as a safety net, five minutes after its minute); a fold that
+ * commits after that is carried into the next flush. A failed flush write is
+ * kept and retried every tick. Shutdown (SIGTERM) flushes every window, so a
+ * deploy loses nothing; only a crash loses the folded counts of the minute in
+ * progress.
+ *
+ * Memory is bounded (safety, not a work cap): windows live for their minute
+ * and are cleared on the next flush; past `maxWindows` windows a minute, or
+ * `maxSamples` held samples, occurrences of new identities pass through
+ * uncollapsed (stored as usual, never lost), and past `maxTrackedIds` folded
+ * ids a minute, retry recognition stops for new folds.
+ */
+export interface LoopCollapseOptions {
+    /** Occurrences stored per window before folding starts; 0 turns collapse off. */
+    after: number;
+    /** Windows (identities) per minute. Default 100,000. */
+    maxWindows?: number;
+    /** Samples held across all windows until they are flushed. Default 5,000. */
+    maxSamples?: number;
+    /** Folded occurrence ids remembered per minute, for retry recognition. Default 500,000. */
+    maxTrackedIds?: number;
+    /** How often due windows are flushed. Default 5 s. */
+    flushIntervalMs?: number;
+    /** A window still held by a request is flushed anyway this long after its minute. Default 4 min. */
+    forceAfterMs?: number;
+    now?: () => number;
+    log?: (message: string) => void;
+}
+/** One window's sample, ready to store: `sample.collapsedCount` = folded − 1. */
+export interface CollapsedSample {
+    route: DropRoute;
+    sample: RuntimeOccurrence;
+    /** Occurrences the sample stands for (itself included). */
+    folded: number;
+}
+/** Store and forward one tenant's samples. A rejection keeps them for the next tick. */
+export type CollapseFlush = (ctx: IngestContext, samples: CollapsedSample[]) => Promise<void>;
+export interface CollapsePlan {
+    /** Occurrences to store and forward now. */
+    kept: RuntimeOccurrence[];
+    /** Occurrences folded into a window's sample by this request. */
+    folded: number;
+    /** Occurrences recognised as already counted (a retried batch); not stored again. */
+    repeated: number;
+    /** The request's writes succeeded: count its folds. */
+    commit(): void;
+    /** The request failed: give its reservations back. */
+    rollback(): void;
+}
+export interface LoopCollapseStats {
+    windows: number;
+    heldSamples: number;
+    /** Occurrences folded (committed) since start. */
+    folded: number;
+    /** Samples stored since start. */
+    samplesFlushed: number;
+    /** Occurrences that passed through uncollapsed because a bound was reached. */
+    overflow: number;
+    /** Flush writes that failed (their samples were kept and retried). */
+    failedFlushes: number;
+}
+/**
+ * The identity two occurrences must share to fold together (see above),
+ * hashed: the raw parts can run to kilobytes (frames, messages).
+ */
+export declare function collapseIdentity(o: RuntimeOccurrence): string;
+export declare class LoopCollapser {
+    private readonly flushSink;
+    private readonly minutes;
+    /** Folds committed after their window was flushed, and failed flushes. */
+    private carry;
+    private heldSamples;
+    private trackedIds;
+    private timer;
+    private flushing;
+    private lastFailureLog;
+    private stats_;
+    private readonly after;
+    private readonly maxWindows;
+    private readonly maxSamples;
+    private readonly maxTrackedIds;
+    private readonly flushIntervalMs;
+    private readonly forceAfterMs;
+    private readonly now;
+    private readonly log;
+    constructor(flushSink: CollapseFlush, options: LoopCollapseOptions);
+    get enabled(): boolean;
+    /**
+     * Split a request's occurrences into those stored now and those folded
+     * into their window's sample. Synchronous; never throws (on an internal
+     * error every occurrence is kept).
+     */
+    plan(ctx: IngestContext, route: DropRoute, occurrences: RuntimeOccurrence[]): CollapsePlan;
+    private planWindows;
+    /** Flush due windows every flushIntervalMs (unref'd). */
+    start(): void;
+    /** Stop the timer and flush every window, the current minute's too (shutdown). */
+    stop(): Promise<void>;
+    /**
+     * Store the samples of every window whose minute is over and that no
+     * request still holds, plus carried folds and earlier failed writes.
+     * `all`: every window, current minute included. One flush at a time;
+     * never rejects.
+     */
+    flush(all?: boolean): Promise<void>;
+    private collectDue;
+    private flushDue;
+    private releaseSlot;
+    stats(): LoopCollapseStats;
+}