@autter/otlp-ingester 1.7.0 → 1.9.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +42 -7
- package/dist/auth.d.ts +15 -0
- package/dist/auth.js +27 -7
- package/dist/clickhouse.d.ts +3 -0
- package/dist/clickhouse.js +41 -1
- package/dist/compat-manifest.json +35 -0
- package/dist/config.d.ts +15 -0
- package/dist/config.js +8 -0
- package/dist/context.d.ts +5 -0
- package/dist/context.js +49 -10
- package/dist/index.d.ts +4 -1
- package/dist/index.js +14 -2
- package/dist/ingest-drops.d.ts +329 -0
- package/dist/ingest-drops.js +454 -0
- package/dist/log-promotion.d.ts +4 -3
- package/dist/log-promotion.js +5 -5
- package/dist/logs.d.ts +16 -1
- package/dist/logs.js +17 -3
- package/dist/loop-collapse.d.ts +144 -0
- package/dist/loop-collapse.js +326 -0
- package/dist/memory.d.ts +9 -1
- package/dist/memory.js +28 -7
- package/dist/migrations.d.ts +20 -0
- package/dist/migrations.js +46 -1
- package/dist/normalize-browser.d.ts +47 -19
- package/dist/normalize-browser.js +56 -10
- package/dist/normalize-otlp.js +165 -45
- package/dist/otlp-proto.d.ts +14 -0
- package/dist/otlp-proto.js +25 -0
- package/dist/profiles.d.ts +4 -0
- package/dist/profiles.js +6 -2
- package/dist/request-spans.d.ts +45 -0
- package/dist/request-spans.js +91 -0
- package/dist/sdk-versions.d.ts +39 -2
- package/dist/sdk-versions.js +49 -8
- package/dist/server.d.ts +6 -0
- package/dist/server.js +367 -105
- package/dist/sink.d.ts +19 -2
- package/dist/sink.js +23 -3
- package/dist/stack-frames.d.ts +5 -2
- package/dist/stack-frames.js +55 -9
- package/dist/suppression.d.ts +79 -0
- package/dist/suppression.js +203 -0
- package/dist/symbolicate.d.ts +13 -0
- package/dist/symbolicate.js +22 -0
- package/dist/types.d.ts +14 -0
- package/package.json +1 -1
package/dist/sink.d.ts
CHANGED
|
@@ -26,6 +26,17 @@ export interface SinkTuning {
|
|
|
26
26
|
/** Deliveries run concurrently while healthy, serially while failing. */
|
|
27
27
|
healthyConcurrency: number;
|
|
28
28
|
}
|
|
29
|
+
/** A batch the sink gave up on (counted in runtime_ingest_drops, route "sink"). */
|
|
30
|
+
export interface SinkDrop {
|
|
31
|
+
orgId: string;
|
|
32
|
+
repositoryId: string;
|
|
33
|
+
/** sink_overflow: buffer overflow, or a batch bigger than the buffer;
|
|
34
|
+
* sink_rejected: the consumer answered a permanent 4xx;
|
|
35
|
+
* sink_gave_up: SINK_MAX_ATTEMPTS deliveries failed. */
|
|
36
|
+
reason: "sink_overflow" | "sink_rejected" | "sink_gave_up";
|
|
37
|
+
/** Occurrences in the batch. */
|
|
38
|
+
occurrences: number;
|
|
39
|
+
}
|
|
29
40
|
/**
|
|
30
41
|
* Operational counters for /healthz. Failure detail is reduced to a fixed
|
|
31
42
|
* category (`timeout`, `connection_error`, `http_<status>`, `error`) — the
|
|
@@ -72,10 +83,13 @@ export declare class SinkForwarder {
|
|
|
72
83
|
private lastFailureAt;
|
|
73
84
|
private lastFailureReason;
|
|
74
85
|
private readonly tuning;
|
|
86
|
+
/** Called for every batch given up on; must not throw (errors are swallowed). */
|
|
87
|
+
onDrop: ((drop: SinkDrop) => void) | null;
|
|
75
88
|
constructor(config: IngesterConfig, fetchImpl?: typeof fetch, tuning?: Partial<SinkTuning>);
|
|
76
89
|
/** Queue a batch for delivery. No-op when there is nothing to send. */
|
|
77
90
|
enqueue(ctx: IngestContext, occurrences: RuntimeOccurrence[], metricPoints?: RuntimeMetricPoint[], llmCalls?: RuntimeLlmCall[]): void;
|
|
78
91
|
stats(): SinkStats;
|
|
92
|
+
private noteDrop;
|
|
79
93
|
pendingCount(): number;
|
|
80
94
|
/** Stop scheduling new deliveries (shutdown). Queued batches are logged
|
|
81
95
|
* by the caller — they are recoverable from ClickHouse, not from here. */
|
|
@@ -110,7 +124,10 @@ export declare class SinkForwarder {
|
|
|
110
124
|
* 1.6.0 adds `frames` (innermost first, ≤ 50), `fingerprints` (every
|
|
111
125
|
* identity candidate, highest precedence first, ending with the legacy
|
|
112
126
|
* `fingerprint`) and `fingerprintRecord`; `fingerprint`/`fingerprintScheme`
|
|
113
|
-
* keep their legacy meaning.
|
|
114
|
-
*
|
|
127
|
+
* keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
|
|
128
|
+
* record stands for, sent only when above 1 (a loop-collapse sample:
|
|
129
|
+
* 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
|
|
130
|
+
* their event counts. Older consumers ignore unknown keys, so the version
|
|
131
|
+
* stays 1.
|
|
115
132
|
*/
|
|
116
133
|
export declare function sinkOccurrence(o: RuntimeOccurrence): Record<string, unknown>;
|
package/dist/sink.js
CHANGED
|
@@ -44,6 +44,8 @@ export class SinkForwarder {
|
|
|
44
44
|
lastFailureAt = null;
|
|
45
45
|
lastFailureReason = null;
|
|
46
46
|
tuning;
|
|
47
|
+
/** Called for every batch given up on; must not throw (errors are swallowed). */
|
|
48
|
+
onDrop = null;
|
|
47
49
|
constructor(config, fetchImpl = fetch, tuning = {}) {
|
|
48
50
|
this.config = config;
|
|
49
51
|
this.fetchImpl = fetchImpl;
|
|
@@ -65,6 +67,7 @@ export class SinkForwarder {
|
|
|
65
67
|
if (batch.bytes > this.maxBufferedBytes()) {
|
|
66
68
|
this.droppedOverflow += 1;
|
|
67
69
|
console.warn(`sink batch ${batch.batchId} (org ${batch.orgId}) exceeds the buffer cap (${batch.bytes} bytes) — dropped; replay signals ${batch.signalsFrom ?? "?"} .. ${batch.signalsTo ?? "?"} from ClickHouse`);
|
|
70
|
+
this.noteDrop(batch, "sink_overflow");
|
|
68
71
|
return;
|
|
69
72
|
}
|
|
70
73
|
this.queue.push(batch);
|
|
@@ -88,6 +91,14 @@ export class SinkForwarder {
|
|
|
88
91
|
oldestQueuedSince: oldest ? new Date(oldest).toISOString() : null,
|
|
89
92
|
};
|
|
90
93
|
}
|
|
94
|
+
noteDrop(batch, reason) {
|
|
95
|
+
try {
|
|
96
|
+
this.onDrop?.({ orgId: batch.orgId, repositoryId: batch.repositoryId, reason, occurrences: batch.occurrences });
|
|
97
|
+
}
|
|
98
|
+
catch {
|
|
99
|
+
// Drop accounting must never affect delivery.
|
|
100
|
+
}
|
|
101
|
+
}
|
|
91
102
|
pendingCount() {
|
|
92
103
|
return this.queue.length + this.inFlight;
|
|
93
104
|
}
|
|
@@ -122,6 +133,8 @@ export class SinkForwarder {
|
|
|
122
133
|
return {
|
|
123
134
|
batchId,
|
|
124
135
|
orgId: ctx.orgId,
|
|
136
|
+
repositoryId: ctx.repositoryId,
|
|
137
|
+
occurrences: occurrences.length,
|
|
125
138
|
body,
|
|
126
139
|
bytes: Buffer.byteLength(body),
|
|
127
140
|
seq: ++this.seqCounter,
|
|
@@ -170,6 +183,7 @@ export class SinkForwarder {
|
|
|
170
183
|
break;
|
|
171
184
|
this.droppedOverflow += 1;
|
|
172
185
|
console.warn(`sink buffer overflow: dropped batch ${dropped.batchId} (org ${dropped.orgId}, signals ${dropped.signalsFrom ?? "?"} .. ${dropped.signalsTo ?? "?"}) — replay this range from ClickHouse`);
|
|
186
|
+
this.noteDrop(dropped, "sink_overflow");
|
|
173
187
|
}
|
|
174
188
|
}
|
|
175
189
|
/** The oldest batch of the org holding the most buffered bytes. */
|
|
@@ -280,6 +294,7 @@ export class SinkForwarder {
|
|
|
280
294
|
if (permanent || batch.attempts >= this.config.sinkMaxAttempts) {
|
|
281
295
|
this.droppedPermanent += 1;
|
|
282
296
|
console.error(`sink delivery gave up after ${batch.attempts} attempt(s) (${detail}): batch ${batch.batchId} (signals ${batch.signalsFrom ?? "?"} .. ${batch.signalsTo ?? "?"}) — replay this range from ClickHouse`);
|
|
297
|
+
this.noteDrop(batch, permanent ? "sink_rejected" : "sink_gave_up");
|
|
283
298
|
return;
|
|
284
299
|
}
|
|
285
300
|
this.retried += 1;
|
|
@@ -304,13 +319,16 @@ export class SinkForwarder {
|
|
|
304
319
|
* 1.6.0 adds `frames` (innermost first, ≤ 50), `fingerprints` (every
|
|
305
320
|
* identity candidate, highest precedence first, ending with the legacy
|
|
306
321
|
* `fingerprint`) and `fingerprintRecord`; `fingerprint`/`fingerprintScheme`
|
|
307
|
-
* keep their legacy meaning.
|
|
308
|
-
*
|
|
322
|
+
* keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
|
|
323
|
+
* record stands for, sent only when above 1 (a loop-collapse sample:
|
|
324
|
+
* 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
|
|
325
|
+
* their event counts. Older consumers ignore unknown keys, so the version
|
|
326
|
+
* stays 1.
|
|
309
327
|
*/
|
|
310
328
|
export function sinkOccurrence(o) {
|
|
311
329
|
const { traceId, route, method, statusCode, errorCode, why, fix, link, expected, requestId, breadcrumbs, frames, fingerprints, fingerprintRecord,
|
|
312
330
|
// Input-only: locates source maps, never leaves the ingester.
|
|
313
|
-
debugIds: _debugIds, ...rest } = o;
|
|
331
|
+
debugIds: _debugIds, collapsedCount, ...rest } = o;
|
|
314
332
|
return {
|
|
315
333
|
...rest,
|
|
316
334
|
occurredAt: o.occurredAt.toISOString(),
|
|
@@ -329,6 +347,8 @@ export function sinkOccurrence(o) {
|
|
|
329
347
|
...(frames?.length ? { frames } : {}),
|
|
330
348
|
...(fingerprints?.length ? { fingerprints } : {}),
|
|
331
349
|
...(fingerprintRecord ? { fingerprintRecord } : {}),
|
|
350
|
+
// 1.9.0: a collapsed sample stands for 1 + collapsedCount occurrences.
|
|
351
|
+
...(collapsedCount && collapsedCount > 0 ? { count: 1 + collapsedCount } : {}),
|
|
332
352
|
};
|
|
333
353
|
}
|
|
334
354
|
/** ISO range across every signal in the batch — the replay hint on drops. */
|
package/dist/stack-frames.d.ts
CHANGED
|
@@ -11,8 +11,10 @@ import type { StackFrame } from "./types.js";
|
|
|
11
11
|
* - V8 (Node, Chrome, Edge): `at fn (file:line:col)` / `at file:line:col`
|
|
12
12
|
* - Firefox / Safari: `fn@url:line:col` / `@url:line:col`
|
|
13
13
|
* - Python: `File "…", line N, in fn` — tracebacks list the OUTERMOST frame
|
|
14
|
-
* first, so the frames are reversed; only the
|
|
15
|
-
* of a chained exception
|
|
14
|
+
* first, so the frames are reversed; only the reported exception's own
|
|
15
|
+
* frames are used: the last top-level block of a chained exception, and
|
|
16
|
+
* for an exception group (3.11+) its own `| `-margined block, never its
|
|
17
|
+
* sub-exceptions
|
|
16
18
|
* - Go: function line + `\t<file>.go:<line> +0x…`, first goroutine only
|
|
17
19
|
* - JVM: `at pkg.Class.method(File.java:N)`, first exception block only
|
|
18
20
|
* (before `Caused by:`)
|
|
@@ -34,6 +36,7 @@ export type FrameLanguage = "javascript" | "python" | "go" | "jvm" | "dotnet" |
|
|
|
34
36
|
* `C:\app\node_modules\x.js` counts too. A frame with no path is in-app.
|
|
35
37
|
*/
|
|
36
38
|
export declare function isInAppPath(path: string | null): boolean;
|
|
39
|
+
export declare function isPythonStdlibPath(path: string | null): boolean;
|
|
37
40
|
/**
|
|
38
41
|
* Contract §3. Source-map ignore lists are applied by symbolicate.ts; the
|
|
39
42
|
* language rules apply only to frames of that language, so a JS object
|
package/dist/stack-frames.js
CHANGED
|
@@ -11,8 +11,10 @@ import { detectStackLanguage } from "./fingerprint.js";
|
|
|
11
11
|
* - V8 (Node, Chrome, Edge): `at fn (file:line:col)` / `at file:line:col`
|
|
12
12
|
* - Firefox / Safari: `fn@url:line:col` / `@url:line:col`
|
|
13
13
|
* - Python: `File "…", line N, in fn` — tracebacks list the OUTERMOST frame
|
|
14
|
-
* first, so the frames are reversed; only the
|
|
15
|
-
* of a chained exception
|
|
14
|
+
* first, so the frames are reversed; only the reported exception's own
|
|
15
|
+
* frames are used: the last top-level block of a chained exception, and
|
|
16
|
+
* for an exception group (3.11+) its own `| `-margined block, never its
|
|
17
|
+
* sub-exceptions
|
|
16
18
|
* - Go: function line + `\t<file>.go:<line> +0x…`, first goroutine only
|
|
17
19
|
* - JVM: `at pkg.Class.method(File.java:N)`, first exception block only
|
|
18
20
|
* (before `Caused by:`)
|
|
@@ -63,6 +65,22 @@ export function isInAppPath(path) {
|
|
|
63
65
|
return false;
|
|
64
66
|
return true;
|
|
65
67
|
}
|
|
68
|
+
/**
|
|
69
|
+
* Python's standard library: frozen modules (`<frozen runpy>`) and the
|
|
70
|
+
* version directory of an installation (`/usr/local/lib/python3.12/`,
|
|
71
|
+
* `…/Versions/3.14/lib/python3.14/`, `C:\Python312\Lib\`); a
|
|
72
|
+
* virtualenv's packages are `site-packages`, covered by the path rule. Like
|
|
73
|
+
* Go's `/usr/local/go/`, they are the runtime, not the app: an asyncio or
|
|
74
|
+
* socket error's first five frames would otherwise all be stdlib frames.
|
|
75
|
+
*/
|
|
76
|
+
const PY_STDLIB_RE = /\/lib\/(?:python|pypy)\d\.\d+t?\/|\/python\d{2,3}t?\/lib\//i;
|
|
77
|
+
export function isPythonStdlibPath(path) {
|
|
78
|
+
if (!path)
|
|
79
|
+
return false;
|
|
80
|
+
if (path.startsWith("<frozen "))
|
|
81
|
+
return true;
|
|
82
|
+
return PY_STDLIB_RE.test(path.replace(/\\/g, "/"));
|
|
83
|
+
}
|
|
66
84
|
/** Fully qualified name for the language rules (`module.function`). */
|
|
67
85
|
function qualifiedName(frame) {
|
|
68
86
|
if (frame.module && frame.function)
|
|
@@ -84,6 +102,8 @@ export function isInAppFrame(frame, language) {
|
|
|
84
102
|
if ((frame.filename ?? "").replace(/\\/g, "/").includes("/usr/local/go/"))
|
|
85
103
|
return false;
|
|
86
104
|
}
|
|
105
|
+
if (language === "python" && isPythonStdlibPath(frame.filename))
|
|
106
|
+
return false;
|
|
87
107
|
if (language === "jvm" && /^(?:java|javax|kotlin|jdk)\./.test(qualified))
|
|
88
108
|
return false;
|
|
89
109
|
if (language === "dotnet" && /^(?:System|Microsoft)\./.test(qualified))
|
|
@@ -225,15 +245,41 @@ function parseGecko(lines) {
|
|
|
225
245
|
// ── Python ────────────────────────────────────────────────────────────────
|
|
226
246
|
const PY_FILE_RE = /^\s*File "(.+)", line (\d+)(?:, in (.+?))?\s*$/;
|
|
227
247
|
const PY_TRACEBACK_RE = /^\s*Traceback \(most recent call last\):\s*$/;
|
|
248
|
+
// 3.11+: an exception group prints its own frames behind a "| " margin,
|
|
249
|
+
// under this header (nested groups' headers carry the margin instead of "+"),
|
|
250
|
+
// then its sub-exceptions after a "+-+---- 1 ----" separator.
|
|
251
|
+
const PY_GROUP_TRACEBACK_RE = /^\s*\+ Exception Group Traceback \(most recent call last\):\s*$/;
|
|
252
|
+
const PY_GROUP_MARGIN_RE = /^\s*\|/;
|
|
253
|
+
const PY_GROUP_SUBEXCEPTIONS_RE = /^\s*\+-/;
|
|
254
|
+
function isPythonFrameLine(line) {
|
|
255
|
+
return PY_FILE_RE.test(line) || PY_FILE_RE.test(line.replace(PY_GROUP_MARGIN_RE, ""));
|
|
256
|
+
}
|
|
228
257
|
function parsePython(lines) {
|
|
229
|
-
// A chained exception prints one
|
|
230
|
-
// (final)
|
|
231
|
-
|
|
232
|
-
|
|
233
|
-
|
|
258
|
+
// A chained exception prints one block per exception, the reported
|
|
259
|
+
// (final) one last, whether the chain says "During handling of the above
|
|
260
|
+
// exception" or "The above exception was the direct cause". Blocks of
|
|
261
|
+
// sub-exceptions and of groups in the chain carry a margin, so the last
|
|
262
|
+
// unmargined header starts the reported exception's own frames.
|
|
263
|
+
let start = -1;
|
|
264
|
+
let group = false;
|
|
265
|
+
for (let i = 0; i < lines.length; i++) {
|
|
266
|
+
if (PY_TRACEBACK_RE.test(lines[i])) {
|
|
267
|
+
start = i;
|
|
268
|
+
group = false;
|
|
269
|
+
}
|
|
270
|
+
else if (PY_GROUP_TRACEBACK_RE.test(lines[i])) {
|
|
234
271
|
start = i;
|
|
272
|
+
group = true;
|
|
273
|
+
}
|
|
274
|
+
}
|
|
235
275
|
const outermostFirst = [];
|
|
236
|
-
for (const
|
|
276
|
+
for (const raw of lines.slice(start + 1)) {
|
|
277
|
+
let line = raw;
|
|
278
|
+
if (group) {
|
|
279
|
+
if (PY_GROUP_SUBEXCEPTIONS_RE.test(line))
|
|
280
|
+
break;
|
|
281
|
+
line = line.replace(PY_GROUP_MARGIN_RE, "");
|
|
282
|
+
}
|
|
237
283
|
const m = PY_FILE_RE.exec(line);
|
|
238
284
|
if (!m)
|
|
239
285
|
continue;
|
|
@@ -391,7 +437,7 @@ export function detectFrameLanguage(lines) {
|
|
|
391
437
|
return "dotnet";
|
|
392
438
|
default:
|
|
393
439
|
}
|
|
394
|
-
if (lines.some(
|
|
440
|
+
if (lines.some(isPythonFrameLine))
|
|
395
441
|
return "python";
|
|
396
442
|
if (lines.some((line) => DOTNET_BARE_RE.test(line)) &&
|
|
397
443
|
!lines.some((line) => /:\d+(?::\d+)?\)?\s*$/.test(line))) {
|
|
@@ -0,0 +1,79 @@
|
|
|
1
|
+
import type { RuntimeOccurrence } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Suppression rules (1.9.0, EI-38): occurrences the user chose not to keep,
|
|
4
|
+
* set per repository in Autter and delivered with the key (the validator's
|
|
5
|
+
* `suppression` answer, cached with the key for 60 s), or on a static key
|
|
6
|
+
* entry for self-hosted ingesters.
|
|
7
|
+
*
|
|
8
|
+
* A rule is a list of filters that must ALL match (AND) and an action:
|
|
9
|
+
*
|
|
10
|
+
* { "id": "rsr_1", "action": "drop",
|
|
11
|
+
* "filters": [{ "key": "errorType", "op": "equals", "value": "AbortError" }] }
|
|
12
|
+
* { "id": "rsr_2", "action": "sample", "sampleRate": 0.1,
|
|
13
|
+
* "filters": [{ "key": "message", "op": "matches", "value": "ResizeObserver loop*" }] }
|
|
14
|
+
*
|
|
15
|
+
* Keys: errorType, message, service, environment, route, release, source
|
|
16
|
+
* (`browser` or `server`). Operators, all case-insensitive: `equals`,
|
|
17
|
+
* `contains`, `starts_with`, `matches` (a glob over the whole value where
|
|
18
|
+
* `*` is any run of characters; matched segment by segment, so time is
|
|
19
|
+
* linear in the value and no user pattern can stall ingest). A missing
|
|
20
|
+
* value (no route, no release) is the empty string.
|
|
21
|
+
*
|
|
22
|
+
* Rules are tried in order and the first that matches decides: `drop`
|
|
23
|
+
* leaves the occurrence out; `sample` keeps `sampleRate` (0 < rate < 1) of
|
|
24
|
+
* them, chosen by the occurrence id, so a retried batch keeps the same
|
|
25
|
+
* ones, and never samples out a `fatal` occurrence. Suppressed occurrences
|
|
26
|
+
* are not stored or forwarded and are counted in `runtime_ingest_drops`
|
|
27
|
+
* (reason `suppressed`). Request and error-rate rollups are not affected:
|
|
28
|
+
* they count requests, not the occurrences kept.
|
|
29
|
+
*
|
|
30
|
+
* Parsing never throws: an invalid rule (unknown key or operator, empty or
|
|
31
|
+
* oversized value, a sample rate outside (0, 1)) is left out on its own.
|
|
32
|
+
*/
|
|
33
|
+
export declare const SUPPRESSION_PROPERTIES: readonly ["errorType", "message", "service", "environment", "route", "release", "source"];
|
|
34
|
+
export type SuppressionProperty = (typeof SUPPRESSION_PROPERTIES)[number];
|
|
35
|
+
export declare const SUPPRESSION_OPERATORS: readonly ["equals", "contains", "starts_with", "matches"];
|
|
36
|
+
export type SuppressionOperator = (typeof SUPPRESSION_OPERATORS)[number];
|
|
37
|
+
export type SuppressionAction = "drop" | "sample";
|
|
38
|
+
export interface SuppressionFilter {
|
|
39
|
+
key: SuppressionProperty;
|
|
40
|
+
op: SuppressionOperator;
|
|
41
|
+
/** Lower-cased at parse time: every comparison is case-insensitive. */
|
|
42
|
+
value: string;
|
|
43
|
+
}
|
|
44
|
+
export interface SuppressionRule {
|
|
45
|
+
id: string;
|
|
46
|
+
filters: SuppressionFilter[];
|
|
47
|
+
action: SuppressionAction;
|
|
48
|
+
/** `sample` only: the share kept, 0 < rate < 1. */
|
|
49
|
+
sampleRate?: number;
|
|
50
|
+
}
|
|
51
|
+
/** Longest filter value accepted; longer ones make the rule invalid. */
|
|
52
|
+
export declare const MAX_SUPPRESSION_VALUE_LENGTH = 500;
|
|
53
|
+
/** One rule, or null when any part of it is invalid (a rule is never half-applied). */
|
|
54
|
+
export declare function parseSuppressionRule(raw: unknown): SuppressionRule | null;
|
|
55
|
+
/** The valid rules of a validator answer or static key entry, in order; [] for anything else. */
|
|
56
|
+
export declare function parseSuppressionRules(raw: unknown): SuppressionRule[];
|
|
57
|
+
/**
|
|
58
|
+
* `*`-glob over the whole text: literal segments must appear in order, the
|
|
59
|
+
* first anchored at the start unless the pattern starts with `*`, the last
|
|
60
|
+
* at the end unless it ends with `*`. Each segment is found leftmost with
|
|
61
|
+
* indexOf, which is always a valid placement for star-only globs, so there
|
|
62
|
+
* is no backtracking. Both sides are expected lower-cased.
|
|
63
|
+
*/
|
|
64
|
+
export declare function globMatch(pattern: string, text: string): boolean;
|
|
65
|
+
/** The first rule whose filters all match, or null. */
|
|
66
|
+
export declare function matchingRule(rules: readonly SuppressionRule[], o: RuntimeOccurrence): SuppressionRule | null;
|
|
67
|
+
/**
|
|
68
|
+
* A stable number in [0, 1) for an occurrence: its id is a content-derived
|
|
69
|
+
* hex digest (occurrenceIdFor), so the same occurrence in a retried batch
|
|
70
|
+
* lands in the same place. Ids that are not hex fall back to FNV-1a.
|
|
71
|
+
*/
|
|
72
|
+
export declare function sampleUnit(occurrenceId: string): number;
|
|
73
|
+
export interface SuppressionOutcome<T> {
|
|
74
|
+
kept: T[];
|
|
75
|
+
/** Occurrences left out (dropped or sampled out). */
|
|
76
|
+
suppressed: number;
|
|
77
|
+
}
|
|
78
|
+
/** Apply the tenant's rules; order of the kept occurrences is preserved. */
|
|
79
|
+
export declare function applySuppression<T extends RuntimeOccurrence>(occurrences: T[], rules: readonly SuppressionRule[] | undefined): SuppressionOutcome<T>;
|
|
@@ -0,0 +1,203 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Suppression rules (1.9.0, EI-38): occurrences the user chose not to keep,
|
|
3
|
+
* set per repository in Autter and delivered with the key (the validator's
|
|
4
|
+
* `suppression` answer, cached with the key for 60 s), or on a static key
|
|
5
|
+
* entry for self-hosted ingesters.
|
|
6
|
+
*
|
|
7
|
+
* A rule is a list of filters that must ALL match (AND) and an action:
|
|
8
|
+
*
|
|
9
|
+
* { "id": "rsr_1", "action": "drop",
|
|
10
|
+
* "filters": [{ "key": "errorType", "op": "equals", "value": "AbortError" }] }
|
|
11
|
+
* { "id": "rsr_2", "action": "sample", "sampleRate": 0.1,
|
|
12
|
+
* "filters": [{ "key": "message", "op": "matches", "value": "ResizeObserver loop*" }] }
|
|
13
|
+
*
|
|
14
|
+
* Keys: errorType, message, service, environment, route, release, source
|
|
15
|
+
* (`browser` or `server`). Operators, all case-insensitive: `equals`,
|
|
16
|
+
* `contains`, `starts_with`, `matches` (a glob over the whole value where
|
|
17
|
+
* `*` is any run of characters; matched segment by segment, so time is
|
|
18
|
+
* linear in the value and no user pattern can stall ingest). A missing
|
|
19
|
+
* value (no route, no release) is the empty string.
|
|
20
|
+
*
|
|
21
|
+
* Rules are tried in order and the first that matches decides: `drop`
|
|
22
|
+
* leaves the occurrence out; `sample` keeps `sampleRate` (0 < rate < 1) of
|
|
23
|
+
* them, chosen by the occurrence id, so a retried batch keeps the same
|
|
24
|
+
* ones, and never samples out a `fatal` occurrence. Suppressed occurrences
|
|
25
|
+
* are not stored or forwarded and are counted in `runtime_ingest_drops`
|
|
26
|
+
* (reason `suppressed`). Request and error-rate rollups are not affected:
|
|
27
|
+
* they count requests, not the occurrences kept.
|
|
28
|
+
*
|
|
29
|
+
* Parsing never throws: an invalid rule (unknown key or operator, empty or
|
|
30
|
+
* oversized value, a sample rate outside (0, 1)) is left out on its own.
|
|
31
|
+
*/
|
|
32
|
+
export const SUPPRESSION_PROPERTIES = ["errorType", "message", "service", "environment", "route", "release", "source"];
|
|
33
|
+
export const SUPPRESSION_OPERATORS = ["equals", "contains", "starts_with", "matches"];
|
|
34
|
+
/** Longest filter value accepted; longer ones make the rule invalid. */
|
|
35
|
+
export const MAX_SUPPRESSION_VALUE_LENGTH = 500;
|
|
36
|
+
const PROPERTIES = new Set(SUPPRESSION_PROPERTIES);
|
|
37
|
+
const OPERATORS = new Set(SUPPRESSION_OPERATORS);
|
|
38
|
+
function parseFilter(raw) {
|
|
39
|
+
if (!raw || typeof raw !== "object")
|
|
40
|
+
return null;
|
|
41
|
+
const { key, op, value } = raw;
|
|
42
|
+
if (typeof key !== "string" || !PROPERTIES.has(key))
|
|
43
|
+
return null;
|
|
44
|
+
if (typeof op !== "string" || !OPERATORS.has(op))
|
|
45
|
+
return null;
|
|
46
|
+
if (typeof value !== "string" || !value || value.length > MAX_SUPPRESSION_VALUE_LENGTH)
|
|
47
|
+
return null;
|
|
48
|
+
return { key: key, op: op, value: value.toLowerCase() };
|
|
49
|
+
}
|
|
50
|
+
/** One rule, or null when any part of it is invalid (a rule is never half-applied). */
|
|
51
|
+
export function parseSuppressionRule(raw) {
|
|
52
|
+
if (!raw || typeof raw !== "object")
|
|
53
|
+
return null;
|
|
54
|
+
const { id, filters, action, sampleRate } = raw;
|
|
55
|
+
if (typeof id !== "string" || !id || id.length > 200)
|
|
56
|
+
return null;
|
|
57
|
+
if (action !== "drop" && action !== "sample")
|
|
58
|
+
return null;
|
|
59
|
+
if (!Array.isArray(filters) || filters.length === 0)
|
|
60
|
+
return null;
|
|
61
|
+
const parsed = [];
|
|
62
|
+
for (const item of filters) {
|
|
63
|
+
const filter = parseFilter(item);
|
|
64
|
+
if (!filter)
|
|
65
|
+
return null;
|
|
66
|
+
parsed.push(filter);
|
|
67
|
+
}
|
|
68
|
+
if (action === "sample") {
|
|
69
|
+
if (typeof sampleRate !== "number" || !Number.isFinite(sampleRate) || sampleRate <= 0 || sampleRate >= 1)
|
|
70
|
+
return null;
|
|
71
|
+
return { id, filters: parsed, action, sampleRate };
|
|
72
|
+
}
|
|
73
|
+
return { id, filters: parsed, action };
|
|
74
|
+
}
|
|
75
|
+
/** The valid rules of a validator answer or static key entry, in order; [] for anything else. */
|
|
76
|
+
export function parseSuppressionRules(raw) {
|
|
77
|
+
if (!Array.isArray(raw))
|
|
78
|
+
return [];
|
|
79
|
+
const rules = [];
|
|
80
|
+
for (const item of raw) {
|
|
81
|
+
try {
|
|
82
|
+
const rule = parseSuppressionRule(item);
|
|
83
|
+
if (rule)
|
|
84
|
+
rules.push(rule);
|
|
85
|
+
}
|
|
86
|
+
catch {
|
|
87
|
+
// A hostile getter or Proxy: leave the rule out.
|
|
88
|
+
}
|
|
89
|
+
}
|
|
90
|
+
return rules;
|
|
91
|
+
}
|
|
92
|
+
/**
|
|
93
|
+
* `*`-glob over the whole text: literal segments must appear in order, the
|
|
94
|
+
* first anchored at the start unless the pattern starts with `*`, the last
|
|
95
|
+
* at the end unless it ends with `*`. Each segment is found leftmost with
|
|
96
|
+
* indexOf, which is always a valid placement for star-only globs, so there
|
|
97
|
+
* is no backtracking. Both sides are expected lower-cased.
|
|
98
|
+
*/
|
|
99
|
+
export function globMatch(pattern, text) {
|
|
100
|
+
const segments = pattern.split("*");
|
|
101
|
+
if (segments.length === 1)
|
|
102
|
+
return pattern === text;
|
|
103
|
+
const first = segments[0];
|
|
104
|
+
const last = segments[segments.length - 1];
|
|
105
|
+
if (!text.startsWith(first))
|
|
106
|
+
return false;
|
|
107
|
+
let at = first.length;
|
|
108
|
+
for (let i = 1; i < segments.length - 1; i++) {
|
|
109
|
+
const segment = segments[i];
|
|
110
|
+
if (!segment)
|
|
111
|
+
continue;
|
|
112
|
+
const found = text.indexOf(segment, at);
|
|
113
|
+
if (found === -1)
|
|
114
|
+
return false;
|
|
115
|
+
at = found + segment.length;
|
|
116
|
+
}
|
|
117
|
+
return text.length - last.length >= at && text.endsWith(last);
|
|
118
|
+
}
|
|
119
|
+
function propertyOf(o, key) {
|
|
120
|
+
switch (key) {
|
|
121
|
+
case "errorType":
|
|
122
|
+
return o.errorType ?? "";
|
|
123
|
+
case "message":
|
|
124
|
+
return o.message ?? "";
|
|
125
|
+
case "service":
|
|
126
|
+
return o.service ?? "";
|
|
127
|
+
case "environment":
|
|
128
|
+
return o.environment ?? "";
|
|
129
|
+
case "route":
|
|
130
|
+
return o.route ?? "";
|
|
131
|
+
case "release":
|
|
132
|
+
return o.release ?? "";
|
|
133
|
+
case "source":
|
|
134
|
+
return o.source ?? "";
|
|
135
|
+
}
|
|
136
|
+
}
|
|
137
|
+
function filterMatches(filter, o) {
|
|
138
|
+
const value = propertyOf(o, filter.key).toLowerCase();
|
|
139
|
+
switch (filter.op) {
|
|
140
|
+
case "equals":
|
|
141
|
+
return value === filter.value;
|
|
142
|
+
case "contains":
|
|
143
|
+
return value.includes(filter.value);
|
|
144
|
+
case "starts_with":
|
|
145
|
+
return value.startsWith(filter.value);
|
|
146
|
+
case "matches":
|
|
147
|
+
return globMatch(filter.value, value);
|
|
148
|
+
}
|
|
149
|
+
}
|
|
150
|
+
/** The first rule whose filters all match, or null. */
|
|
151
|
+
export function matchingRule(rules, o) {
|
|
152
|
+
for (const rule of rules) {
|
|
153
|
+
if (rule.filters.every((filter) => filterMatches(filter, o)))
|
|
154
|
+
return rule;
|
|
155
|
+
}
|
|
156
|
+
return null;
|
|
157
|
+
}
|
|
158
|
+
/**
|
|
159
|
+
* A stable number in [0, 1) for an occurrence: its id is a content-derived
|
|
160
|
+
* hex digest (occurrenceIdFor), so the same occurrence in a retried batch
|
|
161
|
+
* lands in the same place. Ids that are not hex fall back to FNV-1a.
|
|
162
|
+
*/
|
|
163
|
+
export function sampleUnit(occurrenceId) {
|
|
164
|
+
const head = occurrenceId.slice(0, 8);
|
|
165
|
+
if (/^[0-9a-f]{8}$/i.test(head))
|
|
166
|
+
return Number.parseInt(head, 16) / 0x1_0000_0000;
|
|
167
|
+
let hash = 0x811c9dc5;
|
|
168
|
+
for (let i = 0; i < occurrenceId.length; i++) {
|
|
169
|
+
hash ^= occurrenceId.charCodeAt(i);
|
|
170
|
+
hash = Math.imul(hash, 0x01000193) >>> 0;
|
|
171
|
+
}
|
|
172
|
+
return hash / 0x1_0000_0000;
|
|
173
|
+
}
|
|
174
|
+
const NEVER_SAMPLED = new Set(["fatal"]);
|
|
175
|
+
/** Apply the tenant's rules; order of the kept occurrences is preserved. */
|
|
176
|
+
export function applySuppression(occurrences, rules) {
|
|
177
|
+
if (!rules?.length || occurrences.length === 0)
|
|
178
|
+
return { kept: occurrences, suppressed: 0 };
|
|
179
|
+
const kept = [];
|
|
180
|
+
let suppressed = 0;
|
|
181
|
+
for (const o of occurrences) {
|
|
182
|
+
let rule = null;
|
|
183
|
+
try {
|
|
184
|
+
rule = matchingRule(rules, o);
|
|
185
|
+
}
|
|
186
|
+
catch {
|
|
187
|
+
// Never fail ingest on a rule: keep the occurrence.
|
|
188
|
+
}
|
|
189
|
+
if (!rule) {
|
|
190
|
+
kept.push(o);
|
|
191
|
+
}
|
|
192
|
+
else if (rule.action === "drop") {
|
|
193
|
+
suppressed += 1;
|
|
194
|
+
}
|
|
195
|
+
else if (NEVER_SAMPLED.has(o.severity) || sampleUnit(o.occurrenceId) < (rule.sampleRate ?? 1)) {
|
|
196
|
+
kept.push(o);
|
|
197
|
+
}
|
|
198
|
+
else {
|
|
199
|
+
suppressed += 1;
|
|
200
|
+
}
|
|
201
|
+
}
|
|
202
|
+
return { kept, suppressed };
|
|
203
|
+
}
|
package/dist/symbolicate.d.ts
CHANGED
|
@@ -40,6 +40,19 @@ export interface SymbolicationScope {
|
|
|
40
40
|
/** Script URL → debug id (EI-13), when the browser sent them. */
|
|
41
41
|
debugIds?: Record<string, string> | null;
|
|
42
42
|
}
|
|
43
|
+
/**
|
|
44
|
+
* In-app browser frames a source map could resolve, counted after
|
|
45
|
+
* symbolication (EI-27): `resolved` ones, and `unresolved` .js/.mjs/.cjs
|
|
46
|
+
* frames left as parsed (no map uploaded, no release or debug id, no
|
|
47
|
+
* mapping at that position). Frames a map moved out of the app
|
|
48
|
+
* (node_modules, ignoreList) and non-JS frames count in neither. Stored per
|
|
49
|
+
* occurrence (`in_app_frames_resolved` / `in_app_frames_unresolved`) for the
|
|
50
|
+
* EI-14 rule: over 30% unresolved means the release needs source maps.
|
|
51
|
+
*/
|
|
52
|
+
export declare function inAppFrameResolution(frames: readonly StackFrame[] | undefined): {
|
|
53
|
+
resolved: number;
|
|
54
|
+
unresolved: number;
|
|
55
|
+
};
|
|
43
56
|
/** Path-only form of a script URL — the key uploads are stored under. */
|
|
44
57
|
export declare function scriptPath(filename: string): string | null;
|
|
45
58
|
export interface SourceMapCacheOptions {
|
package/dist/symbolicate.js
CHANGED
|
@@ -5,6 +5,28 @@ import { isInAppPath, stripQueryAndHash } from "./stack-frames.js";
|
|
|
5
5
|
const JS_FILE_RE = /\.[cm]?js$/i;
|
|
6
6
|
const MAX_CONTEXT_CHARS = 300;
|
|
7
7
|
const CONTEXT_LINES = 3;
|
|
8
|
+
/**
|
|
9
|
+
* In-app browser frames a source map could resolve, counted after
|
|
10
|
+
* symbolication (EI-27): `resolved` ones, and `unresolved` .js/.mjs/.cjs
|
|
11
|
+
* frames left as parsed (no map uploaded, no release or debug id, no
|
|
12
|
+
* mapping at that position). Frames a map moved out of the app
|
|
13
|
+
* (node_modules, ignoreList) and non-JS frames count in neither. Stored per
|
|
14
|
+
* occurrence (`in_app_frames_resolved` / `in_app_frames_unresolved`) for the
|
|
15
|
+
* EI-14 rule: over 30% unresolved means the release needs source maps.
|
|
16
|
+
*/
|
|
17
|
+
export function inAppFrameResolution(frames) {
|
|
18
|
+
let resolved = 0;
|
|
19
|
+
let unresolved = 0;
|
|
20
|
+
for (const frame of frames ?? []) {
|
|
21
|
+
if (!frame.inApp)
|
|
22
|
+
continue;
|
|
23
|
+
if (frame.resolved)
|
|
24
|
+
resolved += 1;
|
|
25
|
+
else if (frame.filename && JS_FILE_RE.test(frame.filename))
|
|
26
|
+
unresolved += 1;
|
|
27
|
+
}
|
|
28
|
+
return { resolved, unresolved };
|
|
29
|
+
}
|
|
8
30
|
/** Path-only form of a script URL — the key uploads are stored under. */
|
|
9
31
|
export function scriptPath(filename) {
|
|
10
32
|
try {
|
package/dist/types.d.ts
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import type { Breadcrumb } from "./breadcrumbs.js";
|
|
2
|
+
import type { SuppressionRule } from "./suppression.js";
|
|
2
3
|
export type RuntimeSource = "browser" | "server";
|
|
3
4
|
/**
|
|
4
5
|
* Signal severity. Errors AND warnings/info land in the same occurrences
|
|
@@ -131,6 +132,13 @@ export interface RuntimeOccurrence extends RuntimeOccurrenceInput {
|
|
|
131
132
|
topFrames: string[];
|
|
132
133
|
/** topFrames[0] — single-column GROUP BY for "where did this come from". */
|
|
133
134
|
firstFrame: string;
|
|
135
|
+
/**
|
|
136
|
+
* Loop collapse (1.9.0, loop-collapse.ts): the minute's collapsed sample
|
|
137
|
+
* stands for itself plus this many identical occurrences whose payloads
|
|
138
|
+
* were not stored. Stored as `collapsed_count`, sent to the sink as
|
|
139
|
+
* `count` (1 + collapsedCount). Absent (0) on every other occurrence.
|
|
140
|
+
*/
|
|
141
|
+
collapsedCount?: number;
|
|
134
142
|
}
|
|
135
143
|
export interface RuntimeSpanRow {
|
|
136
144
|
service: string;
|
|
@@ -212,4 +220,10 @@ export interface IngestContext {
|
|
|
212
220
|
scope: IngestKeyScope;
|
|
213
221
|
/** client keys only: exact origins allowed to send (empty = any). */
|
|
214
222
|
allowedOrigins: string[];
|
|
223
|
+
/**
|
|
224
|
+
* Suppression rules set by the user for this repository (1.9.0): from the
|
|
225
|
+
* key validator's `suppression` answer (cached with the key, 60 s) or a
|
|
226
|
+
* static key's `suppression` entry. Absent or empty: nothing suppressed.
|
|
227
|
+
*/
|
|
228
|
+
suppression?: SuppressionRule[];
|
|
215
229
|
}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@autter/otlp-ingester",
|
|
3
|
-
"version": "1.
|
|
3
|
+
"version": "1.9.0",
|
|
4
4
|
"description": "Self-hostable OTLP + browser-error ingest service for Autter Runtime: normalises telemetry into a per-repo ClickHouse data model",
|
|
5
5
|
"license": "MIT",
|
|
6
6
|
"type": "module",
|