@autter/otlp-ingester 1.5.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/breadcrumbs.d.ts +41 -0
- package/dist/breadcrumbs.js +151 -0
- package/dist/clickhouse.d.ts +10 -6
- package/dist/clickhouse.js +26 -4
- package/dist/compat-manifest.json +29 -0
- package/dist/compat.d.ts +3 -1
- package/dist/compat.js +3 -1
- package/dist/config.d.ts +6 -0
- package/dist/config.js +1 -0
- package/dist/context.js +4 -1
- package/dist/fingerprint-override.d.ts +42 -0
- package/dist/fingerprint-override.js +76 -0
- package/dist/fingerprint-v3.d.ts +54 -0
- package/dist/fingerprint-v3.js +96 -0
- package/dist/fingerprint.d.ts +25 -0
- package/dist/fingerprint.js +1 -1
- package/dist/logs.js +2 -0
- package/dist/memory.d.ts +2 -2
- package/dist/migrations.d.ts +10 -0
- package/dist/migrations.js +32 -1
- package/dist/normalize-browser.d.ts +55 -22
- package/dist/normalize-browser.js +24 -1
- package/dist/normalize-otlp.js +4 -0
- package/dist/server.js +73 -11
- package/dist/sink.d.ts +5 -1
- package/dist/sink.js +13 -2
- package/dist/source-maps-lookup.d.ts +31 -0
- package/dist/source-maps-lookup.js +64 -0
- package/dist/source-maps.d.ts +63 -2
- package/dist/source-maps.js +121 -13
- package/dist/stack-fixtures.d.ts +10 -2
- package/dist/stack-fixtures.js +13 -3
- package/dist/stack-frames.d.ts +48 -0
- package/dist/stack-frames.js +441 -0
- package/dist/symbolicate.d.ts +79 -0
- package/dist/symbolicate.js +257 -0
- package/dist/types.d.ts +68 -0
- package/package.json +2 -1
package/README.md
CHANGED
|
@@ -75,6 +75,7 @@ the Autter dashboard reads both. See [docs/COMPATIBILITY.md](../../docs/COMPATIB
|
|
|
75
75
|
| `SINK_MAX_ATTEMPTS` | `12` | Delivery attempts per batch (1–60 s backoff) |
|
|
76
76
|
| `SINK_MAX_BUFFERED_BATCHES` | `1000` | Retry buffer cap; oldest drops are logged |
|
|
77
77
|
| `SINK_MAX_BUFFERED_MB` | `64` | Sink retry buffer cap (memory) |
|
|
78
|
+
| `SOURCE_MAP_CACHE_MB` | `64` | Memory budget (estimated decoded size) for parsed source maps used to symbolicate browser frames at ingest; least recently used maps are evicted |
|
|
78
79
|
| `MAX_BODY_BYTES` | `1048576` | Request body cap |
|
|
79
80
|
| `RATE_LIMIT_PER_MINUTE` | `300` | Per-key fixed window (server keys) |
|
|
80
81
|
| `CLIENT_RATE_LIMIT_PER_MINUTE` | `120` | Per-key fixed window (client keys) |
|
|
@@ -0,0 +1,41 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Breadcrumbs — the steps before an error (sink contract §1).
|
|
3
|
+
*
|
|
4
|
+
* Browser events carry them as `breadcrumbs` (runtime-browser's opt-in
|
|
5
|
+
* `/breadcrumbs` module); server SDKs put a JSON string of the same array on
|
|
6
|
+
* the exception span event as `autter.breadcrumbs`. Either way they are
|
|
7
|
+
* untrusted client input, so this is a whitelist gate: unknown fields are
|
|
8
|
+
* dropped, every string is scrubbed, and the list is bounded before it is
|
|
9
|
+
* stored (under `autter.breadcrumbs` in the occurrence's attributes JSON)
|
|
10
|
+
* and forwarded to the sink.
|
|
11
|
+
*/
|
|
12
|
+
export type BreadcrumbType = "console" | "request" | "navigation" | "click" | "log" | "custom";
|
|
13
|
+
export type BreadcrumbLevel = "debug" | "info" | "warning" | "error";
|
|
14
|
+
export interface Breadcrumb {
|
|
15
|
+
type: BreadcrumbType;
|
|
16
|
+
/** ISO timestamp. */
|
|
17
|
+
at: string;
|
|
18
|
+
level?: BreadcrumbLevel;
|
|
19
|
+
/** ≤ 500 chars, scrubbed. */
|
|
20
|
+
message: string;
|
|
21
|
+
/** ≤ 10 keys of primitives; strings ≤ 200 chars, scrubbed. */
|
|
22
|
+
data?: Record<string, string | number | boolean | null>;
|
|
23
|
+
}
|
|
24
|
+
/** Span event / log attribute carrying a JSON string of `Breadcrumb[]`. */
|
|
25
|
+
export declare const BREADCRUMBS_ATTRIBUTE = "autter.breadcrumbs";
|
|
26
|
+
/** Key under which an occurrence's breadcrumbs are stored in its attributes JSON. */
|
|
27
|
+
export declare const BREADCRUMBS_STORAGE_KEY = "autter.breadcrumbs";
|
|
28
|
+
export declare const MAX_BREADCRUMBS = 30;
|
|
29
|
+
/**
|
|
30
|
+
* Validate, bound and scrub breadcrumbs from any sender: an array, or a JSON
|
|
31
|
+
* string of one (the span attribute). Keeps the newest 30 valid entries,
|
|
32
|
+
* oldest first, within 16 KB of JSON; malformed entries are dropped one by
|
|
33
|
+
* one. Undefined when nothing valid remains.
|
|
34
|
+
*/
|
|
35
|
+
export declare function parseBreadcrumbs(raw: unknown): Breadcrumb[] | undefined;
|
|
36
|
+
/** `{ breadcrumbs }` for an occurrence, or `{}` — spread into the input. */
|
|
37
|
+
export declare function breadcrumbsField(...candidates: unknown[]): {
|
|
38
|
+
breadcrumbs?: Breadcrumb[];
|
|
39
|
+
};
|
|
40
|
+
/** Attributes JSON to store: the occurrence's attributes plus its breadcrumbs. */
|
|
41
|
+
export declare function attributesWithBreadcrumbs(attributes: Record<string, unknown> | null, breadcrumbs: Breadcrumb[] | undefined): Record<string, unknown>;
|
|
@@ -0,0 +1,151 @@
|
|
|
1
|
+
import { isSensitiveKey, scrubText } from "./redact.js";
|
|
2
|
+
/** Span event / log attribute carrying a JSON string of `Breadcrumb[]`. */
|
|
3
|
+
export const BREADCRUMBS_ATTRIBUTE = "autter.breadcrumbs";
|
|
4
|
+
/** Key under which an occurrence's breadcrumbs are stored in its attributes JSON. */
|
|
5
|
+
export const BREADCRUMBS_STORAGE_KEY = "autter.breadcrumbs";
|
|
6
|
+
export const MAX_BREADCRUMBS = 30;
|
|
7
|
+
const MAX_MESSAGE = 500;
|
|
8
|
+
const MAX_DATA_KEYS = 10;
|
|
9
|
+
const MAX_DATA_STRING = 200;
|
|
10
|
+
/** Stored JSON per occurrence; the oldest breadcrumbs go first. */
|
|
11
|
+
const MAX_TOTAL_CHARS = 16 * 1024;
|
|
12
|
+
/** Raw JSON accepted from a span attribute before parsing. */
|
|
13
|
+
const MAX_RAW_CHARS = 256 * 1024;
|
|
14
|
+
/** Entries examined from the end of an oversized list. */
|
|
15
|
+
const MAX_SCANNED = 200;
|
|
16
|
+
const TYPES = new Set(["console", "request", "navigation", "click", "log", "custom"]);
|
|
17
|
+
const LEVELS = {
|
|
18
|
+
debug: "debug",
|
|
19
|
+
trace: "debug",
|
|
20
|
+
info: "info",
|
|
21
|
+
log: "info",
|
|
22
|
+
warning: "warning",
|
|
23
|
+
warn: "warning",
|
|
24
|
+
error: "error",
|
|
25
|
+
fatal: "error",
|
|
26
|
+
critical: "error",
|
|
27
|
+
};
|
|
28
|
+
const UNSAFE_KEY = /^(?:__proto__|constructor|prototype)$/;
|
|
29
|
+
/** Drop query strings and fragments from http(s) URLs (procedural: linear). */
|
|
30
|
+
function stripUrlQueries(value) {
|
|
31
|
+
return value.replace(/https?:\/\/[^\s]*/gi, (url) => {
|
|
32
|
+
const rest = url.slice(url.indexOf("://") + 3);
|
|
33
|
+
const cut = rest.search(/[?#]/);
|
|
34
|
+
return cut <= 0 ? url : url.slice(0, url.length - rest.length + cut);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
function cleanText(value, max) {
|
|
38
|
+
// Cut before scrubbing (bounded work), with slack so a secret straddling
|
|
39
|
+
// the cut is masked whole.
|
|
40
|
+
return stripUrlQueries(scrubText(value.slice(0, max + 512))).slice(0, max);
|
|
41
|
+
}
|
|
42
|
+
function cleanData(raw) {
|
|
43
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
44
|
+
return undefined;
|
|
45
|
+
const out = {};
|
|
46
|
+
let count = 0;
|
|
47
|
+
for (const [rawKey, value] of Object.entries(raw).slice(0, 64)) {
|
|
48
|
+
if (count >= MAX_DATA_KEYS)
|
|
49
|
+
break;
|
|
50
|
+
const key = rawKey.slice(0, 100);
|
|
51
|
+
if (!key || UNSAFE_KEY.test(key))
|
|
52
|
+
continue;
|
|
53
|
+
if (isSensitiveKey(key)) {
|
|
54
|
+
out[key] = "[redacted]";
|
|
55
|
+
}
|
|
56
|
+
else if (typeof value === "string") {
|
|
57
|
+
const text = /(?:url|path|route|target)$/i.test(key) ? value.split(/[?#]/)[0] : value;
|
|
58
|
+
out[key] = cleanText(text, MAX_DATA_STRING);
|
|
59
|
+
}
|
|
60
|
+
else if (typeof value === "number") {
|
|
61
|
+
if (!Number.isFinite(value))
|
|
62
|
+
continue;
|
|
63
|
+
out[key] = value;
|
|
64
|
+
}
|
|
65
|
+
else if (typeof value === "boolean" || value === null) {
|
|
66
|
+
out[key] = value;
|
|
67
|
+
}
|
|
68
|
+
else {
|
|
69
|
+
continue;
|
|
70
|
+
}
|
|
71
|
+
count++;
|
|
72
|
+
}
|
|
73
|
+
return count > 0 ? out : undefined;
|
|
74
|
+
}
|
|
75
|
+
function cleanBreadcrumb(raw) {
|
|
76
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
77
|
+
return null;
|
|
78
|
+
const input = raw;
|
|
79
|
+
const at = typeof input.at === "string" || typeof input.at === "number" ? new Date(input.at) : null;
|
|
80
|
+
if (!at || Number.isNaN(at.getTime()))
|
|
81
|
+
return null;
|
|
82
|
+
const message = typeof input.message === "string"
|
|
83
|
+
? input.message
|
|
84
|
+
: typeof input.message === "number" || typeof input.message === "boolean"
|
|
85
|
+
? String(input.message)
|
|
86
|
+
: "";
|
|
87
|
+
const type = typeof input.type === "string" && TYPES.has(input.type) ? input.type : "custom";
|
|
88
|
+
const level = typeof input.level === "string" ? LEVELS[input.level.toLowerCase()] : undefined;
|
|
89
|
+
const data = cleanData(input.data);
|
|
90
|
+
if (!message && !data)
|
|
91
|
+
return null;
|
|
92
|
+
return {
|
|
93
|
+
type: type,
|
|
94
|
+
at: at.toISOString(),
|
|
95
|
+
...(level ? { level } : {}),
|
|
96
|
+
message: cleanText(message, MAX_MESSAGE),
|
|
97
|
+
...(data ? { data } : {}),
|
|
98
|
+
};
|
|
99
|
+
}
|
|
100
|
+
/**
|
|
101
|
+
* Validate, bound and scrub breadcrumbs from any sender: an array, or a JSON
|
|
102
|
+
* string of one (the span attribute). Keeps the newest 30 valid entries,
|
|
103
|
+
* oldest first, within 16 KB of JSON; malformed entries are dropped one by
|
|
104
|
+
* one. Undefined when nothing valid remains.
|
|
105
|
+
*/
|
|
106
|
+
export function parseBreadcrumbs(raw) {
|
|
107
|
+
let list = raw;
|
|
108
|
+
if (typeof list === "string") {
|
|
109
|
+
if (list.length > MAX_RAW_CHARS)
|
|
110
|
+
return undefined;
|
|
111
|
+
try {
|
|
112
|
+
list = JSON.parse(list);
|
|
113
|
+
}
|
|
114
|
+
catch {
|
|
115
|
+
return undefined;
|
|
116
|
+
}
|
|
117
|
+
}
|
|
118
|
+
if (!Array.isArray(list) || list.length === 0)
|
|
119
|
+
return undefined;
|
|
120
|
+
const out = [];
|
|
121
|
+
let chars = 2;
|
|
122
|
+
const first = Math.max(0, list.length - MAX_SCANNED);
|
|
123
|
+
for (let i = list.length - 1; i >= first && out.length < MAX_BREADCRUMBS; i--) {
|
|
124
|
+
const crumb = cleanBreadcrumb(list[i]);
|
|
125
|
+
if (!crumb)
|
|
126
|
+
continue;
|
|
127
|
+
const size = JSON.stringify(crumb).length + 1;
|
|
128
|
+
if (chars + size > MAX_TOTAL_CHARS)
|
|
129
|
+
break;
|
|
130
|
+
chars += size;
|
|
131
|
+
out.push(crumb);
|
|
132
|
+
}
|
|
133
|
+
return out.length > 0 ? out.reverse() : undefined;
|
|
134
|
+
}
|
|
135
|
+
/** `{ breadcrumbs }` for an occurrence, or `{}` — spread into the input. */
|
|
136
|
+
export function breadcrumbsField(...candidates) {
|
|
137
|
+
for (const candidate of candidates) {
|
|
138
|
+
if (candidate === undefined || candidate === null)
|
|
139
|
+
continue;
|
|
140
|
+
const breadcrumbs = parseBreadcrumbs(candidate);
|
|
141
|
+
if (breadcrumbs)
|
|
142
|
+
return { breadcrumbs };
|
|
143
|
+
}
|
|
144
|
+
return {};
|
|
145
|
+
}
|
|
146
|
+
/** Attributes JSON to store: the occurrence's attributes plus its breadcrumbs. */
|
|
147
|
+
export function attributesWithBreadcrumbs(attributes, breadcrumbs) {
|
|
148
|
+
return breadcrumbs?.length
|
|
149
|
+
? { ...(attributes ?? {}), [BREADCRUMBS_STORAGE_KEY]: breadcrumbs }
|
|
150
|
+
: (attributes ?? {});
|
|
151
|
+
}
|
package/dist/clickhouse.d.ts
CHANGED
|
@@ -2,6 +2,8 @@ import type { IngesterConfig } from "./config.js";
|
|
|
2
2
|
import { type LatencyHistogram } from "./latency.js";
|
|
3
3
|
import { type ProfileSample } from "./profiles.js";
|
|
4
4
|
import { type MemorySample, type PlatformEvent } from "./memory.js";
|
|
5
|
+
import { type SourceMapCheck, type ValidSourceMap } from "./source-maps.js";
|
|
6
|
+
import { type SourceMapKey } from "./source-maps-lookup.js";
|
|
5
7
|
import { type RuntimeLogRecord } from "./logs.js";
|
|
6
8
|
import { type SdkVersionRow } from "./sdk-versions.js";
|
|
7
9
|
import type { SchemaStatus } from "./compat.js";
|
|
@@ -40,7 +42,9 @@ export declare class ClickHouseStore {
|
|
|
40
42
|
* this tenant in [from, to] — the cross-batch half of the log-promotion
|
|
41
43
|
* dedupe. `excludeIds` are the candidate occurrences' own deterministic
|
|
42
44
|
* ids, so a retried batch does not dedupe against its own earlier,
|
|
43
|
-
* partially-committed write.
|
|
45
|
+
* partially-committed write. Browser occurrences share the server's trace
|
|
46
|
+
* id (runtime-browser traceparent) but are a different signal, so they
|
|
47
|
+
* never suppress a server error. Throws on failure; the caller decides.
|
|
44
48
|
*/
|
|
45
49
|
recentOccurrenceTraceIds(ctx: IngestContext, traceIds: string[], from: Date, to: Date, excludeIds: string[]): Promise<Set<string>>;
|
|
46
50
|
/** Schema state as of the last ensureSchema attempt — never triggers DDL
|
|
@@ -63,11 +67,11 @@ export declare class ClickHouseStore {
|
|
|
63
67
|
insertProfileSamples(ctx: IngestContext, samples: ProfileSample[]): Promise<void>;
|
|
64
68
|
insertMemorySamples(ctx: IngestContext, samples: MemorySample[]): Promise<void>;
|
|
65
69
|
insertPlatformEvent(ctx: IngestContext, event: PlatformEvent): Promise<void>;
|
|
66
|
-
insertSourceMap(ctx: IngestContext, sourceMap:
|
|
67
|
-
|
|
68
|
-
|
|
69
|
-
|
|
70
|
-
|
|
70
|
+
insertSourceMap(ctx: IngestContext, sourceMap: ValidSourceMap): Promise<void>;
|
|
71
|
+
/** The uploaded source map for a minified file, or null (source-maps-lookup.ts). */
|
|
72
|
+
lookupSourceMap(key: SourceMapKey): Promise<string | null>;
|
|
73
|
+
/** Indexes of `check.entries` this tenant uploaded recently (see presentSourceMaps). */
|
|
74
|
+
sourceMapsPresent(ctx: IngestContext, check: SourceMapCheck): Promise<number[]>;
|
|
71
75
|
insertLlmCalls(ctx: IngestContext, calls: RuntimeLlmCall[]): Promise<void>;
|
|
72
76
|
insertMetricPoints(ctx: IngestContext, points: RuntimeMetricPoint[]): Promise<void>;
|
|
73
77
|
insertLatencyHistograms(ctx: IngestContext, points: LatencyHistogram[]): Promise<void>;
|
package/dist/clickhouse.js
CHANGED
|
@@ -3,9 +3,11 @@ import { latencyTableDDL } from "./latency.js";
|
|
|
3
3
|
import { profileTableDDL } from "./profiles.js";
|
|
4
4
|
import { memoryTableDDL, platformEventTableDDL } from "./memory.js";
|
|
5
5
|
import { sourceMapTableDDL } from "./source-maps.js";
|
|
6
|
+
import { attributesWithBreadcrumbs } from "./breadcrumbs.js";
|
|
7
|
+
import { lookupSourceMap, presentSourceMaps } from "./source-maps-lookup.js";
|
|
6
8
|
import { logTableDDL, requestRollupTableDDL, } from "./logs.js";
|
|
7
9
|
import { ingesterInfoTableDDL, sdkVersionTableDDL, } from "./sdk-versions.js";
|
|
8
|
-
import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
|
|
10
|
+
import { MIGRATIONS, OCCURRENCE_CODE_COLUMNS, OCCURRENCE_GROUPING_COLUMNS, OCCURRENCE_REQUEST_ID_INDEX, migrationsTableDDL, } from "./migrations.js";
|
|
9
11
|
/**
|
|
10
12
|
* ClickHouse persistence. The schema is idempotent and applied lazily once
|
|
11
13
|
* per process. Every row is keyed by (org_id, repository_id) — tenant
|
|
@@ -95,6 +97,7 @@ export class ClickHouseStore {
|
|
|
95
97
|
occurred_at DateTime64(3, 'UTC'),
|
|
96
98
|
ingested_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
97
99
|
${OCCURRENCE_CODE_COLUMNS.join(",\n\t\t\t\t")},
|
|
100
|
+
${OCCURRENCE_GROUPING_COLUMNS.join(",\n\t\t\t\t")},
|
|
98
101
|
${OCCURRENCE_REQUEST_ID_INDEX}
|
|
99
102
|
)
|
|
100
103
|
ENGINE = MergeTree
|
|
@@ -259,7 +262,9 @@ export class ClickHouseStore {
|
|
|
259
262
|
* this tenant in [from, to] — the cross-batch half of the log-promotion
|
|
260
263
|
* dedupe. `excludeIds` are the candidate occurrences' own deterministic
|
|
261
264
|
* ids, so a retried batch does not dedupe against its own earlier,
|
|
262
|
-
* partially-committed write.
|
|
265
|
+
* partially-committed write. Browser occurrences share the server's trace
|
|
266
|
+
* id (runtime-browser traceparent) but are a different signal, so they
|
|
267
|
+
* never suppress a server error. Throws on failure; the caller decides.
|
|
263
268
|
*/
|
|
264
269
|
async recentOccurrenceTraceIds(ctx, traceIds, from, to, excludeIds) {
|
|
265
270
|
if (traceIds.length === 0 || !this.configured)
|
|
@@ -269,6 +274,7 @@ export class ClickHouseStore {
|
|
|
269
274
|
query: `SELECT DISTINCT trace_id FROM ${this.table("runtime_error_occurrences")}
|
|
270
275
|
WHERE org_id = {org:String} AND repository_id = {repo:String}
|
|
271
276
|
AND occurred_at >= {from:DateTime64(3, 'UTC')} AND occurred_at <= {to:DateTime64(3, 'UTC')}
|
|
277
|
+
AND source != 'browser'
|
|
272
278
|
AND has({traces:Array(String)}, trace_id)
|
|
273
279
|
AND NOT has({exclude:Array(String)}, occurrence_id)`,
|
|
274
280
|
query_params: {
|
|
@@ -364,7 +370,7 @@ export class ClickHouseStore {
|
|
|
364
370
|
status_code: o.statusCode ?? 0,
|
|
365
371
|
trace_id: o.traceId ?? "",
|
|
366
372
|
session_id: o.sessionId ?? "",
|
|
367
|
-
attributes: JSON.stringify(o.attributes
|
|
373
|
+
attributes: JSON.stringify(attributesWithBreadcrumbs(o.attributes, o.breadcrumbs)),
|
|
368
374
|
occurred_at: o.occurredAt.toISOString(),
|
|
369
375
|
error_code: o.errorCode ?? "",
|
|
370
376
|
error_why: o.why ?? "",
|
|
@@ -373,6 +379,8 @@ export class ClickHouseStore {
|
|
|
373
379
|
expected: o.expected ? 1 : 0,
|
|
374
380
|
request_id: o.requestId ?? "",
|
|
375
381
|
fingerprint_scheme: o.fingerprintScheme,
|
|
382
|
+
frames: o.frames?.length ? JSON.stringify(o.frames) : "",
|
|
383
|
+
fingerprints: (o.fingerprints ?? []).map((candidate) => candidate.fingerprint),
|
|
376
384
|
})),
|
|
377
385
|
});
|
|
378
386
|
}
|
|
@@ -467,9 +475,23 @@ export class ClickHouseStore {
|
|
|
467
475
|
await this.getClient().insert({
|
|
468
476
|
table: this.table("runtime_source_maps"), format: "JSONEachRow",
|
|
469
477
|
clickhouse_settings: INSERT_SETTINGS,
|
|
470
|
-
values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId,
|
|
478
|
+
values: [{ org_id: ctx.orgId, repository_id: ctx.repositoryId, release: sourceMap.release,
|
|
479
|
+
filename: sourceMap.filename, map: sourceMap.map, debug_id: sourceMap.debugId,
|
|
480
|
+
content_hash: sourceMap.contentHash }],
|
|
471
481
|
});
|
|
472
482
|
}
|
|
483
|
+
/** The uploaded source map for a minified file, or null (source-maps-lookup.ts). */
|
|
484
|
+
async lookupSourceMap(key) {
|
|
485
|
+
if (!this.configured)
|
|
486
|
+
return null;
|
|
487
|
+
await this.ensureSchema();
|
|
488
|
+
return lookupSourceMap(this.getClient(), this.config.clickhouseDatabase, key);
|
|
489
|
+
}
|
|
490
|
+
/** Indexes of `check.entries` this tenant uploaded recently (see presentSourceMaps). */
|
|
491
|
+
async sourceMapsPresent(ctx, check) {
|
|
492
|
+
await this.ensureSchema();
|
|
493
|
+
return presentSourceMaps(this.getClient(), this.config.clickhouseDatabase, ctx, check);
|
|
494
|
+
}
|
|
473
495
|
async insertLlmCalls(ctx, calls) {
|
|
474
496
|
if (calls.length === 0 || !this.configured)
|
|
475
497
|
return;
|
|
@@ -89,6 +89,35 @@
|
|
|
89
89
|
"ingester": "1.5.0",
|
|
90
90
|
"migrations": ["0013-occurrence-codes"],
|
|
91
91
|
"sdks": { "@autter/runtime-node": "1.5.0", "@autter/runtime-next": "1.5.0", "@autter/runtime-browser": "1.4.0", "@autter/runtime-edge": "1.0.0" }
|
|
92
|
+
},
|
|
93
|
+
{
|
|
94
|
+
"id": "breadcrumbs",
|
|
95
|
+
"title": "Breadcrumbs: the steps before an error",
|
|
96
|
+
"ingester": "1.6.0",
|
|
97
|
+
"migrations": [],
|
|
98
|
+
"sdks": { "@autter/runtime-browser": "1.6.0", "@autter/runtime-node": "1.6.0", "@autter/runtime-next": "1.6.0" }
|
|
99
|
+
},
|
|
100
|
+
{
|
|
101
|
+
"id": "source_map_debug_ids",
|
|
102
|
+
"title": "Source map debug ids",
|
|
103
|
+
"ingester": "1.6.0",
|
|
104
|
+
"migrations": ["0017-source-maps-debug-id"],
|
|
105
|
+
"route": "/v1/sourcemaps/check",
|
|
106
|
+
"sdks": { "@autter/runtime-browser": "1.6.0", "@autter/runtime-next": "1.6.0" }
|
|
107
|
+
},
|
|
108
|
+
{
|
|
109
|
+
"id": "fingerprint_override",
|
|
110
|
+
"title": "Fingerprint override",
|
|
111
|
+
"ingester": "1.6.0",
|
|
112
|
+
"migrations": [],
|
|
113
|
+
"sdks": { "@autter/runtime-browser": "1.6.0", "@autter/runtime-node": "1.6.0", "@autter/runtime-next": "1.6.0", "@autter/runtime-edge": "1.1.0" }
|
|
114
|
+
},
|
|
115
|
+
{
|
|
116
|
+
"id": "browser_trace_propagation",
|
|
117
|
+
"title": "Browser trace propagation (browser errors linked to server traces)",
|
|
118
|
+
"ingester": "1.6.0",
|
|
119
|
+
"migrations": [],
|
|
120
|
+
"sdks": { "@autter/runtime-browser": "1.6.0", "@autter/runtime-next": "1.6.0", "@autter/runtime-node": "1.6.0" }
|
|
92
121
|
}
|
|
93
122
|
]
|
|
94
123
|
}
|
package/dist/compat.d.ts
CHANGED
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
* Adding a feature: append an entry to compat-manifest.json with the first
|
|
21
21
|
* ingester release that handles it, the migrations it writes to, its route
|
|
22
22
|
* (if it has its own), and the first SDK release that emits it. The
|
|
23
|
-
* consistency test in compat.test.ts checks ids, migrations and routes
|
|
23
|
+
* consistency test in compat.test.ts checks ids, migrations and routes, and
|
|
24
|
+
* that no version is ahead of its package's package.json: a feature for the
|
|
25
|
+
* next release bumps the packages it names in the same change.
|
|
24
26
|
*/
|
|
25
27
|
export interface CompatFeature {
|
|
26
28
|
id: string;
|
package/dist/compat.js
CHANGED
|
@@ -20,7 +20,9 @@
|
|
|
20
20
|
* Adding a feature: append an entry to compat-manifest.json with the first
|
|
21
21
|
* ingester release that handles it, the migrations it writes to, its route
|
|
22
22
|
* (if it has its own), and the first SDK release that emits it. The
|
|
23
|
-
* consistency test in compat.test.ts checks ids, migrations and routes
|
|
23
|
+
* consistency test in compat.test.ts checks ids, migrations and routes, and
|
|
24
|
+
* that no version is ahead of its package's package.json: a feature for the
|
|
25
|
+
* next release bumps the packages it names in the same change.
|
|
24
26
|
*/
|
|
25
27
|
import manifestJson from "./compat-manifest.json" with { type: "json" };
|
|
26
28
|
export const COMPAT_MANIFEST = manifestJson;
|
package/dist/config.d.ts
CHANGED
|
@@ -54,5 +54,11 @@ export interface IngesterConfig {
|
|
|
54
54
|
*/
|
|
55
55
|
redactValuePatterns?: string[];
|
|
56
56
|
redactKeyPatterns?: string[];
|
|
57
|
+
/**
|
|
58
|
+
* In-memory budget (MiB, estimated decoded size) for parsed source maps
|
|
59
|
+
* used to symbolicate browser frames at ingest (SOURCE_MAP_CACHE_MB,
|
|
60
|
+
* default 64).
|
|
61
|
+
*/
|
|
62
|
+
sourceMapCacheMb?: number;
|
|
57
63
|
}
|
|
58
64
|
export declare function loadConfig(): IngesterConfig;
|
package/dist/config.js
CHANGED
|
@@ -66,6 +66,7 @@ export function loadConfig() {
|
|
|
66
66
|
logTtlDays: intEnv("LOG_TTL_DAYS", 14),
|
|
67
67
|
redactValuePatterns: patternListEnv("AUTTER_REDACT_VALUE_PATTERNS"),
|
|
68
68
|
redactKeyPatterns: patternListEnv("AUTTER_REDACT_KEY_PATTERNS"),
|
|
69
|
+
sourceMapCacheMb: intEnv("SOURCE_MAP_CACHE_MB", 64),
|
|
69
70
|
};
|
|
70
71
|
if (!/^[A-Za-z_][A-Za-z0-9_]*$/.test(config.clickhouseDatabase)) {
|
|
71
72
|
throw new Error(`Invalid CLICKHOUSE_DATABASE name: ${config.clickhouseDatabase}`);
|
package/dist/context.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { isSensitiveKey, scrubText } from "./redact.js";
|
|
2
|
+
import { BREADCRUMBS_ATTRIBUTE } from "./breadcrumbs.js";
|
|
2
3
|
/** Bounded, defensive privacy boundary for custom telemetry from any OTLP SDK. */
|
|
3
4
|
export function sanitizeRuntimeContext(input) {
|
|
4
5
|
let budget = 512;
|
|
@@ -97,6 +98,8 @@ export function decodeOtlpAttributes(attributes) {
|
|
|
97
98
|
.sort((a, b) => Number(/^autter\.(?:operation|event)\./.test(b.key ?? "")) -
|
|
98
99
|
Number(/^autter\.(?:operation|event)\./.test(a.key ?? "")))
|
|
99
100
|
.slice(0, 128)
|
|
100
|
-
.filter(
|
|
101
|
+
.filter(
|
|
102
|
+
// Breadcrumbs are their own validated field (breadcrumbs.ts), never a plain attribute.
|
|
103
|
+
(a) => a.key && a.key !== BREADCRUMBS_ATTRIBUTE && !/^(?:__proto__|constructor|prototype)$/.test(a.key))
|
|
101
104
|
.map((a) => [a.key, decode(a.value)])));
|
|
102
105
|
}
|
|
@@ -0,0 +1,42 @@
|
|
|
1
|
+
import type { RuntimeOccurrenceInput } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* SDK fingerprint override (EI-16) → the `manual-v1` issue identity.
|
|
4
|
+
*
|
|
5
|
+
* Where it comes from:
|
|
6
|
+
* - browser events: `fingerprint` (a string or an array of strings), from
|
|
7
|
+
* `captureException(err, { fingerprint })` or `beforeSend`;
|
|
8
|
+
* - OTLP exception events (or their span) and promoted error logs:
|
|
9
|
+
* `autter.fingerprint`, a string or a JSON array of strings (the SDKs always
|
|
10
|
+
* send the JSON array, so a single value never reads as a list by accident).
|
|
11
|
+
*
|
|
12
|
+
* The normalisers carry it as `RuntimeOccurrenceInput.fingerprintOverride`:
|
|
13
|
+
* strings only, empty values dropped, at most 10 values, each scrubbed like
|
|
14
|
+
* the rest of the occurrence and then cut to 200 characters. A malformed
|
|
15
|
+
* override is dropped on its own; it never rejects the signal.
|
|
16
|
+
*
|
|
17
|
+
* `manualFingerprint` hashes it. `manual-v1` takes precedence over every
|
|
18
|
+
* other scheme (rule-v1, code-v1, frames-v3, message-v1) and is only ever
|
|
19
|
+
* computed here, never on the platform:
|
|
20
|
+
*
|
|
21
|
+
* sha256("manual-v1" \0 service \0 values.join("\u0001")).hex()[:32]
|
|
22
|
+
*/
|
|
23
|
+
export declare const MANUAL_FINGERPRINT_SCHEME = "manual-v1";
|
|
24
|
+
export declare const FINGERPRINT_ATTRIBUTE = "autter.fingerprint";
|
|
25
|
+
export declare const MAX_FINGERPRINT_VALUES = 10;
|
|
26
|
+
export declare const MAX_FINGERPRINT_VALUE_LENGTH = 200;
|
|
27
|
+
/**
|
|
28
|
+
* The override values carried on the wire, cleaned: `value` is a string, a
|
|
29
|
+
* JSON array string, or an already-decoded array. Undefined when absent or
|
|
30
|
+
* unusable.
|
|
31
|
+
*/
|
|
32
|
+
export declare function parseFingerprintOverride(value: unknown): string[] | undefined;
|
|
33
|
+
/** `{ fingerprintOverride }` to spread into an occurrence, or `{}`. */
|
|
34
|
+
export declare function fingerprintOverrideOf(value: unknown): {
|
|
35
|
+
fingerprintOverride?: string[];
|
|
36
|
+
};
|
|
37
|
+
/**
|
|
38
|
+
* The `manual-v1` fingerprint (32 lowercase hex) of an occurrence's override,
|
|
39
|
+
* or null when it has none. The values are bounded again here (at most 10,
|
|
40
|
+
* each ≤ 200 chars), so the hash never depends on how the input was built.
|
|
41
|
+
*/
|
|
42
|
+
export declare function manualFingerprint(input: Pick<RuntimeOccurrenceInput, "service" | "fingerprintOverride">): string | null;
|
|
@@ -0,0 +1,76 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { scrubText } from "./redact.js";
|
|
3
|
+
/**
|
|
4
|
+
* SDK fingerprint override (EI-16) → the `manual-v1` issue identity.
|
|
5
|
+
*
|
|
6
|
+
* Where it comes from:
|
|
7
|
+
* - browser events: `fingerprint` (a string or an array of strings), from
|
|
8
|
+
* `captureException(err, { fingerprint })` or `beforeSend`;
|
|
9
|
+
* - OTLP exception events (or their span) and promoted error logs:
|
|
10
|
+
* `autter.fingerprint`, a string or a JSON array of strings (the SDKs always
|
|
11
|
+
* send the JSON array, so a single value never reads as a list by accident).
|
|
12
|
+
*
|
|
13
|
+
* The normalisers carry it as `RuntimeOccurrenceInput.fingerprintOverride`:
|
|
14
|
+
* strings only, empty values dropped, at most 10 values, each scrubbed like
|
|
15
|
+
* the rest of the occurrence and then cut to 200 characters. A malformed
|
|
16
|
+
* override is dropped on its own; it never rejects the signal.
|
|
17
|
+
*
|
|
18
|
+
* `manualFingerprint` hashes it. `manual-v1` takes precedence over every
|
|
19
|
+
* other scheme (rule-v1, code-v1, frames-v3, message-v1) and is only ever
|
|
20
|
+
* computed here, never on the platform:
|
|
21
|
+
*
|
|
22
|
+
* sha256("manual-v1" \0 service \0 values.join("\u0001")).hex()[:32]
|
|
23
|
+
*/
|
|
24
|
+
export const MANUAL_FINGERPRINT_SCHEME = "manual-v1";
|
|
25
|
+
export const FINGERPRINT_ATTRIBUTE = "autter.fingerprint";
|
|
26
|
+
export const MAX_FINGERPRINT_VALUES = 10;
|
|
27
|
+
export const MAX_FINGERPRINT_VALUE_LENGTH = 200;
|
|
28
|
+
/** Strings only, empty dropped, first 10, each cut to 200 chars; null when none remain. */
|
|
29
|
+
function boundValues(list, scrub) {
|
|
30
|
+
const values = list
|
|
31
|
+
.filter((value) => typeof value === "string" && value !== "")
|
|
32
|
+
.slice(0, MAX_FINGERPRINT_VALUES)
|
|
33
|
+
// Cut with slack, scrub, then cut: a secret straddling the limit is
|
|
34
|
+
// masked whole.
|
|
35
|
+
.map((value) => scrub(value.slice(0, MAX_FINGERPRINT_VALUE_LENGTH + 512)).slice(0, MAX_FINGERPRINT_VALUE_LENGTH))
|
|
36
|
+
.filter((value) => value !== "");
|
|
37
|
+
return values.length > 0 ? values : null;
|
|
38
|
+
}
|
|
39
|
+
/**
|
|
40
|
+
* The override values carried on the wire, cleaned: `value` is a string, a
|
|
41
|
+
* JSON array string, or an already-decoded array. Undefined when absent or
|
|
42
|
+
* unusable.
|
|
43
|
+
*/
|
|
44
|
+
export function parseFingerprintOverride(value) {
|
|
45
|
+
let list = value;
|
|
46
|
+
if (typeof value === "string" && /^\s*\[/.test(value)) {
|
|
47
|
+
try {
|
|
48
|
+
list = JSON.parse(value);
|
|
49
|
+
}
|
|
50
|
+
catch {
|
|
51
|
+
/* not JSON: one plain value */
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
return boundValues(Array.isArray(list) ? list : [list], scrubText) ?? undefined;
|
|
55
|
+
}
|
|
56
|
+
/** `{ fingerprintOverride }` to spread into an occurrence, or `{}`. */
|
|
57
|
+
export function fingerprintOverrideOf(value) {
|
|
58
|
+
const values = parseFingerprintOverride(value);
|
|
59
|
+
return values ? { fingerprintOverride: values } : {};
|
|
60
|
+
}
|
|
61
|
+
/**
|
|
62
|
+
* The `manual-v1` fingerprint (32 lowercase hex) of an occurrence's override,
|
|
63
|
+
* or null when it has none. The values are bounded again here (at most 10,
|
|
64
|
+
* each ≤ 200 chars), so the hash never depends on how the input was built.
|
|
65
|
+
*/
|
|
66
|
+
export function manualFingerprint(input) {
|
|
67
|
+
if (!Array.isArray(input.fingerprintOverride))
|
|
68
|
+
return null;
|
|
69
|
+
const values = boundValues(input.fingerprintOverride, (text) => text);
|
|
70
|
+
if (!values)
|
|
71
|
+
return null;
|
|
72
|
+
return createHash("sha256")
|
|
73
|
+
.update(`${MANUAL_FINGERPRINT_SCHEME}\u0000${input.service}\u0000${values.join("\u0001")}`)
|
|
74
|
+
.digest("hex")
|
|
75
|
+
.slice(0, 32);
|
|
76
|
+
}
|
|
@@ -0,0 +1,54 @@
|
|
|
1
|
+
import type { FingerprintCandidate, FingerprintRecord, RuntimeOccurrenceInput, StackFrame } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* `frames-v3` — the structural issue identity (contract §2). Byte-for-byte
|
|
4
|
+
* shared with the platform mirror (`runtime-frames-fingerprint.ts`); the
|
|
5
|
+
* vectors in fingerprint-v3-vectors.json pin both.
|
|
6
|
+
*
|
|
7
|
+
* It hashes service + error type + up to five in-app frames reduced to
|
|
8
|
+
* `function@file`, where file is a lowercased basename with build hashes
|
|
9
|
+
* (`index-5f3a9c1e.js` → `index-*.js`) templated out. Line and column
|
|
10
|
+
* numbers, route, source (browser vs server) and release are never part of
|
|
11
|
+
* it, so one bug stays one issue across deploys, pages, preview domains and
|
|
12
|
+
* tiers. The normalised message is hashed only when no frame was picked.
|
|
13
|
+
*/
|
|
14
|
+
export declare const FRAMES_V3 = "frames-v3";
|
|
15
|
+
/** §2.3 — the function half of a frame token. */
|
|
16
|
+
export declare function frameTokenFunction(frame: Pick<StackFrame, "function" | "resolved">): string;
|
|
17
|
+
/** §2.3 — the file half of a frame token. */
|
|
18
|
+
export declare function frameTokenFile(frame: Pick<StackFrame, "filename" | "module">): string;
|
|
19
|
+
export declare function frameToken(frame: StackFrame): string;
|
|
20
|
+
/** §2.1–2.2 — in-app frames in order, else only the first frame; at most 5. */
|
|
21
|
+
export declare function pickFrames(frames: StackFrame[]): StackFrame[];
|
|
22
|
+
export interface FramesV3Input {
|
|
23
|
+
service: string;
|
|
24
|
+
errorType: string;
|
|
25
|
+
message: string;
|
|
26
|
+
frames: StackFrame[];
|
|
27
|
+
}
|
|
28
|
+
export interface FramesV3Result {
|
|
29
|
+
fingerprint: string;
|
|
30
|
+
record: FingerprintRecord;
|
|
31
|
+
}
|
|
32
|
+
export declare function framesV3Fingerprint(input: FramesV3Input): FramesV3Result;
|
|
33
|
+
export interface OccurrenceIdentity {
|
|
34
|
+
/** Legacy identity, unchanged: what `fingerprint`/`fingerprintScheme` carry. */
|
|
35
|
+
legacy: {
|
|
36
|
+
fingerprint: string;
|
|
37
|
+
scheme: "message-v1" | "code-v1";
|
|
38
|
+
};
|
|
39
|
+
/** Every applicable scheme, highest precedence first; ends with the legacy one. */
|
|
40
|
+
candidates: FingerprintCandidate[];
|
|
41
|
+
/** What went into the frames-v3 fingerprint, when one was computed. */
|
|
42
|
+
record: FingerprintRecord | null;
|
|
43
|
+
}
|
|
44
|
+
/**
|
|
45
|
+
* The candidate list the sink carries (contract §1/§4), highest precedence
|
|
46
|
+
* first:
|
|
47
|
+
* manual-v1 (SDK override) → code-v1 (valid error code) → frames-v3 →
|
|
48
|
+
* message-v1 (legacy).
|
|
49
|
+
* A coded error's legacy identity IS code-v1, which outranks every auto
|
|
50
|
+
* scheme, so its list is [manual-v1?, code-v1]; an uncoded one is
|
|
51
|
+
* [manual-v1?, frames-v3, message-v1]. Either way the list ends with the
|
|
52
|
+
* legacy fingerprint, so the platform can always fall back to it.
|
|
53
|
+
*/
|
|
54
|
+
export declare function occurrenceIdentity(input: RuntimeOccurrenceInput, frames: StackFrame[]): OccurrenceIdentity;
|