@autter/otlp-ingester 1.5.0 → 1.6.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -0
- package/dist/breadcrumbs.d.ts +41 -0
- package/dist/breadcrumbs.js +151 -0
- package/dist/clickhouse.d.ts +10 -6
- package/dist/clickhouse.js +38 -5
- package/dist/compat-manifest.json +29 -0
- package/dist/compat.d.ts +3 -1
- package/dist/compat.js +3 -1
- package/dist/config.d.ts +6 -0
- package/dist/config.js +1 -0
- package/dist/context.js +4 -1
- package/dist/fingerprint-override.d.ts +42 -0
- package/dist/fingerprint-override.js +76 -0
- package/dist/fingerprint-v3.d.ts +54 -0
- package/dist/fingerprint-v3.js +96 -0
- package/dist/fingerprint.d.ts +25 -0
- package/dist/fingerprint.js +1 -1
- package/dist/logs.js +2 -0
- package/dist/memory.d.ts +2 -2
- package/dist/migrations.d.ts +10 -0
- package/dist/migrations.js +32 -1
- package/dist/normalize-browser.d.ts +67 -34
- package/dist/normalize-browser.js +24 -1
- package/dist/normalize-otlp.js +4 -0
- package/dist/server.js +94 -18
- package/dist/sink.d.ts +5 -1
- package/dist/sink.js +13 -2
- package/dist/source-map-mappings.d.ts +57 -0
- package/dist/source-map-mappings.js +245 -0
- package/dist/source-map-scopes.d.ts +71 -0
- package/dist/source-map-scopes.js +190 -0
- package/dist/source-maps-lookup.d.ts +31 -0
- package/dist/source-maps-lookup.js +64 -0
- package/dist/source-maps.d.ts +72 -2
- package/dist/source-maps.js +134 -14
- package/dist/stack-fixtures.d.ts +10 -2
- package/dist/stack-fixtures.js +13 -3
- package/dist/stack-frames.d.ts +48 -0
- package/dist/stack-frames.js +441 -0
- package/dist/symbolicate.d.ts +117 -0
- package/dist/symbolicate.js +363 -0
- package/dist/types.d.ts +68 -0
- package/package.json +2 -1
|
@@ -0,0 +1,190 @@
|
|
|
1
|
+
import { forEachVlqGroup } from "./source-map-mappings.js";
|
|
2
|
+
/**
|
|
3
|
+
* Function scopes uploaded with a source map (`x_autter_scopes`, written by
|
|
4
|
+
* `autter-runtime sourcemaps upload` in @autter/runtime-cli after 1.0.0):
|
|
5
|
+
* every function of the bundle as served, with its generated range and its
|
|
6
|
+
* ORIGINAL name. A frame is named after the innermost function that
|
|
7
|
+
* contains it — the name of the function it is in, as Sentry's
|
|
8
|
+
* js-source-scopes and PostHog cymbal's `token.scope()` do — instead of the
|
|
9
|
+
* name mapped at its caller's position, which is wrong whenever the call
|
|
10
|
+
* goes through a variable or a framework.
|
|
11
|
+
*
|
|
12
|
+
* Shape: `{ version: 1, names: (string | null)[], scopes: string }`.
|
|
13
|
+
* `scopes` is encoded like `mappings`: base64 VLQ, `;` between generated
|
|
14
|
+
* lines (the line each scope starts on), `,` between the scopes that start
|
|
15
|
+
* on one line, in start order (outer first on a tie). Each scope is 3 or 4
|
|
16
|
+
* values, 0-based and in UTF-16 columns:
|
|
17
|
+
*
|
|
18
|
+
* 1. start column, relative to the previous scope's start on the same
|
|
19
|
+
* line (absolute for a line's first scope);
|
|
20
|
+
* 2. end line, relative to the start line;
|
|
21
|
+
* 3. end column (exclusive), relative to the start column when the scope
|
|
22
|
+
* ends on its start line, else absolute;
|
|
23
|
+
* 4. the name, when the function has one: an index into the map's own
|
|
24
|
+
* `names` followed by `x_autter_scopes.names`, relative to the last
|
|
25
|
+
* name index written (starting from 0). A `null` entry in
|
|
26
|
+
* `x_autter_scopes.names` is a name the CLI could not map back.
|
|
27
|
+
*
|
|
28
|
+
* A 3-value scope is anonymous. Anything else (wrong version, bad VLQ,
|
|
29
|
+
* out-of-range names, ranges that overlap without nesting) makes the whole
|
|
30
|
+
* field unusable, and the map is used as if it had none.
|
|
31
|
+
*/
|
|
32
|
+
export const SCOPES_KEY = "x_autter_scopes";
|
|
33
|
+
const ANONYMOUS = -1;
|
|
34
|
+
/** startLine, startCol, endLine, endCol, nameIndex, parent. */
|
|
35
|
+
const STRIDE = 6;
|
|
36
|
+
const MAX_POSITION = 0x7fffffff;
|
|
37
|
+
const ANONYMOUS_SCOPE = { kind: "anonymous" };
|
|
38
|
+
const UNKNOWN_SCOPE = { kind: "unknown" };
|
|
39
|
+
class MalformedScopes extends Error {
|
|
40
|
+
}
|
|
41
|
+
/** a < b for (line, column) pairs. */
|
|
42
|
+
function before(lineA, colA, lineB, colB) {
|
|
43
|
+
return lineA < lineB || (lineA === lineB && colA < colB);
|
|
44
|
+
}
|
|
45
|
+
/** Decode the `scopes` string into [startLine, startCol, endLine, endCol, nameIndex] rows. */
|
|
46
|
+
function decodeScopes(text, isName) {
|
|
47
|
+
const rows = [];
|
|
48
|
+
let currentLine = 0;
|
|
49
|
+
let startCol = 0;
|
|
50
|
+
let name = 0;
|
|
51
|
+
forEachVlqGroup(text, (line, values, count) => {
|
|
52
|
+
if (count !== 3 && count !== 4)
|
|
53
|
+
throw new MalformedScopes("a scope has 3 or 4 values");
|
|
54
|
+
if (line !== currentLine) {
|
|
55
|
+
currentLine = line;
|
|
56
|
+
startCol = 0;
|
|
57
|
+
}
|
|
58
|
+
startCol += values[0];
|
|
59
|
+
const endLine = line + values[1];
|
|
60
|
+
const endCol = values[1] === 0 ? startCol + values[2] : values[2];
|
|
61
|
+
let nameIndex = ANONYMOUS;
|
|
62
|
+
if (count === 4) {
|
|
63
|
+
name += values[3];
|
|
64
|
+
if (!isName(name))
|
|
65
|
+
throw new MalformedScopes("name out of range");
|
|
66
|
+
nameIndex = name;
|
|
67
|
+
}
|
|
68
|
+
if (startCol < 0 || values[1] < 0 || endCol < 0 || endLine > MAX_POSITION || Math.max(startCol, endCol) > MAX_POSITION) {
|
|
69
|
+
throw new MalformedScopes("position out of range");
|
|
70
|
+
}
|
|
71
|
+
// Non-empty: a function is at least `()=>0`.
|
|
72
|
+
if (!before(line, startCol, endLine, endCol))
|
|
73
|
+
throw new MalformedScopes("empty scope");
|
|
74
|
+
rows.push([line, startCol, endLine, endCol, nameIndex]);
|
|
75
|
+
});
|
|
76
|
+
return rows;
|
|
77
|
+
}
|
|
78
|
+
export class ScopeIndex {
|
|
79
|
+
table;
|
|
80
|
+
mapNames;
|
|
81
|
+
extraNames;
|
|
82
|
+
constructor(
|
|
83
|
+
/** STRIDE ints per scope, sorted by start (outer first on a tie). */
|
|
84
|
+
table,
|
|
85
|
+
/** The map's own `names`, then `x_autter_scopes.names`. */
|
|
86
|
+
mapNames, extraNames) {
|
|
87
|
+
this.table = table;
|
|
88
|
+
this.mapNames = mapNames;
|
|
89
|
+
this.extraNames = extraNames;
|
|
90
|
+
}
|
|
91
|
+
/** Number of function scopes. */
|
|
92
|
+
get size() {
|
|
93
|
+
return this.table.length / STRIDE;
|
|
94
|
+
}
|
|
95
|
+
/** Estimated memory, for the source map cache's byte budget (the map's
|
|
96
|
+
* own `names` are counted with the map). */
|
|
97
|
+
get bytes() {
|
|
98
|
+
let bytes = this.table.byteLength + 64;
|
|
99
|
+
for (const name of this.extraNames)
|
|
100
|
+
bytes += 2 * (name?.length ?? 0) + 32;
|
|
101
|
+
return bytes;
|
|
102
|
+
}
|
|
103
|
+
/**
|
|
104
|
+
* The parsed `x_autter_scopes` value, or null when it is absent or
|
|
105
|
+
* malformed. `mapNames` is the map's own `names` array, which scope name
|
|
106
|
+
* indexes continue.
|
|
107
|
+
*/
|
|
108
|
+
static from(raw, mapNames = []) {
|
|
109
|
+
if (!raw || typeof raw !== "object" || Array.isArray(raw))
|
|
110
|
+
return null;
|
|
111
|
+
const { version, names, scopes } = raw;
|
|
112
|
+
if (version !== 1 || !Array.isArray(names) || typeof scopes !== "string")
|
|
113
|
+
return null;
|
|
114
|
+
if (!names.every((name) => typeof name === "string" || name === null))
|
|
115
|
+
return null;
|
|
116
|
+
const extraNames = names;
|
|
117
|
+
const isName = (index) => index >= 0 &&
|
|
118
|
+
(index < mapNames.length
|
|
119
|
+
? typeof mapNames[index] === "string"
|
|
120
|
+
: index - mapNames.length < extraNames.length);
|
|
121
|
+
let entries;
|
|
122
|
+
try {
|
|
123
|
+
entries = decodeScopes(scopes, isName);
|
|
124
|
+
}
|
|
125
|
+
catch {
|
|
126
|
+
return null;
|
|
127
|
+
}
|
|
128
|
+
// Encoded in order already; sort anyway so a hand-written field works too.
|
|
129
|
+
entries.sort((a, b) => a[0] - b[0] || a[1] - b[1] || b[2] - a[2] || b[3] - a[3]);
|
|
130
|
+
const table = new Int32Array(entries.length * STRIDE);
|
|
131
|
+
// Open scopes, outermost first: each new scope either nests in the
|
|
132
|
+
// innermost one still open or starts after it has ended.
|
|
133
|
+
const open = [];
|
|
134
|
+
for (let i = 0; i < entries.length; i++) {
|
|
135
|
+
const [startLine, startCol, endLine, endCol, nameIndex] = entries[i];
|
|
136
|
+
while (open.length > 0) {
|
|
137
|
+
const top = open[open.length - 1] * STRIDE;
|
|
138
|
+
if (before(startLine, startCol, table[top + 2], table[top + 3]))
|
|
139
|
+
break;
|
|
140
|
+
open.pop();
|
|
141
|
+
}
|
|
142
|
+
const parent = open.length > 0 ? open[open.length - 1] : -1;
|
|
143
|
+
if (parent >= 0 && before(table[parent * STRIDE + 2], table[parent * STRIDE + 3], endLine, endCol)) {
|
|
144
|
+
return null; // overlaps its parent without nesting in it
|
|
145
|
+
}
|
|
146
|
+
const at = i * STRIDE;
|
|
147
|
+
table[at] = startLine;
|
|
148
|
+
table[at + 1] = startCol;
|
|
149
|
+
table[at + 2] = endLine;
|
|
150
|
+
table[at + 3] = endCol;
|
|
151
|
+
table[at + 4] = nameIndex;
|
|
152
|
+
table[at + 5] = parent;
|
|
153
|
+
open.push(i);
|
|
154
|
+
}
|
|
155
|
+
return new ScopeIndex(table, mapNames, extraNames);
|
|
156
|
+
}
|
|
157
|
+
nameOf(index) {
|
|
158
|
+
if (index === ANONYMOUS)
|
|
159
|
+
return ANONYMOUS_SCOPE;
|
|
160
|
+
const name = index < this.mapNames.length ? this.mapNames[index] : this.extraNames[index - this.mapNames.length];
|
|
161
|
+
return typeof name === "string" ? { kind: "named", name } : UNKNOWN_SCOPE;
|
|
162
|
+
}
|
|
163
|
+
/**
|
|
164
|
+
* The name for a frame at a 0-based generated (line, column): the
|
|
165
|
+
* innermost function containing it. Outside every function (top-level
|
|
166
|
+
* code) it is anonymous.
|
|
167
|
+
*/
|
|
168
|
+
nameAt(line, column) {
|
|
169
|
+
const t = this.table;
|
|
170
|
+
// Last scope that starts at or before the position…
|
|
171
|
+
let lo = 0;
|
|
172
|
+
let hi = this.size - 1;
|
|
173
|
+
let found = -1;
|
|
174
|
+
while (lo <= hi) {
|
|
175
|
+
const mid = (lo + hi) >> 1;
|
|
176
|
+
if (before(line, column, t[mid * STRIDE], t[mid * STRIDE + 1]))
|
|
177
|
+
hi = mid - 1;
|
|
178
|
+
else {
|
|
179
|
+
found = mid;
|
|
180
|
+
lo = mid + 1;
|
|
181
|
+
}
|
|
182
|
+
}
|
|
183
|
+
// …or, when it ended before the position, its closest ancestor that hasn't.
|
|
184
|
+
for (let i = found; i >= 0; i = t[i * STRIDE + 5]) {
|
|
185
|
+
if (before(line, column, t[i * STRIDE + 2], t[i * STRIDE + 3]))
|
|
186
|
+
return this.nameOf(t[i * STRIDE + 4]);
|
|
187
|
+
}
|
|
188
|
+
return ANONYMOUS_SCOPE;
|
|
189
|
+
}
|
|
190
|
+
}
|
|
@@ -0,0 +1,31 @@
|
|
|
1
|
+
import type { ClickHouseClient } from "@clickhouse/client";
|
|
2
|
+
import { type SourceMapCheck } from "./source-maps.js";
|
|
3
|
+
/**
|
|
4
|
+
* Source map lookups against `runtime_source_maps` (EI-13). Uploads come
|
|
5
|
+
* from `autter-runtime sourcemaps upload` / the bundler plugins, or any
|
|
6
|
+
* client of `POST /v1/sourcemaps`.
|
|
7
|
+
*/
|
|
8
|
+
export interface SourceMapKey {
|
|
9
|
+
orgId: string;
|
|
10
|
+
repositoryId: string;
|
|
11
|
+
release: string | null;
|
|
12
|
+
filename: string;
|
|
13
|
+
debugId?: string | null;
|
|
14
|
+
}
|
|
15
|
+
/** The stored map JSON for a minified file: by debug id first when given, else by release + filename (path only, no origin/query). Null when none. */
|
|
16
|
+
export declare function lookupSourceMap(client: ClickHouseClient, database: string, key: SourceMapKey): Promise<string | null>;
|
|
17
|
+
/**
|
|
18
|
+
* Re-uploading a map refreshes its 30-day TTL, so a map only counts as
|
|
19
|
+
* already uploaded when it was stored this recently; older ones are sent
|
|
20
|
+
* again and stay alive while the release is still being built.
|
|
21
|
+
*/
|
|
22
|
+
export declare const SOURCE_MAP_FRESH_DAYS = 7;
|
|
23
|
+
/**
|
|
24
|
+
* Which check entries the tenant already has: same release, path, debug id
|
|
25
|
+
* and content hash, uploaded in the last SOURCE_MAP_FRESH_DAYS. Returns the
|
|
26
|
+
* indexes of `check.entries` that need no upload.
|
|
27
|
+
*/
|
|
28
|
+
export declare function presentSourceMaps(client: ClickHouseClient, database: string, tenant: {
|
|
29
|
+
orgId: string;
|
|
30
|
+
repositoryId: string;
|
|
31
|
+
}, check: SourceMapCheck): Promise<number[]>;
|
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
import { normalizeDebugId, sourceMapPath, } from "./source-maps.js";
|
|
2
|
+
async function firstMap(client, database, where, params) {
|
|
3
|
+
const rows = await client.query({
|
|
4
|
+
query: `SELECT map FROM ${database}.runtime_source_maps
|
|
5
|
+
WHERE org_id = {org:String} AND repository_id = {repo:String} AND ${where}
|
|
6
|
+
ORDER BY uploaded_at DESC LIMIT 1`,
|
|
7
|
+
query_params: params,
|
|
8
|
+
format: "JSONEachRow",
|
|
9
|
+
});
|
|
10
|
+
const [row] = await rows.json();
|
|
11
|
+
return row?.map ?? null;
|
|
12
|
+
}
|
|
13
|
+
/** The stored map JSON for a minified file: by debug id first when given, else by release + filename (path only, no origin/query). Null when none. */
|
|
14
|
+
export async function lookupSourceMap(client, database, key) {
|
|
15
|
+
const tenant = { org: key.orgId, repo: key.repositoryId };
|
|
16
|
+
// A debug id names the exact bundle bytes, so it matches whatever release
|
|
17
|
+
// the SDK reported (or none). Fall through to release + path when no map
|
|
18
|
+
// was uploaded with this id (old CLI, plain curl upload).
|
|
19
|
+
const debugId = normalizeDebugId(key.debugId);
|
|
20
|
+
if (debugId) {
|
|
21
|
+
const map = await firstMap(client, database, "debug_id = {debugId:String}", { ...tenant, debugId });
|
|
22
|
+
if (map !== null)
|
|
23
|
+
return map;
|
|
24
|
+
}
|
|
25
|
+
const filename = sourceMapPath(key.filename);
|
|
26
|
+
if (!key.release || !filename)
|
|
27
|
+
return null;
|
|
28
|
+
return firstMap(client, database, "release = {release:String} AND filename = {filename:String}", { ...tenant, release: key.release, filename });
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Re-uploading a map refreshes its 30-day TTL, so a map only counts as
|
|
32
|
+
* already uploaded when it was stored this recently; older ones are sent
|
|
33
|
+
* again and stay alive while the release is still being built.
|
|
34
|
+
*/
|
|
35
|
+
export const SOURCE_MAP_FRESH_DAYS = 7;
|
|
36
|
+
/**
|
|
37
|
+
* Which check entries the tenant already has: same release, path, debug id
|
|
38
|
+
* and content hash, uploaded in the last SOURCE_MAP_FRESH_DAYS. Returns the
|
|
39
|
+
* indexes of `check.entries` that need no upload.
|
|
40
|
+
*/
|
|
41
|
+
export async function presentSourceMaps(client, database, tenant, check) {
|
|
42
|
+
if (check.entries.length === 0)
|
|
43
|
+
return [];
|
|
44
|
+
const rows = await client.query({
|
|
45
|
+
query: `SELECT DISTINCT filename, debug_id, content_hash FROM ${database}.runtime_source_maps
|
|
46
|
+
WHERE org_id = {org:String} AND repository_id = {repo:String} AND release = {release:String}
|
|
47
|
+
AND has({hashes:Array(String)}, content_hash)
|
|
48
|
+
AND uploaded_at >= now64(3) - INTERVAL ${SOURCE_MAP_FRESH_DAYS} DAY`,
|
|
49
|
+
query_params: {
|
|
50
|
+
org: tenant.orgId,
|
|
51
|
+
repo: tenant.repositoryId,
|
|
52
|
+
release: check.release,
|
|
53
|
+
hashes: [...new Set(check.entries.map((entry) => entry.hash))],
|
|
54
|
+
},
|
|
55
|
+
format: "JSONEachRow",
|
|
56
|
+
});
|
|
57
|
+
const stored = new Set((await rows.json()).map((row) => `${row.filename}\u0000${row.debug_id}\u0000${row.content_hash}`));
|
|
58
|
+
const present = [];
|
|
59
|
+
check.entries.forEach((entry, index) => {
|
|
60
|
+
if (stored.has(`${entry.filename}\u0000${entry.debugId}\u0000${entry.hash}`))
|
|
61
|
+
present.push(index);
|
|
62
|
+
});
|
|
63
|
+
return present;
|
|
64
|
+
}
|
package/dist/source-maps.d.ts
CHANGED
|
@@ -1,6 +1,76 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Largest stored map (after `sourcesContent` is removed), in characters:
|
|
3
|
+
* 32 MiB, room for vendor chunks of about 20 MB of minified code. Storage
|
|
4
|
+
* is a ZSTD-compressed String column, so size only costs disk; a map this
|
|
5
|
+
* big is still symbolicated, without being kept in the parse cache unless
|
|
6
|
+
* SOURCE_MAP_CACHE_MB allows it (symbolicate.ts). Was 5 MiB up to 1.6.0.
|
|
7
|
+
*/
|
|
8
|
+
export declare const SOURCE_MAP_MAX_CHARS: number;
|
|
9
|
+
/** `/v1/sourcemaps` body limit, counted after a gzip or deflate body is
|
|
10
|
+
* inflated: the largest map plus room for the rest of the request. */
|
|
11
|
+
export declare const SOURCE_MAP_MAX_BODY_BYTES: number;
|
|
12
|
+
/** A debug id: 8-4-4-4-12 lowercase hex. Written by `autter-runtime
|
|
13
|
+
* sourcemaps inject`, and by Rollup, Turbopack, esbuild and Sentry tooling. */
|
|
14
|
+
export declare const DEBUG_ID_RE: RegExp;
|
|
15
|
+
/** Lowercased debug id, or null when the value isn't one. */
|
|
16
|
+
export declare function normalizeDebugId(value: unknown): string | null;
|
|
17
|
+
/**
|
|
18
|
+
* The stored form of a bundle's filename: path only, with no origin, query
|
|
19
|
+
* or hash ("https://cdn.example.com/assets/app.js?v=2" → "/assets/app.js").
|
|
20
|
+
* Uploads and lookups both go through it so they always agree.
|
|
21
|
+
*/
|
|
22
|
+
export declare function sourceMapPath(filename: string): string | null;
|
|
23
|
+
/** Migration 0016 columns — shared with the baseline DDL below. */
|
|
24
|
+
export declare const SOURCE_MAP_DEBUG_COLUMNS: string[];
|
|
25
|
+
export declare const SOURCE_MAP_DEBUG_ID_INDEX = "INDEX idx_sm_debug_id debug_id TYPE bloom_filter GRANULARITY 1";
|
|
1
26
|
export declare function sourceMapTableDDL(db: string): string;
|
|
2
|
-
export
|
|
27
|
+
export interface ValidSourceMap {
|
|
3
28
|
release: string;
|
|
29
|
+
/** sourceMapPath() of the uploaded filename. */
|
|
4
30
|
filename: string;
|
|
31
|
+
/** The map JSON without `sourcesContent`. */
|
|
5
32
|
map: string;
|
|
6
|
-
|
|
33
|
+
/** Lowercase debug id, or "" when the upload has none. */
|
|
34
|
+
debugId: string;
|
|
35
|
+
/** sha256 hex of `map`. */
|
|
36
|
+
contentHash: string;
|
|
37
|
+
}
|
|
38
|
+
/** The content hash of a stored map — `autter-runtime sourcemaps upload`
|
|
39
|
+
* computes the same value to skip maps the ingester already has. */
|
|
40
|
+
export declare function sourceMapContentHash(map: string): string;
|
|
41
|
+
/**
|
|
42
|
+
* Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
|
|
43
|
+
* `map` may be an object or a JSON string. The debug id comes from the body,
|
|
44
|
+
* else from the map's own `debugId` (or legacy `debug_id`); an invalid one
|
|
45
|
+
* rejects the upload. Null when the upload is invalid.
|
|
46
|
+
*/
|
|
47
|
+
export declare function validateSourceMap(body: unknown): ValidSourceMap | null;
|
|
48
|
+
/** One entry of a `/v1/sourcemaps/check` request. */
|
|
49
|
+
export interface SourceMapCheckEntry {
|
|
50
|
+
filename: string;
|
|
51
|
+
debugId: string;
|
|
52
|
+
hash: string;
|
|
53
|
+
}
|
|
54
|
+
export interface SourceMapCheck {
|
|
55
|
+
release: string;
|
|
56
|
+
entries: SourceMapCheckEntry[];
|
|
57
|
+
}
|
|
58
|
+
/** Most entries one check request may name. */
|
|
59
|
+
export declare const SOURCE_MAP_CHECK_MAX = 500;
|
|
60
|
+
/**
|
|
61
|
+
* Validate a `/v1/sourcemaps/check` request:
|
|
62
|
+
* `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
|
|
63
|
+
* filenames and debug ids are normalised exactly as an upload would store
|
|
64
|
+
* them. Null when invalid.
|
|
65
|
+
*/
|
|
66
|
+
export declare function validateSourceMapCheck(body: unknown): SourceMapCheck | null;
|
|
67
|
+
/** Most scripts one browser event may name in `debugIds`. */
|
|
68
|
+
export declare const DEBUG_IDS_MAX = 50;
|
|
69
|
+
/**
|
|
70
|
+
* The `debugIds` of a browser event (runtime-browser 1.6.0+, EI-13): script
|
|
71
|
+
* URL → debug id. Keys lose their query and hash and are scrubbed, invalid ids
|
|
72
|
+
* are dropped one by one, at most 50 are kept. `{ debugIds }` for the occurrence, or `{}`.
|
|
73
|
+
*/
|
|
74
|
+
export declare function browserDebugIds(raw: Record<string, unknown> | undefined): {
|
|
75
|
+
debugIds?: Record<string, string>;
|
|
76
|
+
};
|
package/dist/source-maps.js
CHANGED
|
@@ -1,27 +1,83 @@
|
|
|
1
|
+
import { createHash } from "node:crypto";
|
|
2
|
+
import { scrubText } from "./redact.js";
|
|
3
|
+
/**
|
|
4
|
+
* Largest stored map (after `sourcesContent` is removed), in characters:
|
|
5
|
+
* 32 MiB, room for vendor chunks of about 20 MB of minified code. Storage
|
|
6
|
+
* is a ZSTD-compressed String column, so size only costs disk; a map this
|
|
7
|
+
* big is still symbolicated, without being kept in the parse cache unless
|
|
8
|
+
* SOURCE_MAP_CACHE_MB allows it (symbolicate.ts). Was 5 MiB up to 1.6.0.
|
|
9
|
+
*/
|
|
10
|
+
export const SOURCE_MAP_MAX_CHARS = 32 * 1024 * 1024;
|
|
11
|
+
/** `/v1/sourcemaps` body limit, counted after a gzip or deflate body is
|
|
12
|
+
* inflated: the largest map plus room for the rest of the request. */
|
|
13
|
+
export const SOURCE_MAP_MAX_BODY_BYTES = SOURCE_MAP_MAX_CHARS + 1024 * 1024;
|
|
14
|
+
/** Bundles a map can be uploaded for: `.js`, `.mjs` and `.cjs` (1.6.0+). */
|
|
15
|
+
const BUNDLE_PATH_RE = /\.[cm]?js$/;
|
|
16
|
+
/** A debug id: 8-4-4-4-12 lowercase hex. Written by `autter-runtime
|
|
17
|
+
* sourcemaps inject`, and by Rollup, Turbopack, esbuild and Sentry tooling. */
|
|
18
|
+
export const DEBUG_ID_RE = /^[0-9a-f]{8}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{4}-[0-9a-f]{12}$/;
|
|
19
|
+
/** Lowercased debug id, or null when the value isn't one. */
|
|
20
|
+
export function normalizeDebugId(value) {
|
|
21
|
+
if (typeof value !== "string")
|
|
22
|
+
return null;
|
|
23
|
+
const id = value.trim().toLowerCase();
|
|
24
|
+
return DEBUG_ID_RE.test(id) ? id : null;
|
|
25
|
+
}
|
|
26
|
+
/**
|
|
27
|
+
* The stored form of a bundle's filename: path only, with no origin, query
|
|
28
|
+
* or hash ("https://cdn.example.com/assets/app.js?v=2" → "/assets/app.js").
|
|
29
|
+
* Uploads and lookups both go through it so they always agree.
|
|
30
|
+
*/
|
|
31
|
+
export function sourceMapPath(filename) {
|
|
32
|
+
if (filename.length > 1000)
|
|
33
|
+
return null;
|
|
34
|
+
try {
|
|
35
|
+
return new URL(filename, "https://autter.invalid").pathname;
|
|
36
|
+
}
|
|
37
|
+
catch {
|
|
38
|
+
return null;
|
|
39
|
+
}
|
|
40
|
+
}
|
|
41
|
+
/** Migration 0016 columns — shared with the baseline DDL below. */
|
|
42
|
+
export const SOURCE_MAP_DEBUG_COLUMNS = [
|
|
43
|
+
// The bundle's debug id ('' for maps uploaded without one).
|
|
44
|
+
"debug_id String DEFAULT ''",
|
|
45
|
+
// sha256 of the stored map, so CI can skip maps it already uploaded.
|
|
46
|
+
"content_hash String DEFAULT ''",
|
|
47
|
+
];
|
|
48
|
+
export const SOURCE_MAP_DEBUG_ID_INDEX = "INDEX idx_sm_debug_id debug_id TYPE bloom_filter GRANULARITY 1";
|
|
1
49
|
export function sourceMapTableDDL(db) {
|
|
2
50
|
return `CREATE TABLE IF NOT EXISTS ${db}.runtime_source_maps (
|
|
3
51
|
org_id String, repository_id String, release String, filename String,
|
|
4
|
-
map String CODEC(ZSTD(3)), uploaded_at DateTime64(3, 'UTC') DEFAULT now64(3)
|
|
52
|
+
map String CODEC(ZSTD(3)), uploaded_at DateTime64(3, 'UTC') DEFAULT now64(3),
|
|
53
|
+
${SOURCE_MAP_DEBUG_COLUMNS.join(", ")},
|
|
54
|
+
${SOURCE_MAP_DEBUG_ID_INDEX}
|
|
5
55
|
) ENGINE = ReplacingMergeTree(uploaded_at)
|
|
6
56
|
ORDER BY (org_id, repository_id, release, filename)
|
|
7
57
|
TTL toDateTime(uploaded_at) + INTERVAL 30 DAY`;
|
|
8
58
|
}
|
|
59
|
+
const RELEASE_RE = /^[a-zA-Z0-9._-]{1,200}$/;
|
|
60
|
+
/** The content hash of a stored map — `autter-runtime sourcemaps upload`
|
|
61
|
+
* computes the same value to skip maps the ingester already has. */
|
|
62
|
+
export function sourceMapContentHash(map) {
|
|
63
|
+
return createHash("sha256").update(map).digest("hex");
|
|
64
|
+
}
|
|
65
|
+
/**
|
|
66
|
+
* Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
|
|
67
|
+
* `map` may be an object or a JSON string. The debug id comes from the body,
|
|
68
|
+
* else from the map's own `debugId` (or legacy `debug_id`); an invalid one
|
|
69
|
+
* rejects the upload. Null when the upload is invalid.
|
|
70
|
+
*/
|
|
9
71
|
export function validateSourceMap(body) {
|
|
10
72
|
if (!body || typeof body !== "object")
|
|
11
73
|
return null;
|
|
12
74
|
const input = body;
|
|
13
|
-
if (typeof input.release !== "string" ||
|
|
75
|
+
if (typeof input.release !== "string" || !RELEASE_RE.test(input.release))
|
|
14
76
|
return null;
|
|
15
|
-
if (typeof input.filename !== "string"
|
|
77
|
+
if (typeof input.filename !== "string")
|
|
16
78
|
return null;
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
filename = new URL(input.filename, "https://autter.invalid").pathname;
|
|
20
|
-
}
|
|
21
|
-
catch {
|
|
22
|
-
return null;
|
|
23
|
-
}
|
|
24
|
-
if (!filename.endsWith(".js"))
|
|
79
|
+
const filename = sourceMapPath(input.filename);
|
|
80
|
+
if (!filename || !BUNDLE_PATH_RE.test(filename))
|
|
25
81
|
return null;
|
|
26
82
|
let parsed;
|
|
27
83
|
try {
|
|
@@ -32,10 +88,74 @@ export function validateSourceMap(body) {
|
|
|
32
88
|
}
|
|
33
89
|
if (!parsed || parsed.version !== 3 || typeof parsed.mappings !== "string" || !Array.isArray(parsed.sources))
|
|
34
90
|
return null;
|
|
35
|
-
|
|
91
|
+
const rawDebugId = input.debugId ?? parsed.debugId ?? parsed.debug_id;
|
|
92
|
+
const debugId = rawDebugId === undefined || rawDebugId === null || rawDebugId === "" ? "" : normalizeDebugId(rawDebugId);
|
|
93
|
+
if (debugId === null)
|
|
94
|
+
return null;
|
|
95
|
+
// Source contents are unnecessary for position lookup and may contain
|
|
96
|
+
// secrets. Everything else is stored as sent, extension fields included:
|
|
97
|
+
// symbolication reads `x_autter_scopes` (function scopes) from it, and the
|
|
98
|
+
// CLI hashes exactly this JSON to skip maps already uploaded.
|
|
36
99
|
const { sourcesContent: _omitted, ...safe } = parsed;
|
|
37
100
|
const map = JSON.stringify(safe);
|
|
38
|
-
if (map.length >
|
|
101
|
+
if (map.length > SOURCE_MAP_MAX_CHARS)
|
|
102
|
+
return null;
|
|
103
|
+
return { release: input.release, filename, map, debugId, contentHash: sourceMapContentHash(map) };
|
|
104
|
+
}
|
|
105
|
+
/** Most entries one check request may name. */
|
|
106
|
+
export const SOURCE_MAP_CHECK_MAX = 500;
|
|
107
|
+
/**
|
|
108
|
+
* Validate a `/v1/sourcemaps/check` request:
|
|
109
|
+
* `{ release, files: [{ filename, debugId?, hash }] }` (≤ 500 files). The
|
|
110
|
+
* filenames and debug ids are normalised exactly as an upload would store
|
|
111
|
+
* them. Null when invalid.
|
|
112
|
+
*/
|
|
113
|
+
export function validateSourceMapCheck(body) {
|
|
114
|
+
if (!body || typeof body !== "object")
|
|
39
115
|
return null;
|
|
40
|
-
|
|
116
|
+
const input = body;
|
|
117
|
+
if (typeof input.release !== "string" || !RELEASE_RE.test(input.release))
|
|
118
|
+
return null;
|
|
119
|
+
if (!Array.isArray(input.files) || input.files.length > SOURCE_MAP_CHECK_MAX)
|
|
120
|
+
return null;
|
|
121
|
+
const entries = [];
|
|
122
|
+
for (const raw of input.files) {
|
|
123
|
+
if (!raw || typeof raw !== "object")
|
|
124
|
+
return null;
|
|
125
|
+
const file = raw;
|
|
126
|
+
if (typeof file.filename !== "string" || typeof file.hash !== "string" || !/^[0-9a-f]{64}$/.test(file.hash))
|
|
127
|
+
return null;
|
|
128
|
+
const filename = sourceMapPath(file.filename);
|
|
129
|
+
if (!filename)
|
|
130
|
+
return null;
|
|
131
|
+
const debugId = file.debugId === undefined || file.debugId === null || file.debugId === "" ? "" : normalizeDebugId(file.debugId);
|
|
132
|
+
if (debugId === null)
|
|
133
|
+
return null;
|
|
134
|
+
entries.push({ filename, debugId, hash: file.hash });
|
|
135
|
+
}
|
|
136
|
+
return { release: input.release, entries };
|
|
137
|
+
}
|
|
138
|
+
/** Most scripts one browser event may name in `debugIds`. */
|
|
139
|
+
export const DEBUG_IDS_MAX = 50;
|
|
140
|
+
/**
|
|
141
|
+
* The `debugIds` of a browser event (runtime-browser 1.6.0+, EI-13): script
|
|
142
|
+
* URL → debug id. Keys lose their query and hash and are scrubbed, invalid ids
|
|
143
|
+
* are dropped one by one, at most 50 are kept. `{ debugIds }` for the occurrence, or `{}`.
|
|
144
|
+
*/
|
|
145
|
+
export function browserDebugIds(raw) {
|
|
146
|
+
if (!raw)
|
|
147
|
+
return {};
|
|
148
|
+
const debugIds = {};
|
|
149
|
+
let count = 0;
|
|
150
|
+
for (const [url, value] of Object.entries(raw)) {
|
|
151
|
+
const id = normalizeDebugId(value);
|
|
152
|
+
// Scrubbed like the stack it was taken from, so keys still match its frames.
|
|
153
|
+
const script = scrubText(url.split(/[?#]/)[0] ?? "");
|
|
154
|
+
if (!id || !script)
|
|
155
|
+
continue;
|
|
156
|
+
debugIds[script] = id;
|
|
157
|
+
if (++count >= DEBUG_IDS_MAX)
|
|
158
|
+
break;
|
|
159
|
+
}
|
|
160
|
+
return count ? { debugIds } : {};
|
|
41
161
|
}
|
package/dist/stack-fixtures.d.ts
CHANGED
|
@@ -12,8 +12,10 @@
|
|
|
12
12
|
* parsing, Go/Rust frames were dropped and these collided into one issue).
|
|
13
13
|
* - `redeploy` — the primary defect after a rebuild that shifted every line
|
|
14
14
|
* number (and, for Go, pointer offsets/addresses/goroutine ids). It must
|
|
15
|
-
* fingerprint IDENTICALLY to `primary
|
|
16
|
-
* re-deploys and repeated ingestion
|
|
15
|
+
* fingerprint IDENTICALLY to `primary` under frames-v3, proving grouping
|
|
16
|
+
* is stable across re-deploys and repeated ingestion — and under the
|
|
17
|
+
* legacy message-v1 too, except where `legacyKeepsLineNumbers` says the
|
|
18
|
+
* frozen legacy normaliser hashes line numbers (Python).
|
|
17
19
|
*/
|
|
18
20
|
export interface StackFixture {
|
|
19
21
|
primary: string;
|
|
@@ -21,5 +23,11 @@ export interface StackFixture {
|
|
|
21
23
|
redeploy: string;
|
|
22
24
|
/** Golden normalised top frames for `primary` (locks the parser output). */
|
|
23
25
|
primaryFrames: string[];
|
|
26
|
+
/**
|
|
27
|
+
* message-v1 keeps this language's line numbers (Python `File "…", line
|
|
28
|
+
* N` lines are hashed verbatim, and must stay so for existing issues), so
|
|
29
|
+
* only frames-v3 survives `redeploy`.
|
|
30
|
+
*/
|
|
31
|
+
legacyKeepsLineNumbers?: boolean;
|
|
24
32
|
}
|
|
25
33
|
export declare const STACK_FIXTURES: Record<string, StackFixture>;
|
package/dist/stack-fixtures.js
CHANGED
|
@@ -12,8 +12,10 @@
|
|
|
12
12
|
* parsing, Go/Rust frames were dropped and these collided into one issue).
|
|
13
13
|
* - `redeploy` — the primary defect after a rebuild that shifted every line
|
|
14
14
|
* number (and, for Go, pointer offsets/addresses/goroutine ids). It must
|
|
15
|
-
* fingerprint IDENTICALLY to `primary
|
|
16
|
-
* re-deploys and repeated ingestion
|
|
15
|
+
* fingerprint IDENTICALLY to `primary` under frames-v3, proving grouping
|
|
16
|
+
* is stable across re-deploys and repeated ingestion — and under the
|
|
17
|
+
* legacy message-v1 too, except where `legacyKeepsLineNumbers` says the
|
|
18
|
+
* frozen legacy normaliser hashes line numbers (Python).
|
|
17
19
|
*/
|
|
18
20
|
// ── Go ──────────────────────────────────────────────────────────────────────
|
|
19
21
|
const GO_PRIMARY = `panic: runtime error: invalid memory address or nil pointer dereference
|
|
@@ -154,6 +156,13 @@ const PYTHON_PRIMARY = `Traceback (most recent call last):
|
|
|
154
156
|
File "/app/orders/service.py", line 88, in process
|
|
155
157
|
raise ValueError(f"order {order_id} is invalid")
|
|
156
158
|
ValueError: order 4821 is invalid`;
|
|
159
|
+
// Same defect after a deploy that shifted both functions down the file.
|
|
160
|
+
const PYTHON_REDEPLOY = `Traceback (most recent call last):
|
|
161
|
+
File "/app/web/handler.py", line 47, in handle
|
|
162
|
+
return self.service.process(order)
|
|
163
|
+
File "/app/orders/service.py", line 95, in process
|
|
164
|
+
raise ValueError(f"order {order_id} is invalid")
|
|
165
|
+
ValueError: order 9113 is invalid`;
|
|
157
166
|
const PYTHON_SIBLING = `Traceback (most recent call last):
|
|
158
167
|
File "/app/web/handler.py", line 42, in handle
|
|
159
168
|
return self.service.process(order)
|
|
@@ -222,10 +231,11 @@ export const STACK_FIXTURES = {
|
|
|
222
231
|
python: {
|
|
223
232
|
primary: PYTHON_PRIMARY,
|
|
224
233
|
sibling: PYTHON_SIBLING,
|
|
225
|
-
redeploy:
|
|
234
|
+
redeploy: PYTHON_REDEPLOY,
|
|
226
235
|
primaryFrames: [
|
|
227
236
|
'File "/app/web/handler.py", line 42, in handle',
|
|
228
237
|
'File "/app/orders/service.py", line 88, in process',
|
|
229
238
|
],
|
|
239
|
+
legacyKeepsLineNumbers: true,
|
|
230
240
|
},
|
|
231
241
|
};
|
|
@@ -0,0 +1,48 @@
|
|
|
1
|
+
import type { StackFrame } from "./types.js";
|
|
2
|
+
/**
|
|
3
|
+
* Raw stack text → structured frames, INNERMOST (crash site) FIRST.
|
|
4
|
+
*
|
|
5
|
+
* The legacy `message-v1` fingerprint hashes stack lines as text, which
|
|
6
|
+
* keeps Python line numbers, takes Python's outermost frames, and counts
|
|
7
|
+
* dependency frames. `frames-v3` (fingerprint-v3.ts) hashes structured
|
|
8
|
+
* frames instead, so every supported language is parsed into the sink's
|
|
9
|
+
* StackFrame shape here, in one consistent orientation:
|
|
10
|
+
*
|
|
11
|
+
* - V8 (Node, Chrome, Edge): `at fn (file:line:col)` / `at file:line:col`
|
|
12
|
+
* - Firefox / Safari: `fn@url:line:col` / `@url:line:col`
|
|
13
|
+
* - Python: `File "…", line N, in fn` — tracebacks list the OUTERMOST frame
|
|
14
|
+
* first, so the frames are reversed; only the last (reported) traceback
|
|
15
|
+
* of a chained exception is used
|
|
16
|
+
* - Go: function line + `\t<file>.go:<line> +0x…`, first goroutine only
|
|
17
|
+
* - JVM: `at pkg.Class.method(File.java:N)`, first exception block only
|
|
18
|
+
* (before `Caused by:`)
|
|
19
|
+
* - .NET: `at Ns.Type.Method(args) in <file>:line N`; for wrapped
|
|
20
|
+
* exceptions only the frames after the last "End of inner exception"
|
|
21
|
+
* - Rust: `N: path::to::fn` + `at <file>:<line>:<col>`
|
|
22
|
+
* - Ruby / PHP best effort, so those stacks do not collapse to the message
|
|
23
|
+
*
|
|
24
|
+
* `inApp` follows the contract §3 rule (isInAppFrame). `resolved` is always
|
|
25
|
+
* false here; symbolicate.ts sets it for frames a source map resolved.
|
|
26
|
+
* Shared parity vectors: stack-parse-vectors.json.
|
|
27
|
+
*/
|
|
28
|
+
/** Frames kept per stack (contract §1: at most 50). */
|
|
29
|
+
export declare const MAX_FRAMES = 50;
|
|
30
|
+
export type FrameLanguage = "javascript" | "python" | "go" | "jvm" | "dotnet" | "rust" | "ruby" | "php";
|
|
31
|
+
/**
|
|
32
|
+
* Path half of §3: dependency directories, Node internals and browser
|
|
33
|
+
* extension scripts are not in-app. Windows separators are normalised so
|
|
34
|
+
* `C:\app\node_modules\x.js` counts too. A frame with no path is in-app.
|
|
35
|
+
*/
|
|
36
|
+
export declare function isInAppPath(path: string | null): boolean;
|
|
37
|
+
/**
|
|
38
|
+
* Contract §3. Source-map ignore lists are applied by symbolicate.ts; the
|
|
39
|
+
* language rules apply only to frames of that language, so a JS object
|
|
40
|
+
* that happens to be called `runtime` or `System` stays in-app.
|
|
41
|
+
*/
|
|
42
|
+
export declare function isInAppFrame(frame: Pick<StackFrame, "function" | "module" | "filename">, language: FrameLanguage | null): boolean;
|
|
43
|
+
/** Strip a query string and/or hash from a path or URL. */
|
|
44
|
+
export declare function stripQueryAndHash(path: string): string;
|
|
45
|
+
/** The language of a raw stack, or null when it has no recognisable frames. */
|
|
46
|
+
export declare function detectFrameLanguage(lines: string[]): FrameLanguage | null;
|
|
47
|
+
/** Parse a raw stack into frames, innermost first (≤ MAX_FRAMES). */
|
|
48
|
+
export declare function parseStackFrames(stack: string | null | undefined): StackFrame[];
|