@autter/otlp-ingester 1.9.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cause-chain.d.ts +32 -0
- package/dist/cause-chain.js +122 -0
- package/dist/clickhouse.js +31 -2
- package/dist/compat-manifest.json +37 -0
- package/dist/fingerprint-v3.d.ts +63 -7
- package/dist/fingerprint-v3.js +96 -7
- package/dist/ingest-drops.d.ts +10 -0
- package/dist/ingest-drops.js +13 -0
- package/dist/latency.d.ts +42 -0
- package/dist/latency.js +80 -0
- package/dist/log-promotion.d.ts +36 -5
- package/dist/log-promotion.js +95 -21
- package/dist/logs.d.ts +3 -3
- package/dist/logs.js +13 -4
- package/dist/migrations.d.ts +3 -2
- package/dist/migrations.js +12 -3
- package/dist/normalize-browser.d.ts +12 -12
- package/dist/normalize-browser.js +17 -3
- package/dist/normalize-otlp.js +39 -5
- package/dist/server.js +58 -20
- package/dist/sink.d.ts +5 -2
- package/dist/sink.js +7 -3
- package/dist/source-maps-lookup.d.ts +14 -1
- package/dist/source-maps-lookup.js +13 -3
- package/dist/source-maps.d.ts +49 -6
- package/dist/source-maps.js +122 -12
- package/dist/symbolicate.d.ts +83 -11
- package/dist/symbolicate.js +183 -31
- package/dist/trace-dedupe.d.ts +1 -1
- package/dist/trace-dedupe.js +3 -1
- package/dist/types.d.ts +26 -11
- package/package.json +1 -1
package/dist/server.js
CHANGED
|
@@ -1,14 +1,15 @@
|
|
|
1
1
|
import express from "express";
|
|
2
2
|
import { KeyResolver, RateLimiter } from "./auth.js";
|
|
3
3
|
import { countTruncations } from "./context.js";
|
|
4
|
-
import { autterKeyHash, cumulativeHttpHistogramPoints, dataPointsInRequest, dropRoute, IngestDropCounter, PARTIAL_MESSAGES, partialSuccess, recordsInBody, spansInRequest, } from "./ingest-drops.js";
|
|
4
|
+
import { autterKeyHash, caughtLimitedRecords, cumulativeHttpHistogramPoints, dataPointsInRequest, dropRoute, IngestDropCounter, PARTIAL_MESSAGES, partialSuccess, recordsInBody, spansInRequest, } from "./ingest-drops.js";
|
|
5
5
|
import { ClickHouseStore } from "./clickhouse.js";
|
|
6
6
|
import { normalizeLatencyHistograms } from "./latency.js";
|
|
7
7
|
import { deriveFields, occurrenceIdFor } from "./fingerprint.js";
|
|
8
|
+
import { causeTexts } from "./cause-chain.js";
|
|
8
9
|
import { occurrenceIdentity } from "./fingerprint-v3.js";
|
|
9
10
|
import { parseStackFrames } from "./stack-frames.js";
|
|
10
11
|
import { SourceMapCache, Symbolicator } from "./symbolicate.js";
|
|
11
|
-
import { logPromotionCandidates, promotionLookups,
|
|
12
|
+
import { logPromotionCandidates, promotionLookups, logRollups, } from "./log-promotion.js";
|
|
12
13
|
import { normalizeBrowserPayload, parseBrowserPayload, } from "./normalize-browser.js";
|
|
13
14
|
import { normalizeMetrics, normalizeTraces, } from "./normalize-otlp.js";
|
|
14
15
|
import { decodeMetricsRequest, decodeTraceRequest, decodeLogsRequest, encodeExportResponse } from "./otlp-proto.js";
|
|
@@ -45,8 +46,9 @@ const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
|
45
46
|
export function createIngesterApp(config) {
|
|
46
47
|
configureRedaction(config);
|
|
47
48
|
const store = new ClickHouseStore(config);
|
|
48
|
-
// Browser JS frames are resolved through uploaded source
|
|
49
|
-
// are fingerprinted (frames-v3), with parsed maps cached
|
|
49
|
+
// Browser, Node and edge JS frames are resolved through uploaded source
|
|
50
|
+
// maps before they are fingerprinted (frames-v3), with parsed maps cached
|
|
51
|
+
// per release+file.
|
|
50
52
|
const symbolicator = new Symbolicator((key) => store.lookupSourceMap(key), new SourceMapCache({ maxBytes: (config.sourceMapCacheMb ?? 64) * 1024 * 1024 }));
|
|
51
53
|
// Fingerprinted occurrences feed the consumer's issue grouping, metric
|
|
52
54
|
// points feed the request/error-rate rollups, LLM calls feed spend
|
|
@@ -392,30 +394,63 @@ export function createIngesterApp(config) {
|
|
|
392
394
|
}
|
|
393
395
|
return ctx;
|
|
394
396
|
}
|
|
395
|
-
/**
|
|
396
|
-
|
|
397
|
-
* view: parsed innermost-first, then — browser only, where bundles are
|
|
398
|
-
* minified — resolved through the release's source maps. Symbolication
|
|
399
|
-
* never fails ingest (it falls back to the parsed frames).
|
|
400
|
-
*/
|
|
401
|
-
async function framesFor(ctx, input) {
|
|
402
|
-
let frames;
|
|
397
|
+
/** A stack's frames, innermost first; none when it does not parse. */
|
|
398
|
+
function parsedFrames(stack) {
|
|
403
399
|
try {
|
|
404
|
-
|
|
400
|
+
return parseStackFrames(stack);
|
|
405
401
|
}
|
|
406
402
|
catch (err) {
|
|
407
403
|
console.warn("stack parse failed — no frames:", err instanceof Error ? err.message : err);
|
|
408
404
|
return [];
|
|
409
405
|
}
|
|
410
|
-
|
|
411
|
-
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Frames resolved through the release's source maps: minified browser
|
|
409
|
+
* bundles, and bundled Node and edge code (RF-05: tsup, esbuild, webpack
|
|
410
|
+
* and Next.js server output) — only JS frames a map was uploaded for
|
|
411
|
+
* change. Never fails ingest (it falls back to the parsed frames).
|
|
412
|
+
*/
|
|
413
|
+
function symbolicated(ctx, input, frames) {
|
|
414
|
+
if (frames.length === 0)
|
|
415
|
+
return Promise.resolve(frames);
|
|
412
416
|
return symbolicator.symbolicate(frames, {
|
|
413
417
|
orgId: ctx.orgId,
|
|
414
418
|
repositoryId: ctx.repositoryId,
|
|
415
419
|
release: input.release,
|
|
416
420
|
debugIds: input.debugIds ?? null,
|
|
421
|
+
server: input.source !== "browser",
|
|
417
422
|
});
|
|
418
423
|
}
|
|
424
|
+
/**
|
|
425
|
+
* Structured frames for the frames-v3 identity and the platform's stack
|
|
426
|
+
* view: parsed innermost-first, then resolved through the release's
|
|
427
|
+
* source maps, with context lines on in-app frames when the map carries
|
|
428
|
+
* their source.
|
|
429
|
+
*/
|
|
430
|
+
function framesFor(ctx, input) {
|
|
431
|
+
return symbolicated(ctx, input, parsedFrames(input.stack));
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* The reported exception's causes, nearest first, with their frames
|
|
435
|
+
* resolved like the reported one's (RF-06: the cause-v1 identity hashes
|
|
436
|
+
* the root cause's innermost in-app frame, which must read the same as
|
|
437
|
+
* when that error is reported unwrapped). Only used for the identity:
|
|
438
|
+
* cause frames are not stored.
|
|
439
|
+
*/
|
|
440
|
+
async function causesFor(ctx, input) {
|
|
441
|
+
let causes;
|
|
442
|
+
try {
|
|
443
|
+
causes = causeTexts(input.stack, input.attributes);
|
|
444
|
+
}
|
|
445
|
+
catch (err) {
|
|
446
|
+
console.warn("cause chain parse failed — no causes:", err instanceof Error ? err.message : err);
|
|
447
|
+
return [];
|
|
448
|
+
}
|
|
449
|
+
return Promise.all(causes.map(async (cause) => ({
|
|
450
|
+
type: cause.type,
|
|
451
|
+
frames: await symbolicated(ctx, input, parsedFrames(cause.stack)),
|
|
452
|
+
})));
|
|
453
|
+
}
|
|
419
454
|
/** Ids are content-derived (occurrenceIdFor), NOT random: an exporter
|
|
420
455
|
* that retries a batch — after a 503 from a partially-failed ClickHouse
|
|
421
456
|
* write, or when only our 2xx got lost — must produce the same ids, so
|
|
@@ -424,8 +459,8 @@ export function createIngesterApp(config) {
|
|
|
424
459
|
*
|
|
425
460
|
* `fingerprint` / `fingerprintScheme` stay the legacy identity
|
|
426
461
|
* (message-v1 or code-v1) byte-for-byte; `fingerprints` carries every
|
|
427
|
-
* candidate (manual-v1, code-v1, frames-v3, message-v1) for
|
|
428
|
-
* to resolve the issue from.
|
|
462
|
+
* candidate (manual-v1, code-v1, frames-v3, message-v1, cause-v1) for
|
|
463
|
+
* the platform to resolve the issue from.
|
|
429
464
|
*
|
|
430
465
|
* `batchIndexes` (traces): each occurrence's position among the batch's
|
|
431
466
|
* signals before the per-trace dedupe, so a merged occurrence keeps the
|
|
@@ -439,8 +474,8 @@ export function createIngesterApp(config) {
|
|
|
439
474
|
// through verbatim. Attributes were already sanitised by the
|
|
440
475
|
// normalisers (sanitizeRuntimeContext).
|
|
441
476
|
const input = scrubOccurrence(raw);
|
|
442
|
-
const frames = await framesFor(ctx, input);
|
|
443
|
-
const identity = occurrenceIdentity(input, frames);
|
|
477
|
+
const [frames, causes] = await Promise.all([framesFor(ctx, input), causesFor(ctx, input)]);
|
|
478
|
+
const identity = occurrenceIdentity(input, frames, causes);
|
|
444
479
|
const { fingerprint, scheme } = identity.legacy;
|
|
445
480
|
// Debug ids only locate source maps; they are not stored or forwarded.
|
|
446
481
|
const { debugIds: _debugIds, ...stored } = input;
|
|
@@ -672,6 +707,7 @@ export function createIngesterApp(config) {
|
|
|
672
707
|
countAccepted(ctx, req, [
|
|
673
708
|
{ reason: "spans_over_limit", records: overLimit },
|
|
674
709
|
{ reason: "attributes_truncated", records: truncated },
|
|
710
|
+
{ reason: "caught_limited", records: caughtLimitedRecords(occurrences.map((o) => o.attributes)) },
|
|
675
711
|
]);
|
|
676
712
|
}));
|
|
677
713
|
app.post("/v1/logs", asyncRoute(async (req, res) => {
|
|
@@ -700,7 +736,7 @@ export function createIngesterApp(config) {
|
|
|
700
736
|
// and those are promoted to occurrences here — see log-promotion.ts.
|
|
701
737
|
const promoted = await promoteLogs(ctx, logs);
|
|
702
738
|
// Rollups count every promoted error, collapsed or suppressed ones too.
|
|
703
|
-
const metricPoints =
|
|
739
|
+
const metricPoints = logRollups(logs, promoted);
|
|
704
740
|
const admitted = admit(ctx, req, promoted);
|
|
705
741
|
try {
|
|
706
742
|
await Promise.all([
|
|
@@ -723,6 +759,8 @@ export function createIngesterApp(config) {
|
|
|
723
759
|
countAccepted(ctx, req, [
|
|
724
760
|
{ reason: "log_records_over_limit", records: stats.overLimit },
|
|
725
761
|
{ reason: "attributes_truncated", records: truncated },
|
|
762
|
+
// Every record, promoted or not (the SDK's own count records too).
|
|
763
|
+
{ reason: "caught_limited", records: caughtLimitedRecords(logs.map((row) => row.attributes)) },
|
|
726
764
|
]);
|
|
727
765
|
}));
|
|
728
766
|
app.post("/v1/metrics", asyncRoute(async (req, res) => {
|
package/dist/sink.d.ts
CHANGED
|
@@ -127,7 +127,10 @@ export declare class SinkForwarder {
|
|
|
127
127
|
* keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
|
|
128
128
|
* record stands for, sent only when above 1 (a loop-collapse sample:
|
|
129
129
|
* 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
|
|
130
|
-
* their event counts.
|
|
131
|
-
*
|
|
130
|
+
* their event counts. Unreleased (RF-06): `fingerprints` may end with a
|
|
131
|
+
* `cause-v1` candidate after the legacy one; consumers that do not know the
|
|
132
|
+
* scheme drop it (the platform validates schemes one by one). RF-05: in-app
|
|
133
|
+
* frames' context lines are up to 5 on each side (3 before). Older consumers
|
|
134
|
+
* ignore unknown keys, so the version stays 1.
|
|
132
135
|
*/
|
|
133
136
|
export declare function sinkOccurrence(o: RuntimeOccurrence): Record<string, unknown>;
|
package/dist/sink.js
CHANGED
|
@@ -120,7 +120,8 @@ export class SinkForwarder {
|
|
|
120
120
|
orgId: ctx.orgId,
|
|
121
121
|
repositoryId: ctx.repositoryId,
|
|
122
122
|
occurrences: occurrences.map(sinkOccurrence),
|
|
123
|
-
|
|
123
|
+
// Latency samples feed runtime_route_latency_1m only (IP-02).
|
|
124
|
+
metrics: metricPoints.map(({ latency: _latency, ...p }) => ({
|
|
124
125
|
...p,
|
|
125
126
|
bucketAt: p.bucketAt.toISOString(),
|
|
126
127
|
})),
|
|
@@ -322,8 +323,11 @@ export class SinkForwarder {
|
|
|
322
323
|
* keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
|
|
323
324
|
* record stands for, sent only when above 1 (a loop-collapse sample:
|
|
324
325
|
* 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
|
|
325
|
-
* their event counts.
|
|
326
|
-
*
|
|
326
|
+
* their event counts. Unreleased (RF-06): `fingerprints` may end with a
|
|
327
|
+
* `cause-v1` candidate after the legacy one; consumers that do not know the
|
|
328
|
+
* scheme drop it (the platform validates schemes one by one). RF-05: in-app
|
|
329
|
+
* frames' context lines are up to 5 on each side (3 before). Older consumers
|
|
330
|
+
* ignore unknown keys, so the version stays 1.
|
|
327
331
|
*/
|
|
328
332
|
export function sinkOccurrence(o) {
|
|
329
333
|
const { traceId, route, method, statusCode, errorCode, why, fix, link, expected, requestId, breadcrumbs, frames, fingerprints, fingerprintRecord,
|
|
@@ -11,8 +11,21 @@ export interface SourceMapKey {
|
|
|
11
11
|
release: string | null;
|
|
12
12
|
filename: string;
|
|
13
13
|
debugId?: string | null;
|
|
14
|
+
/**
|
|
15
|
+
* Server frames (RF-05): `filename` is the path the process ran the
|
|
16
|
+
* bundle from (`/app/dist/server.js`, `/var/task/.next/server/app/page.js`),
|
|
17
|
+
* which the upload can't know. Without a debug id, the release's map
|
|
18
|
+
* whose stored path is the longest suffix of it wins (`/server.js`,
|
|
19
|
+
* `/.next/server/app/page.js`); an exact path is the longest suffix of all.
|
|
20
|
+
*/
|
|
21
|
+
suffix?: boolean;
|
|
14
22
|
}
|
|
15
|
-
/**
|
|
23
|
+
/**
|
|
24
|
+
* The stored map JSON for a minified file: by debug id first when given,
|
|
25
|
+
* else by release + filename (path only, no origin/query) — for a server
|
|
26
|
+
* frame (`suffix`), the release's map stored under the longest suffix of
|
|
27
|
+
* its path. Null when none.
|
|
28
|
+
*/
|
|
16
29
|
export declare function lookupSourceMap(client: ClickHouseClient, database: string, key: SourceMapKey): Promise<string | null>;
|
|
17
30
|
/**
|
|
18
31
|
* Re-uploading a map refreshes its 30-day TTL, so a map only counts as
|
|
@@ -1,16 +1,21 @@
|
|
|
1
1
|
import { normalizeDebugId, sourceMapPath, } from "./source-maps.js";
|
|
2
|
-
async function firstMap(client, database, where, params) {
|
|
2
|
+
async function firstMap(client, database, where, params, order = "uploaded_at DESC") {
|
|
3
3
|
const rows = await client.query({
|
|
4
4
|
query: `SELECT map FROM ${database}.runtime_source_maps
|
|
5
5
|
WHERE org_id = {org:String} AND repository_id = {repo:String} AND ${where}
|
|
6
|
-
ORDER BY
|
|
6
|
+
ORDER BY ${order} LIMIT 1`,
|
|
7
7
|
query_params: params,
|
|
8
8
|
format: "JSONEachRow",
|
|
9
9
|
});
|
|
10
10
|
const [row] = await rows.json();
|
|
11
11
|
return row?.map ?? null;
|
|
12
12
|
}
|
|
13
|
-
/**
|
|
13
|
+
/**
|
|
14
|
+
* The stored map JSON for a minified file: by debug id first when given,
|
|
15
|
+
* else by release + filename (path only, no origin/query) — for a server
|
|
16
|
+
* frame (`suffix`), the release's map stored under the longest suffix of
|
|
17
|
+
* its path. Null when none.
|
|
18
|
+
*/
|
|
14
19
|
export async function lookupSourceMap(client, database, key) {
|
|
15
20
|
const tenant = { org: key.orgId, repo: key.repositoryId };
|
|
16
21
|
// A debug id names the exact bundle bytes, so it matches whatever release
|
|
@@ -25,6 +30,11 @@ export async function lookupSourceMap(client, database, key) {
|
|
|
25
30
|
const filename = sourceMapPath(key.filename);
|
|
26
31
|
if (!key.release || !filename)
|
|
27
32
|
return null;
|
|
33
|
+
if (key.suffix) {
|
|
34
|
+
// Stored paths always start with "/", so a suffix match ends on a
|
|
35
|
+
// path segment boundary. The release scan is a primary-key range.
|
|
36
|
+
return firstMap(client, database, "release = {release:String} AND endsWith({filename:String}, filename)", { ...tenant, release: key.release, filename }, "length(filename) DESC, uploaded_at DESC");
|
|
37
|
+
}
|
|
28
38
|
return firstMap(client, database, "release = {release:String} AND filename = {filename:String}", { ...tenant, release: key.release, filename });
|
|
29
39
|
}
|
|
30
40
|
/**
|
package/dist/source-maps.d.ts
CHANGED
|
@@ -1,9 +1,11 @@
|
|
|
1
1
|
/**
|
|
2
|
-
* Largest stored map
|
|
3
|
-
*
|
|
4
|
-
*
|
|
5
|
-
*
|
|
6
|
-
*
|
|
2
|
+
* Largest stored map, in characters: 32 MiB, room for vendor chunks of
|
|
3
|
+
* about 20 MB of minified code. Storage is a ZSTD-compressed String column,
|
|
4
|
+
* so size only costs disk; a map this big is still symbolicated, without
|
|
5
|
+
* being kept in the parse cache unless SOURCE_MAP_CACHE_MB allows it
|
|
6
|
+
* (symbolicate.ts). Was 5 MiB up to 1.6.0. The map without
|
|
7
|
+
* `sourcesContent` must fit it; the source contents kept (RF-05) fit in
|
|
8
|
+
* what is left of it, counted in UTF-8 bytes (storedSourcesContent).
|
|
7
9
|
*/
|
|
8
10
|
export declare const SOURCE_MAP_MAX_CHARS: number;
|
|
9
11
|
/** `/v1/sourcemaps` body limit, counted after a gzip or deflate body is
|
|
@@ -28,7 +30,7 @@ export interface ValidSourceMap {
|
|
|
28
30
|
release: string;
|
|
29
31
|
/** sourceMapPath() of the uploaded filename. */
|
|
30
32
|
filename: string;
|
|
31
|
-
/** The map JSON
|
|
33
|
+
/** The map JSON as stored: `sourcesContent` bounded by storedSourceMapJson. */
|
|
32
34
|
map: string;
|
|
33
35
|
/** Lowercase debug id, or "" when the upload has none. */
|
|
34
36
|
debugId: string;
|
|
@@ -38,6 +40,30 @@ export interface ValidSourceMap {
|
|
|
38
40
|
/** The content hash of a stored map — `autter-runtime sourcemaps upload`
|
|
39
41
|
* computes the same value to skip maps the ingester already has. */
|
|
40
42
|
export declare function sourceMapContentHash(map: string): string;
|
|
43
|
+
/**
|
|
44
|
+
* The `sourcesContent` a stored map keeps (RF-05), or null to leave the
|
|
45
|
+
* field out. Context lines for in-app frames come from it, so only in-app
|
|
46
|
+
* sources keep theirs: a source the map's `ignoreList` /
|
|
47
|
+
* `x_google_ignoreList` names, or whose path the in-app rule rejects
|
|
48
|
+
* (`node_modules/`, `/.pnpm/`, `/vendor/`, … — isInAppPath), gets `null`,
|
|
49
|
+
* like any entry that is not a string.
|
|
50
|
+
*
|
|
51
|
+
* The map must stay within the stored-map limit, so contents are kept
|
|
52
|
+
* whole, smallest first (ties in source order), while the whole JSON fits
|
|
53
|
+
* `limit` UTF-8 bytes given the `baseBytes` of the map without the field;
|
|
54
|
+
* the rest become `null`. Nothing is ever cut inside a file, so kept line
|
|
55
|
+
* numbers stay right. `@autter/runtime-cli` (upload.ts storedMapJson) does
|
|
56
|
+
* exactly this, so it hashes the map the ingester stores and skips it on
|
|
57
|
+
* the next run.
|
|
58
|
+
*/
|
|
59
|
+
export declare function storedSourcesContent(fields: Record<string, unknown>, baseBytes: number, limit?: number): Array<string | null> | null;
|
|
60
|
+
/**
|
|
61
|
+
* The stored JSON of a map: every field as sent except `sourcesContent`,
|
|
62
|
+
* which goes last, bounded by storedSourcesContent (left out when nothing
|
|
63
|
+
* is kept, so a map without in-app contents is stored byte-for-byte as it
|
|
64
|
+
* was before contents were kept).
|
|
65
|
+
*/
|
|
66
|
+
export declare function storedSourceMapJson(fields: Record<string, unknown>, limit?: number): string;
|
|
41
67
|
/**
|
|
42
68
|
* Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
|
|
43
69
|
* `map` may be an object or a JSON string. The debug id comes from the body,
|
|
@@ -78,3 +104,20 @@ export declare const DEBUG_IDS_MAX = 50;
|
|
|
78
104
|
export declare function browserDebugIds(raw: Record<string, unknown> | undefined): {
|
|
79
105
|
debugIds?: Record<string, string>;
|
|
80
106
|
};
|
|
107
|
+
/**
|
|
108
|
+
* The attribute server SDKs put the debug ids of the bundles in an
|
|
109
|
+
* exception's stack under (`@autter/runtime-core` errorAttributes, RF-05):
|
|
110
|
+
* a JSON object of script path, as it appears in the stack → debug id.
|
|
111
|
+
*/
|
|
112
|
+
export declare const DEBUG_IDS_ATTRIBUTE = "autter.debug_ids";
|
|
113
|
+
/**
|
|
114
|
+
* `{ debugIds }` from a server exception's `autter.debug_ids` (a JSON
|
|
115
|
+
* string, or the object the sanitiser made of it), validated exactly like a
|
|
116
|
+
* browser event's (browserDebugIds); `{}` when absent or malformed. Only
|
|
117
|
+
* locates source maps: like the browser's, never stored or forwarded.
|
|
118
|
+
*/
|
|
119
|
+
export declare function serverDebugIds(raw: unknown): {
|
|
120
|
+
debugIds?: Record<string, string>;
|
|
121
|
+
};
|
|
122
|
+
/** `attributes` without `autter.debug_ids`, which locates maps and is not kept. */
|
|
123
|
+
export declare function withoutDebugIdsAttribute(attributes: Record<string, unknown>): Record<string, unknown>;
|
package/dist/source-maps.js
CHANGED
|
@@ -1,11 +1,14 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { scrubText } from "./redact.js";
|
|
3
|
+
import { isInAppPath } from "./stack-frames.js";
|
|
3
4
|
/**
|
|
4
|
-
* Largest stored map
|
|
5
|
-
*
|
|
6
|
-
*
|
|
7
|
-
*
|
|
8
|
-
*
|
|
5
|
+
* Largest stored map, in characters: 32 MiB, room for vendor chunks of
|
|
6
|
+
* about 20 MB of minified code. Storage is a ZSTD-compressed String column,
|
|
7
|
+
* so size only costs disk; a map this big is still symbolicated, without
|
|
8
|
+
* being kept in the parse cache unless SOURCE_MAP_CACHE_MB allows it
|
|
9
|
+
* (symbolicate.ts). Was 5 MiB up to 1.6.0. The map without
|
|
10
|
+
* `sourcesContent` must fit it; the source contents kept (RF-05) fit in
|
|
11
|
+
* what is left of it, counted in UTF-8 bytes (storedSourcesContent).
|
|
9
12
|
*/
|
|
10
13
|
export const SOURCE_MAP_MAX_CHARS = 32 * 1024 * 1024;
|
|
11
14
|
/** `/v1/sourcemaps` body limit, counted after a gzip or deflate body is
|
|
@@ -62,6 +65,78 @@ const RELEASE_RE = /^[a-zA-Z0-9._-]{1,200}$/;
|
|
|
62
65
|
export function sourceMapContentHash(map) {
|
|
63
66
|
return createHash("sha256").update(map).digest("hex");
|
|
64
67
|
}
|
|
68
|
+
/** UTF-8 bytes `,"sourcesContent":[` and `]` add around the entries. */
|
|
69
|
+
const SOURCES_CONTENT_WRAPPER_BYTES = Buffer.byteLength(',"sourcesContent":[]');
|
|
70
|
+
/** UTF-8 bytes of a `null` entry. */
|
|
71
|
+
const NULL_ENTRY_BYTES = 4;
|
|
72
|
+
/**
|
|
73
|
+
* The `sourcesContent` a stored map keeps (RF-05), or null to leave the
|
|
74
|
+
* field out. Context lines for in-app frames come from it, so only in-app
|
|
75
|
+
* sources keep theirs: a source the map's `ignoreList` /
|
|
76
|
+
* `x_google_ignoreList` names, or whose path the in-app rule rejects
|
|
77
|
+
* (`node_modules/`, `/.pnpm/`, `/vendor/`, … — isInAppPath), gets `null`,
|
|
78
|
+
* like any entry that is not a string.
|
|
79
|
+
*
|
|
80
|
+
* The map must stay within the stored-map limit, so contents are kept
|
|
81
|
+
* whole, smallest first (ties in source order), while the whole JSON fits
|
|
82
|
+
* `limit` UTF-8 bytes given the `baseBytes` of the map without the field;
|
|
83
|
+
* the rest become `null`. Nothing is ever cut inside a file, so kept line
|
|
84
|
+
* numbers stay right. `@autter/runtime-cli` (upload.ts storedMapJson) does
|
|
85
|
+
* exactly this, so it hashes the map the ingester stores and skips it on
|
|
86
|
+
* the next run.
|
|
87
|
+
*/
|
|
88
|
+
export function storedSourcesContent(fields, baseBytes, limit = SOURCE_MAP_MAX_CHARS) {
|
|
89
|
+
const { sources, sourcesContent } = fields;
|
|
90
|
+
if (!Array.isArray(sources) || !Array.isArray(sourcesContent) || sources.length === 0)
|
|
91
|
+
return null;
|
|
92
|
+
const ignored = new Set();
|
|
93
|
+
for (const list of [fields.ignoreList, fields.x_google_ignoreList]) {
|
|
94
|
+
if (!Array.isArray(list))
|
|
95
|
+
continue;
|
|
96
|
+
for (const index of list)
|
|
97
|
+
if (Number.isInteger(index))
|
|
98
|
+
ignored.add(index);
|
|
99
|
+
}
|
|
100
|
+
const count = sources.length;
|
|
101
|
+
const candidates = [];
|
|
102
|
+
for (let index = 0; index < count; index++) {
|
|
103
|
+
const source = sources[index];
|
|
104
|
+
const content = sourcesContent[index];
|
|
105
|
+
if (typeof source !== "string" || typeof content !== "string")
|
|
106
|
+
continue;
|
|
107
|
+
if (ignored.has(index) || !isInAppPath(source))
|
|
108
|
+
continue;
|
|
109
|
+
candidates.push({ index, content, bytes: Buffer.byteLength(JSON.stringify(content)) });
|
|
110
|
+
}
|
|
111
|
+
if (candidates.length === 0)
|
|
112
|
+
return null;
|
|
113
|
+
candidates.sort((a, b) => a.bytes - b.bytes || a.index - b.index);
|
|
114
|
+
// Every entry starts as `null`, with a comma between entries.
|
|
115
|
+
let remaining = limit - baseBytes - SOURCES_CONTENT_WRAPPER_BYTES - NULL_ENTRY_BYTES * count - (count - 1);
|
|
116
|
+
const kept = new Array(count).fill(null);
|
|
117
|
+
let any = false;
|
|
118
|
+
for (const candidate of candidates) {
|
|
119
|
+
const extra = candidate.bytes - NULL_ENTRY_BYTES;
|
|
120
|
+
if (extra > remaining)
|
|
121
|
+
break;
|
|
122
|
+
remaining -= extra;
|
|
123
|
+
kept[candidate.index] = candidate.content;
|
|
124
|
+
any = true;
|
|
125
|
+
}
|
|
126
|
+
return any ? kept : null;
|
|
127
|
+
}
|
|
128
|
+
/**
|
|
129
|
+
* The stored JSON of a map: every field as sent except `sourcesContent`,
|
|
130
|
+
* which goes last, bounded by storedSourcesContent (left out when nothing
|
|
131
|
+
* is kept, so a map without in-app contents is stored byte-for-byte as it
|
|
132
|
+
* was before contents were kept).
|
|
133
|
+
*/
|
|
134
|
+
export function storedSourceMapJson(fields, limit = SOURCE_MAP_MAX_CHARS) {
|
|
135
|
+
const { sourcesContent: _content, ...rest } = fields;
|
|
136
|
+
const base = JSON.stringify(rest);
|
|
137
|
+
const content = storedSourcesContent(fields, Buffer.byteLength(base), limit);
|
|
138
|
+
return content ? JSON.stringify({ ...rest, sourcesContent: content }) : base;
|
|
139
|
+
}
|
|
65
140
|
/**
|
|
66
141
|
* Validate one `/v1/sourcemaps` upload: `{ release, filename, map, debugId? }`.
|
|
67
142
|
* `map` may be an object or a JSON string. The debug id comes from the body,
|
|
@@ -92,14 +167,16 @@ export function validateSourceMap(body) {
|
|
|
92
167
|
const debugId = rawDebugId === undefined || rawDebugId === null || rawDebugId === "" ? "" : normalizeDebugId(rawDebugId);
|
|
93
168
|
if (debugId === null)
|
|
94
169
|
return null;
|
|
95
|
-
//
|
|
96
|
-
//
|
|
97
|
-
//
|
|
98
|
-
//
|
|
99
|
-
|
|
100
|
-
|
|
101
|
-
|
|
170
|
+
// Everything is stored as sent, extension fields included: symbolication
|
|
171
|
+
// reads `x_autter_scopes` (function scopes) from it, and the CLI hashes
|
|
172
|
+
// exactly this JSON to skip maps already uploaded. `sourcesContent` is
|
|
173
|
+
// kept for in-app sources (RF-05: context lines around in-app frames),
|
|
174
|
+
// within the stored-map limit; the lines taken from it are scrubbed when
|
|
175
|
+
// they are attached to an occurrence (symbolicate.ts).
|
|
176
|
+
const { sourcesContent: _content, ...withoutContent } = parsed;
|
|
177
|
+
if (JSON.stringify(withoutContent).length > SOURCE_MAP_MAX_CHARS)
|
|
102
178
|
return null;
|
|
179
|
+
const map = storedSourceMapJson(parsed);
|
|
103
180
|
return { release: input.release, filename, map, debugId, contentHash: sourceMapContentHash(map) };
|
|
104
181
|
}
|
|
105
182
|
/** Most entries one check request may name. */
|
|
@@ -167,3 +244,36 @@ export function browserDebugIds(raw) {
|
|
|
167
244
|
}
|
|
168
245
|
return count ? { debugIds } : {};
|
|
169
246
|
}
|
|
247
|
+
/**
|
|
248
|
+
* The attribute server SDKs put the debug ids of the bundles in an
|
|
249
|
+
* exception's stack under (`@autter/runtime-core` errorAttributes, RF-05):
|
|
250
|
+
* a JSON object of script path, as it appears in the stack → debug id.
|
|
251
|
+
*/
|
|
252
|
+
export const DEBUG_IDS_ATTRIBUTE = "autter.debug_ids";
|
|
253
|
+
/**
|
|
254
|
+
* `{ debugIds }` from a server exception's `autter.debug_ids` (a JSON
|
|
255
|
+
* string, or the object the sanitiser made of it), validated exactly like a
|
|
256
|
+
* browser event's (browserDebugIds); `{}` when absent or malformed. Only
|
|
257
|
+
* locates source maps: like the browser's, never stored or forwarded.
|
|
258
|
+
*/
|
|
259
|
+
export function serverDebugIds(raw) {
|
|
260
|
+
let value = raw;
|
|
261
|
+
if (typeof value === "string") {
|
|
262
|
+
try {
|
|
263
|
+
value = JSON.parse(value);
|
|
264
|
+
}
|
|
265
|
+
catch {
|
|
266
|
+
return {};
|
|
267
|
+
}
|
|
268
|
+
}
|
|
269
|
+
if (!value || typeof value !== "object" || Array.isArray(value))
|
|
270
|
+
return {};
|
|
271
|
+
return browserDebugIds(value);
|
|
272
|
+
}
|
|
273
|
+
/** `attributes` without `autter.debug_ids`, which locates maps and is not kept. */
|
|
274
|
+
export function withoutDebugIdsAttribute(attributes) {
|
|
275
|
+
if (!(DEBUG_IDS_ATTRIBUTE in attributes))
|
|
276
|
+
return attributes;
|
|
277
|
+
const { [DEBUG_IDS_ATTRIBUTE]: _ids, ...rest } = attributes;
|
|
278
|
+
return rest;
|
|
279
|
+
}
|