@autter/otlp-ingester 1.10.0 → 1.11.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +1 -1
- package/dist/cause-chain.d.ts +32 -0
- package/dist/cause-chain.js +122 -0
- package/dist/clickhouse.js +31 -2
- package/dist/compat-manifest.json +29 -0
- package/dist/fingerprint-v3.d.ts +63 -7
- package/dist/fingerprint-v3.js +96 -7
- package/dist/ingest-drops.d.ts +10 -0
- package/dist/ingest-drops.js +13 -0
- package/dist/latency.d.ts +42 -0
- package/dist/latency.js +80 -0
- package/dist/log-promotion.js +20 -1
- package/dist/logs.d.ts +3 -3
- package/dist/logs.js +13 -4
- package/dist/migrations.d.ts +3 -2
- package/dist/migrations.js +12 -3
- package/dist/normalize-browser.d.ts +12 -12
- package/dist/normalize-browser.js +17 -3
- package/dist/normalize-otlp.js +39 -5
- package/dist/server.js +56 -18
- package/dist/sink.d.ts +5 -2
- package/dist/sink.js +7 -3
- package/dist/source-maps-lookup.d.ts +14 -1
- package/dist/source-maps-lookup.js +13 -3
- package/dist/source-maps.d.ts +49 -6
- package/dist/source-maps.js +122 -12
- package/dist/symbolicate.d.ts +83 -11
- package/dist/symbolicate.js +183 -31
- package/dist/trace-dedupe.d.ts +1 -1
- package/dist/trace-dedupe.js +3 -1
- package/dist/types.d.ts +26 -11
- package/package.json +1 -1
package/dist/latency.js
CHANGED
|
@@ -1,5 +1,85 @@
|
|
|
1
1
|
import { createHash } from "node:crypto";
|
|
2
2
|
import { normalizeRoute } from "./fingerprint.js";
|
|
3
|
+
/** Points one histogram bucket is spread over (uniform within the bucket). */
|
|
4
|
+
const BUCKET_SPREAD = 8;
|
|
5
|
+
/**
|
|
6
|
+
* Samples for a delta histogram: each bucket's count spread evenly over
|
|
7
|
+
* [previous bound, bound] (at most 8 points), the overflow bucket at the
|
|
8
|
+
* highest bound. Undefined for a histogram without usable buckets.
|
|
9
|
+
*/
|
|
10
|
+
export function histogramLatency(boundsMs, counts) {
|
|
11
|
+
if (boundsMs.length === 0 || boundsMs.length > 256 || counts.length !== boundsMs.length + 1)
|
|
12
|
+
return undefined;
|
|
13
|
+
if (boundsMs.some((bound, i) => !Number.isFinite(bound) || bound < 0 || (i > 0 && bound <= boundsMs[i - 1])))
|
|
14
|
+
return undefined;
|
|
15
|
+
if (counts.some((count) => !Number.isSafeInteger(count) || count < 0))
|
|
16
|
+
return undefined;
|
|
17
|
+
const valuesMs = [];
|
|
18
|
+
const weights = [];
|
|
19
|
+
counts.forEach((count, i) => {
|
|
20
|
+
if (count === 0)
|
|
21
|
+
return;
|
|
22
|
+
const lower = i === 0 ? 0 : boundsMs[i - 1];
|
|
23
|
+
if (i === boundsMs.length) {
|
|
24
|
+
valuesMs.push(lower);
|
|
25
|
+
weights.push(count);
|
|
26
|
+
return;
|
|
27
|
+
}
|
|
28
|
+
const upper = boundsMs[i];
|
|
29
|
+
const points = Math.min(count, BUCKET_SPREAD);
|
|
30
|
+
const base = Math.floor(count / points);
|
|
31
|
+
for (let j = 0; j < points; j++) {
|
|
32
|
+
valuesMs.push(lower + ((upper - lower) * (j + 0.5)) / points);
|
|
33
|
+
weights.push(base + (j < count - base * points ? 1 : 0));
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
return valuesMs.length ? { valuesMs, weights } : undefined;
|
|
37
|
+
}
|
|
38
|
+
/** One request's duration as samples. */
|
|
39
|
+
export function requestLatency(durationMs) {
|
|
40
|
+
return Number.isFinite(durationMs) && durationMs >= 0 ? { valuesMs: [durationMs], weights: [1] } : undefined;
|
|
41
|
+
}
|
|
42
|
+
/** Concatenate samples of points folded into one rollup row. */
|
|
43
|
+
export function mergeLatency(a, b) {
|
|
44
|
+
if (!a)
|
|
45
|
+
return b && { valuesMs: [...b.valuesMs], weights: [...b.weights] };
|
|
46
|
+
if (b) {
|
|
47
|
+
a.valuesMs.push(...b.valuesMs);
|
|
48
|
+
a.weights.push(...b.weights);
|
|
49
|
+
}
|
|
50
|
+
return a;
|
|
51
|
+
}
|
|
52
|
+
/** Null input table: rows pass to the view, nothing is stored. */
|
|
53
|
+
export function routeLatencyInputDDL(database) {
|
|
54
|
+
return `CREATE TABLE IF NOT EXISTS ${database}.runtime_route_latency_input (
|
|
55
|
+
org_id String, repository_id String, service String, environment String,
|
|
56
|
+
release String, route String, bucket_at DateTime('UTC'),
|
|
57
|
+
values_ms Array(Float64), weights Array(UInt64)
|
|
58
|
+
) ENGINE = Null`;
|
|
59
|
+
}
|
|
60
|
+
export function routeLatencyTableDDL(database, ttlDays = 90) {
|
|
61
|
+
return `CREATE TABLE IF NOT EXISTS ${database}.runtime_route_latency_1m (
|
|
62
|
+
org_id String, repository_id String,
|
|
63
|
+
service LowCardinality(String), environment LowCardinality(String),
|
|
64
|
+
release String DEFAULT '', route String DEFAULT '', bucket_at DateTime('UTC'),
|
|
65
|
+
request_count SimpleAggregateFunction(sum, UInt64),
|
|
66
|
+
duration_quantiles AggregateFunction(quantilesTDigestWeighted(0.5, 0.95, 0.99), Float64, UInt64)
|
|
67
|
+
) ENGINE = AggregatingMergeTree PARTITION BY toYYYYMM(bucket_at)
|
|
68
|
+
ORDER BY (org_id, repository_id, service, environment, release, route, bucket_at)
|
|
69
|
+
TTL bucket_at + INTERVAL ${Math.max(1, Math.round(ttlDays))} DAY`;
|
|
70
|
+
}
|
|
71
|
+
/** Needs both tables above; reads nothing older than itself. */
|
|
72
|
+
export function routeLatencyViewDDL(database) {
|
|
73
|
+
return `CREATE MATERIALIZED VIEW IF NOT EXISTS ${database}.runtime_route_latency_1m_mv
|
|
74
|
+
TO ${database}.runtime_route_latency_1m AS
|
|
75
|
+
SELECT
|
|
76
|
+
org_id, repository_id, service, environment, release, route, bucket_at,
|
|
77
|
+
sum(weight) AS request_count,
|
|
78
|
+
quantilesTDigestWeightedState(0.5, 0.95, 0.99)(value_ms, weight) AS duration_quantiles
|
|
79
|
+
FROM ${database}.runtime_route_latency_input
|
|
80
|
+
ARRAY JOIN values_ms AS value_ms, weights AS weight
|
|
81
|
+
GROUP BY org_id, repository_id, service, environment, release, route, bucket_at`;
|
|
82
|
+
}
|
|
3
83
|
export function latencyTableDDL(database, ttlDays = 30) {
|
|
4
84
|
return `CREATE TABLE IF NOT EXISTS ${database}.runtime_latency_histograms (
|
|
5
85
|
org_id String, repository_id String, point_id String,
|
package/dist/log-promotion.js
CHANGED
|
@@ -1,4 +1,5 @@
|
|
|
1
1
|
import { normalizeRoute } from "./fingerprint.js";
|
|
2
|
+
import { mergeLatency, requestLatency } from "./latency.js";
|
|
2
3
|
/**
|
|
3
4
|
* Log promotion — errors from SDKs running WITHOUT a trace pipeline.
|
|
4
5
|
*
|
|
@@ -96,6 +97,10 @@ export const COUNTED_BY_ATTRIBUTE = "autter.request.counted_by";
|
|
|
96
97
|
/** On a counted summary: an error of the request was reported (an
|
|
97
98
|
* `autter.capture.mode = "log"` record), which makes it a failed request. */
|
|
98
99
|
export const ERROR_REPORTED_ATTRIBUTE = "autter.request.error_reported";
|
|
100
|
+
function withLatency(durationMs) {
|
|
101
|
+
const latency = requestLatency(durationMs);
|
|
102
|
+
return latency ? { latency } : {};
|
|
103
|
+
}
|
|
99
104
|
export function countedBySummary(attributes) {
|
|
100
105
|
return attributes?.[COUNTED_BY_ATTRIBUTE] === "summary";
|
|
101
106
|
}
|
|
@@ -130,9 +135,14 @@ export function logRollups(rows, promoted) {
|
|
|
130
135
|
existing.requestCount += rest.requestCount;
|
|
131
136
|
existing.errorCount += rest.errorCount;
|
|
132
137
|
existing.durationSumMs += rest.durationSumMs;
|
|
138
|
+
const latency = mergeLatency(existing.latency, rest.latency);
|
|
139
|
+
if (latency)
|
|
140
|
+
existing.latency = latency;
|
|
133
141
|
return;
|
|
134
142
|
}
|
|
135
|
-
|
|
143
|
+
const { latency, ...counts } = rest;
|
|
144
|
+
const copy = mergeLatency(undefined, latency);
|
|
145
|
+
rollups.set(key, { ...counts, bucketAt, sessionCount: 0, ...(copy ? { latency: copy } : {}) });
|
|
136
146
|
};
|
|
137
147
|
for (const row of rows) {
|
|
138
148
|
if (row.kind !== "request" || !countedBySummary(row.attributes))
|
|
@@ -150,9 +160,18 @@ export function logRollups(rows, promoted) {
|
|
|
150
160
|
requestCount: 1,
|
|
151
161
|
errorCount: failed ? 1 : 0,
|
|
152
162
|
durationSumMs: row.durationMs,
|
|
163
|
+
...withLatency(row.durationMs),
|
|
153
164
|
});
|
|
154
165
|
}
|
|
155
166
|
for (const occ of promoted) {
|
|
167
|
+
// A caught error or deliberate fallback is an occurrence, never a
|
|
168
|
+
// counted failure. (A deliberate fallback is a warning and is already
|
|
169
|
+
// excluded above; a caught error is kept out here.)
|
|
170
|
+
if (occ.severity !== "error" && occ.severity !== "fatal")
|
|
171
|
+
continue;
|
|
172
|
+
const handling = occ.attributes?.["autter.handling"];
|
|
173
|
+
if (handling === "caught" || handling === "deliberate")
|
|
174
|
+
continue;
|
|
156
175
|
// Its request's summary counts this failure.
|
|
157
176
|
if (countedBySummary(occ.attributes))
|
|
158
177
|
continue;
|
package/dist/logs.d.ts
CHANGED
|
@@ -33,9 +33,9 @@ export interface RuntimeLogRecord {
|
|
|
33
33
|
aiCalls: number;
|
|
34
34
|
/**
|
|
35
35
|
* Not persisted on runtime_logs. Set for logger-only/edge error records
|
|
36
|
-
* (`autter.capture.mode = "log"`, severity ≥ error,
|
|
37
|
-
*
|
|
38
|
-
* see log-promotion.ts.
|
|
36
|
+
* (`autter.capture.mode = "log"`, severity ≥ error, or a handled warning
|
|
37
|
+
* such as `runtimeContext.degraded`, with `exception.*` present): the
|
|
38
|
+
* occurrence /v1/logs promotes into issue grouping — see log-promotion.ts.
|
|
39
39
|
*/
|
|
40
40
|
occurrence?: RuntimeOccurrenceInput;
|
|
41
41
|
}
|
package/dist/logs.js
CHANGED
|
@@ -3,6 +3,7 @@ import { decodeOtlpAttributes, sanitizeRuntimeContext, } from "./context.js";
|
|
|
3
3
|
import { declaredLink, liftErrorFields } from "./error-fields.js";
|
|
4
4
|
import { fingerprintOverrideOf } from "./fingerprint-override.js";
|
|
5
5
|
import { normalizeRoute } from "./fingerprint.js";
|
|
6
|
+
import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js";
|
|
6
7
|
/**
|
|
7
8
|
* Log records one request may carry. From 1.8.0 the ones past it are
|
|
8
9
|
* rejected on their own (counted in `stats.overLimit`, answered in the OTLP
|
|
@@ -52,11 +53,15 @@ export function normalizeLogs(request, stats) {
|
|
|
52
53
|
!Array.isArray(decodedBody))
|
|
53
54
|
event = sanitizeRuntimeContext(decodedBody);
|
|
54
55
|
const metadata = Object.fromEntries(Object.entries(attrs).filter(([key]) => /^autter\.(?:operation|event)\./.test(key)));
|
|
55
|
-
|
|
56
|
+
// Server SDKs (runtime-core errorAttributes): the debug ids of the bundles in the
|
|
57
|
+
// exception's stack (RF-05) locate their source maps; read raw
|
|
58
|
+
// (the sanitiser cuts long keys) and not kept.
|
|
59
|
+
const { [DEBUG_IDS_ATTRIBUTE]: rawDebugIds, ...plainAttrs } = attrs;
|
|
60
|
+
const attributes = withoutDebugIdsAttribute(sanitizeRuntimeContext({
|
|
56
61
|
...metadata,
|
|
57
62
|
...event,
|
|
58
|
-
...
|
|
59
|
-
});
|
|
63
|
+
...plainAttrs,
|
|
64
|
+
}));
|
|
60
65
|
const nanos = record.timeUnixNano ?? record.observedTimeUnixNano;
|
|
61
66
|
const occurredAt = nanos === undefined
|
|
62
67
|
? new Date()
|
|
@@ -113,8 +118,11 @@ export function normalizeLogs(request, stats) {
|
|
|
113
118
|
const aiRollup = ai && typeof ai === "object" && !Array.isArray(ai)
|
|
114
119
|
? ai
|
|
115
120
|
: {};
|
|
121
|
+
// Warnings too when handled: `runtimeContext.degraded` (a
|
|
122
|
+
// deliberate fallback) is recorded at warning severity.
|
|
116
123
|
const occurrence = attributes["autter.capture.mode"] === "log" &&
|
|
117
|
-
(severity === "error" || severity === "fatal"
|
|
124
|
+
(severity === "error" || severity === "fatal" ||
|
|
125
|
+
(severity === "warning" && attributes["autter.handled"] === true)) &&
|
|
118
126
|
hasException(attributes)
|
|
119
127
|
? {
|
|
120
128
|
source: "server",
|
|
@@ -138,6 +146,7 @@ export function normalizeLogs(request, stats) {
|
|
|
138
146
|
occurredAt,
|
|
139
147
|
...declared,
|
|
140
148
|
...fingerprintOverrideOf(attributes["autter.fingerprint"]),
|
|
149
|
+
...serverDebugIds(rawDebugIds),
|
|
141
150
|
}
|
|
142
151
|
: undefined;
|
|
143
152
|
rows.push({
|
package/dist/migrations.d.ts
CHANGED
|
@@ -39,8 +39,9 @@ export declare const OCCURRENCE_REQUEST_ID_INDEX = "INDEX idx_occ_request_id req
|
|
|
39
39
|
* Migration 0016 columns — shared with the baseline DDL in clickhouse.ts.
|
|
40
40
|
* `frames`: the structured (symbolicated) frames as JSON, '' when none.
|
|
41
41
|
* `fingerprints`: every identity candidate (manual-v1, code-v1, frames-v3,
|
|
42
|
-
* message-v1 — highest precedence first, ending with the legacy one
|
|
43
|
-
* readers find an issue keyed by ANY
|
|
42
|
+
* message-v1 — highest precedence first, ending with the legacy one — then
|
|
43
|
+
* cause-v1 when computed, RF-06), so readers find an issue keyed by ANY
|
|
44
|
+
* scheme with
|
|
44
45
|
* `(fingerprint = X OR has(fingerprints, X))`; rows written before 0016
|
|
45
46
|
* read `[]` and still match on the legacy `fingerprint`.
|
|
46
47
|
*/
|
package/dist/migrations.js
CHANGED
|
@@ -32,7 +32,7 @@
|
|
|
32
32
|
* ],
|
|
33
33
|
* },
|
|
34
34
|
*/
|
|
35
|
-
import { latencyTableDDL } from "./latency.js";
|
|
35
|
+
import { latencyTableDDL, routeLatencyInputDDL, routeLatencyTableDDL, routeLatencyViewDDL, } from "./latency.js";
|
|
36
36
|
import { profileTableDDL } from "./profiles.js";
|
|
37
37
|
import { SOURCE_MAP_DEBUG_COLUMNS, SOURCE_MAP_DEBUG_ID_INDEX, sourceMapTableDDL, } from "./source-maps.js";
|
|
38
38
|
import { LOG_REQUEST_COLUMNS, LOG_REQUEST_ID_INDEX, logTableDDL, requestRollupTableDDL, requestRollupViewDDL, } from "./logs.js";
|
|
@@ -54,8 +54,9 @@ export const OCCURRENCE_REQUEST_ID_INDEX = "INDEX idx_occ_request_id request_id
|
|
|
54
54
|
* Migration 0016 columns — shared with the baseline DDL in clickhouse.ts.
|
|
55
55
|
* `frames`: the structured (symbolicated) frames as JSON, '' when none.
|
|
56
56
|
* `fingerprints`: every identity candidate (manual-v1, code-v1, frames-v3,
|
|
57
|
-
* message-v1 — highest precedence first, ending with the legacy one
|
|
58
|
-
* readers find an issue keyed by ANY
|
|
57
|
+
* message-v1 — highest precedence first, ending with the legacy one — then
|
|
58
|
+
* cause-v1 when computed, RF-06), so readers find an issue keyed by ANY
|
|
59
|
+
* scheme with
|
|
59
60
|
* `(fingerprint = X OR has(fingerprints, X))`; rows written before 0016
|
|
60
61
|
* read `[]` and still match on the legacy `fingerprint`.
|
|
61
62
|
*/
|
|
@@ -247,6 +248,14 @@ export const MIGRATIONS = [
|
|
|
247
248
|
id: "0021-occurrence-collapsed-count",
|
|
248
249
|
statements: OCCURRENCE_COLLAPSE_COLUMNS.map((column) => `ALTER TABLE {db}.runtime_error_occurrences ADD COLUMN IF NOT EXISTS ${column}`),
|
|
249
250
|
},
|
|
251
|
+
// Per-route p50/p95/p99 in the 1-minute rollups (IP-02, latency.ts): a
|
|
252
|
+
// Null input table the ingester writes latency samples to, the
|
|
253
|
+
// AggregatingMergeTree keyed like runtime_metrics_1m, and the view between
|
|
254
|
+
// them. Additive: runtime_metrics_1m and its readers are unchanged.
|
|
255
|
+
{
|
|
256
|
+
id: "0022-route-latency-1m",
|
|
257
|
+
statements: [routeLatencyInputDDL("{db}"), routeLatencyTableDDL("{db}"), routeLatencyViewDDL("{db}")],
|
|
258
|
+
},
|
|
250
259
|
];
|
|
251
260
|
/** The tracking table itself — created by the runner before anything else. */
|
|
252
261
|
export function migrationsTableDDL(db) {
|
|
@@ -53,38 +53,38 @@ export declare const browserPayloadSchema: z.ZodObject<{
|
|
|
53
53
|
timestamp: string;
|
|
54
54
|
breadcrumbs?: unknown[] | undefined;
|
|
55
55
|
route?: string | undefined;
|
|
56
|
-
errorType?: string | undefined;
|
|
57
|
-
id?: string | undefined;
|
|
58
56
|
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
57
|
+
errorType?: string | undefined;
|
|
59
58
|
stack?: string | undefined;
|
|
60
59
|
traceId?: string | undefined;
|
|
61
60
|
debugIds?: Record<string, unknown> | undefined;
|
|
62
61
|
durationMs?: number | undefined;
|
|
63
62
|
filename?: string | undefined;
|
|
63
|
+
id?: string | undefined;
|
|
64
64
|
name?: string | undefined;
|
|
65
65
|
column?: number | undefined;
|
|
66
66
|
line?: number | undefined;
|
|
67
|
-
context?: Record<string, unknown> | undefined;
|
|
68
67
|
fingerprint?: string | string[] | undefined;
|
|
68
|
+
context?: Record<string, unknown> | undefined;
|
|
69
69
|
}, {
|
|
70
70
|
type: "request_failure" | "timing" | "outcome" | "csp_violation" | "message" | "exception" | "unhandled_rejection" | "session_start" | "track_event";
|
|
71
71
|
timestamp: string;
|
|
72
72
|
breadcrumbs?: unknown;
|
|
73
73
|
route?: string | undefined;
|
|
74
74
|
message?: string | undefined;
|
|
75
|
-
errorType?: string | undefined;
|
|
76
|
-
id?: string | undefined;
|
|
77
75
|
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
76
|
+
errorType?: string | undefined;
|
|
78
77
|
stack?: string | undefined;
|
|
79
78
|
traceId?: unknown;
|
|
80
79
|
debugIds?: unknown;
|
|
81
80
|
durationMs?: number | undefined;
|
|
82
81
|
filename?: string | undefined;
|
|
82
|
+
id?: string | undefined;
|
|
83
83
|
name?: string | undefined;
|
|
84
84
|
column?: number | undefined;
|
|
85
85
|
line?: number | undefined;
|
|
86
|
-
context?: Record<string, unknown> | undefined;
|
|
87
86
|
fingerprint?: unknown;
|
|
87
|
+
context?: Record<string, unknown> | undefined;
|
|
88
88
|
}>, "many">;
|
|
89
89
|
version: z.ZodLiteral<1>;
|
|
90
90
|
sessionId: z.ZodOptional<z.ZodString>;
|
|
@@ -103,19 +103,19 @@ export declare const browserPayloadSchema: z.ZodObject<{
|
|
|
103
103
|
timestamp: string;
|
|
104
104
|
breadcrumbs?: unknown[] | undefined;
|
|
105
105
|
route?: string | undefined;
|
|
106
|
-
errorType?: string | undefined;
|
|
107
|
-
id?: string | undefined;
|
|
108
106
|
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
107
|
+
errorType?: string | undefined;
|
|
109
108
|
stack?: string | undefined;
|
|
110
109
|
traceId?: string | undefined;
|
|
111
110
|
debugIds?: Record<string, unknown> | undefined;
|
|
112
111
|
durationMs?: number | undefined;
|
|
113
112
|
filename?: string | undefined;
|
|
113
|
+
id?: string | undefined;
|
|
114
114
|
name?: string | undefined;
|
|
115
115
|
column?: number | undefined;
|
|
116
116
|
line?: number | undefined;
|
|
117
|
-
context?: Record<string, unknown> | undefined;
|
|
118
117
|
fingerprint?: string | string[] | undefined;
|
|
118
|
+
context?: Record<string, unknown> | undefined;
|
|
119
119
|
}[];
|
|
120
120
|
release?: string | undefined;
|
|
121
121
|
sessionId?: string | undefined;
|
|
@@ -130,19 +130,19 @@ export declare const browserPayloadSchema: z.ZodObject<{
|
|
|
130
130
|
breadcrumbs?: unknown;
|
|
131
131
|
route?: string | undefined;
|
|
132
132
|
message?: string | undefined;
|
|
133
|
-
errorType?: string | undefined;
|
|
134
|
-
id?: string | undefined;
|
|
135
133
|
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
134
|
+
errorType?: string | undefined;
|
|
136
135
|
stack?: string | undefined;
|
|
137
136
|
traceId?: unknown;
|
|
138
137
|
debugIds?: unknown;
|
|
139
138
|
durationMs?: number | undefined;
|
|
140
139
|
filename?: string | undefined;
|
|
140
|
+
id?: string | undefined;
|
|
141
141
|
name?: string | undefined;
|
|
142
142
|
column?: number | undefined;
|
|
143
143
|
line?: number | undefined;
|
|
144
|
-
context?: Record<string, unknown> | undefined;
|
|
145
144
|
fingerprint?: unknown;
|
|
145
|
+
context?: Record<string, unknown> | undefined;
|
|
146
146
|
}[];
|
|
147
147
|
release?: string | undefined;
|
|
148
148
|
sessionId?: string | undefined;
|
|
@@ -201,11 +201,17 @@ export function normalizeBrowserPayload(payload) {
|
|
|
201
201
|
continue;
|
|
202
202
|
}
|
|
203
203
|
const severity = asSeverity(event.severity, TYPE_TO_SEVERITY[event.type] ?? "error");
|
|
204
|
+
const context = event.context ? scrubContext(event.context) : undefined;
|
|
205
|
+
// runtime-browser 1.8.0 sends an Error passed to `console.error` with
|
|
206
|
+
// the log line as `autter.log.message` only (its bundle is size-capped):
|
|
207
|
+
// mark it the way the server SDKs mark a caught error.
|
|
208
|
+
const logged = event.type === "exception" && typeof context?.["autter.log.message"] === "string";
|
|
204
209
|
// A browser error belongs to no server request: it counts in
|
|
205
210
|
// `error_count` only (fatal/error severity), never `request_count`,
|
|
206
211
|
// so a page that only reports errors still shows them without
|
|
207
|
-
// inventing requests.
|
|
208
|
-
|
|
212
|
+
// inventing requests. A handled (caught) error is an occurrence but
|
|
213
|
+
// no counted failure; warnings and info count nothing.
|
|
214
|
+
if ((severity === "fatal" || severity === "error") && !logged) {
|
|
209
215
|
bumpRollup(event.route ? (event.route.split("?")[0] ?? "") : "", occurredAt, { errorCount: 1 });
|
|
210
216
|
}
|
|
211
217
|
occurrences.push({
|
|
@@ -228,7 +234,15 @@ export function normalizeBrowserPayload(payload) {
|
|
|
228
234
|
...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
|
|
229
235
|
...(event.line !== undefined ? { line: event.line } : {}),
|
|
230
236
|
...(event.column !== undefined ? { column: event.column } : {}),
|
|
231
|
-
...(
|
|
237
|
+
...(context ? { context } : {}),
|
|
238
|
+
...(logged
|
|
239
|
+
? {
|
|
240
|
+
"autter.handled": true,
|
|
241
|
+
"autter.handling": "caught",
|
|
242
|
+
"autter.log.source": typeof context?.["autter.log.source"] === "string" ? context["autter.log.source"] : "console",
|
|
243
|
+
"autter.log.message": context?.["autter.log.message"],
|
|
244
|
+
}
|
|
245
|
+
: {}),
|
|
232
246
|
},
|
|
233
247
|
occurredAt,
|
|
234
248
|
...liftErrorFields(event.context),
|
package/dist/normalize-otlp.js
CHANGED
|
@@ -7,6 +7,8 @@ import { extractLlmCall } from "./llm.js";
|
|
|
7
7
|
import { BREADCRUMBS_ATTRIBUTE, breadcrumbsField } from "./breadcrumbs.js";
|
|
8
8
|
import { dedupeTraceSignals, hasAutterKey, hasNextContext } from "./trace-dedupe.js";
|
|
9
9
|
import { requestMembership } from "./request-spans.js";
|
|
10
|
+
import { DEBUG_IDS_ATTRIBUTE, serverDebugIds, withoutDebugIdsAttribute } from "./source-maps.js";
|
|
11
|
+
import { histogramLatency, mergeLatency, requestLatency } from "./latency.js";
|
|
10
12
|
import { asSeverity, } from "./types.js";
|
|
11
13
|
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
|
|
12
14
|
function attrMap(attributes) {
|
|
@@ -40,6 +42,17 @@ function spanDurationMs(span) {
|
|
|
40
42
|
function isErrorStatus(code) {
|
|
41
43
|
return code === 2 || code === "STATUS_CODE_ERROR";
|
|
42
44
|
}
|
|
45
|
+
/**
|
|
46
|
+
* An error the app handled on purpose and recorded through Autter: a caught
|
|
47
|
+
* error (`autter.handling = "caught"`, RF-01) or a deliberate fallback
|
|
48
|
+
* (`"deliberate"`, RF-02). These are occurrences and issues, but they never
|
|
49
|
+
* fail their span or request and never count in `error_count`. The bare
|
|
50
|
+
* `autter.handled` marker (the opt-in V8 sampler) keeps its old meaning and
|
|
51
|
+
* still counts.
|
|
52
|
+
*/
|
|
53
|
+
function softlyHandled(handling) {
|
|
54
|
+
return handling === "caught" || handling === "deliberate";
|
|
55
|
+
}
|
|
43
56
|
const SPAN_KINDS = {
|
|
44
57
|
"0": "unspecified",
|
|
45
58
|
"1": "internal",
|
|
@@ -137,8 +150,11 @@ export function normalizeTraces(request) {
|
|
|
137
150
|
const startedAt = nanosToDate(span.startTimeUnixNano);
|
|
138
151
|
const durationMs = spanDurationMs(span);
|
|
139
152
|
const kind = spanKind(span.kind);
|
|
140
|
-
|
|
141
|
-
|
|
153
|
+
// An exception the app handled (`autter.handled`: caught and
|
|
154
|
+
// logged, or a deliberate fallback) never fails its span.
|
|
155
|
+
const hasFailureEvent = (span.events ?? []).some((event) => (event.name === "exception" && !softlyHandled(attrMap(event.attributes).get("autter.handling")))
|
|
156
|
+
|| (event.name === "autter.outcome"
|
|
157
|
+
&& attrMap(event.attributes).get("autter.outcome.status") === "error"));
|
|
142
158
|
const isError = statusCode !== null && statusCode >= 400 && statusCode < 500
|
|
143
159
|
? hasFailureEvent
|
|
144
160
|
: hasFailureEvent || isErrorStatus(span.status?.code) || (statusCode !== null && statusCode >= 500);
|
|
@@ -207,8 +223,11 @@ export function normalizeTraces(request) {
|
|
|
207
223
|
statusCode,
|
|
208
224
|
traceId: span.traceId ?? null,
|
|
209
225
|
sessionId: null,
|
|
210
|
-
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
|
|
211
|
-
...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }),
|
|
226
|
+
attributes: withoutDebugIdsAttribute(sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
|
|
227
|
+
...(handled ? { "autter.handled": true, ...(eventAttrs.has("autter.sampled") ? { "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) } : {}) })),
|
|
228
|
+
// Server SDKs (runtime-core errorAttributes): the debug ids of the bundles in the
|
|
229
|
+
// stack (RF-05), which locate their source maps.
|
|
230
|
+
...serverDebugIds(eventAttrs.get(DEBUG_IDS_ATTRIBUTE)),
|
|
212
231
|
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
213
232
|
// Event attributes first: one span can carry several
|
|
214
233
|
// exceptions, each with its own code.
|
|
@@ -337,6 +356,11 @@ export function normalizeTraces(request) {
|
|
|
337
356
|
const occ = occurrences[i];
|
|
338
357
|
if (occ.severity !== "fatal" && occ.severity !== "error")
|
|
339
358
|
return;
|
|
359
|
+
// A caught error or deliberate fallback is an occurrence and an issue,
|
|
360
|
+
// but no failure: it never fails its request and never counts in
|
|
361
|
+
// `error_count`.
|
|
362
|
+
if (softlyHandled(occ.attributes?.["autter.handling"]))
|
|
363
|
+
return;
|
|
340
364
|
const request = members.map(requestOfSignal).find((r) => r !== undefined);
|
|
341
365
|
if (request) {
|
|
342
366
|
request.hasErrorOccurrence = true;
|
|
@@ -373,6 +397,7 @@ export function normalizeTraces(request) {
|
|
|
373
397
|
errorCount: failed ? 1 : 0,
|
|
374
398
|
// The request's latency is its outermost span's.
|
|
375
399
|
durationSumMs: span.durationMs,
|
|
400
|
+
...withLatency(requestLatency(span.durationMs)),
|
|
376
401
|
});
|
|
377
402
|
break;
|
|
378
403
|
case "metrics": {
|
|
@@ -515,6 +540,9 @@ function methodFromSpan(span) {
|
|
|
515
540
|
const raw = span.attributes?.["http.request.method"];
|
|
516
541
|
return typeof raw === "string" && raw.trim() ? raw.trim().toUpperCase().slice(0, 16) : null;
|
|
517
542
|
}
|
|
543
|
+
function withLatency(latency) {
|
|
544
|
+
return latency ? { latency } : {};
|
|
545
|
+
}
|
|
518
546
|
function minuteBucket(date) {
|
|
519
547
|
return new Date(Math.floor(date.getTime() / 60_000) * 60_000);
|
|
520
548
|
}
|
|
@@ -531,13 +559,17 @@ function addToRollup(rollups, point) {
|
|
|
531
559
|
const key = rollupKey(point);
|
|
532
560
|
const existing = rollups.get(key);
|
|
533
561
|
if (!existing) {
|
|
534
|
-
|
|
562
|
+
const latency = mergeLatency(undefined, point.latency);
|
|
563
|
+
rollups.set(key, { ...point, ...withLatency(latency) });
|
|
535
564
|
return;
|
|
536
565
|
}
|
|
537
566
|
existing.requestCount += point.requestCount;
|
|
538
567
|
existing.errorCount += point.errorCount;
|
|
539
568
|
existing.durationSumMs += point.durationSumMs;
|
|
540
569
|
existing.sessionCount += point.sessionCount;
|
|
570
|
+
const latency = mergeLatency(existing.latency, point.latency);
|
|
571
|
+
if (latency)
|
|
572
|
+
existing.latency = latency;
|
|
541
573
|
}
|
|
542
574
|
// HTTP-server duration instruments we know how to fold into rollups.
|
|
543
575
|
// `http.server.duration` (old semconv) is milliseconds;
|
|
@@ -598,6 +630,8 @@ export function normalizeMetrics(request) {
|
|
|
598
630
|
errorCount: statusCode !== null && statusCode >= 500 ? count : 0,
|
|
599
631
|
durationSumMs: (dataPoint.sum ?? 0) * multiplier,
|
|
600
632
|
sessionCount: 0,
|
|
633
|
+
// p50/p95/p99 from the same buckets (IP-02, latency.ts).
|
|
634
|
+
...withLatency(histogramLatency((dataPoint.explicitBounds ?? []).map((bound) => Number(bound) * multiplier), (dataPoint.bucketCounts ?? []).map(Number))),
|
|
601
635
|
});
|
|
602
636
|
}
|
|
603
637
|
}
|
package/dist/server.js
CHANGED
|
@@ -1,10 +1,11 @@
|
|
|
1
1
|
import express from "express";
|
|
2
2
|
import { KeyResolver, RateLimiter } from "./auth.js";
|
|
3
3
|
import { countTruncations } from "./context.js";
|
|
4
|
-
import { autterKeyHash, cumulativeHttpHistogramPoints, dataPointsInRequest, dropRoute, IngestDropCounter, PARTIAL_MESSAGES, partialSuccess, recordsInBody, spansInRequest, } from "./ingest-drops.js";
|
|
4
|
+
import { autterKeyHash, caughtLimitedRecords, cumulativeHttpHistogramPoints, dataPointsInRequest, dropRoute, IngestDropCounter, PARTIAL_MESSAGES, partialSuccess, recordsInBody, spansInRequest, } from "./ingest-drops.js";
|
|
5
5
|
import { ClickHouseStore } from "./clickhouse.js";
|
|
6
6
|
import { normalizeLatencyHistograms } from "./latency.js";
|
|
7
7
|
import { deriveFields, occurrenceIdFor } from "./fingerprint.js";
|
|
8
|
+
import { causeTexts } from "./cause-chain.js";
|
|
8
9
|
import { occurrenceIdentity } from "./fingerprint-v3.js";
|
|
9
10
|
import { parseStackFrames } from "./stack-frames.js";
|
|
10
11
|
import { SourceMapCache, Symbolicator } from "./symbolicate.js";
|
|
@@ -45,8 +46,9 @@ const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
|
45
46
|
export function createIngesterApp(config) {
|
|
46
47
|
configureRedaction(config);
|
|
47
48
|
const store = new ClickHouseStore(config);
|
|
48
|
-
// Browser JS frames are resolved through uploaded source
|
|
49
|
-
// are fingerprinted (frames-v3), with parsed maps cached
|
|
49
|
+
// Browser, Node and edge JS frames are resolved through uploaded source
|
|
50
|
+
// maps before they are fingerprinted (frames-v3), with parsed maps cached
|
|
51
|
+
// per release+file.
|
|
50
52
|
const symbolicator = new Symbolicator((key) => store.lookupSourceMap(key), new SourceMapCache({ maxBytes: (config.sourceMapCacheMb ?? 64) * 1024 * 1024 }));
|
|
51
53
|
// Fingerprinted occurrences feed the consumer's issue grouping, metric
|
|
52
54
|
// points feed the request/error-rate rollups, LLM calls feed spend
|
|
@@ -392,30 +394,63 @@ export function createIngesterApp(config) {
|
|
|
392
394
|
}
|
|
393
395
|
return ctx;
|
|
394
396
|
}
|
|
395
|
-
/**
|
|
396
|
-
|
|
397
|
-
* view: parsed innermost-first, then — browser only, where bundles are
|
|
398
|
-
* minified — resolved through the release's source maps. Symbolication
|
|
399
|
-
* never fails ingest (it falls back to the parsed frames).
|
|
400
|
-
*/
|
|
401
|
-
async function framesFor(ctx, input) {
|
|
402
|
-
let frames;
|
|
397
|
+
/** A stack's frames, innermost first; none when it does not parse. */
|
|
398
|
+
function parsedFrames(stack) {
|
|
403
399
|
try {
|
|
404
|
-
|
|
400
|
+
return parseStackFrames(stack);
|
|
405
401
|
}
|
|
406
402
|
catch (err) {
|
|
407
403
|
console.warn("stack parse failed — no frames:", err instanceof Error ? err.message : err);
|
|
408
404
|
return [];
|
|
409
405
|
}
|
|
410
|
-
|
|
411
|
-
|
|
406
|
+
}
|
|
407
|
+
/**
|
|
408
|
+
* Frames resolved through the release's source maps: minified browser
|
|
409
|
+
* bundles, and bundled Node and edge code (RF-05: tsup, esbuild, webpack
|
|
410
|
+
* and Next.js server output) — only JS frames a map was uploaded for
|
|
411
|
+
* change. Never fails ingest (it falls back to the parsed frames).
|
|
412
|
+
*/
|
|
413
|
+
function symbolicated(ctx, input, frames) {
|
|
414
|
+
if (frames.length === 0)
|
|
415
|
+
return Promise.resolve(frames);
|
|
412
416
|
return symbolicator.symbolicate(frames, {
|
|
413
417
|
orgId: ctx.orgId,
|
|
414
418
|
repositoryId: ctx.repositoryId,
|
|
415
419
|
release: input.release,
|
|
416
420
|
debugIds: input.debugIds ?? null,
|
|
421
|
+
server: input.source !== "browser",
|
|
417
422
|
});
|
|
418
423
|
}
|
|
424
|
+
/**
|
|
425
|
+
* Structured frames for the frames-v3 identity and the platform's stack
|
|
426
|
+
* view: parsed innermost-first, then resolved through the release's
|
|
427
|
+
* source maps, with context lines on in-app frames when the map carries
|
|
428
|
+
* their source.
|
|
429
|
+
*/
|
|
430
|
+
function framesFor(ctx, input) {
|
|
431
|
+
return symbolicated(ctx, input, parsedFrames(input.stack));
|
|
432
|
+
}
|
|
433
|
+
/**
|
|
434
|
+
* The reported exception's causes, nearest first, with their frames
|
|
435
|
+
* resolved like the reported one's (RF-06: the cause-v1 identity hashes
|
|
436
|
+
* the root cause's innermost in-app frame, which must read the same as
|
|
437
|
+
* when that error is reported unwrapped). Only used for the identity:
|
|
438
|
+
* cause frames are not stored.
|
|
439
|
+
*/
|
|
440
|
+
async function causesFor(ctx, input) {
|
|
441
|
+
let causes;
|
|
442
|
+
try {
|
|
443
|
+
causes = causeTexts(input.stack, input.attributes);
|
|
444
|
+
}
|
|
445
|
+
catch (err) {
|
|
446
|
+
console.warn("cause chain parse failed — no causes:", err instanceof Error ? err.message : err);
|
|
447
|
+
return [];
|
|
448
|
+
}
|
|
449
|
+
return Promise.all(causes.map(async (cause) => ({
|
|
450
|
+
type: cause.type,
|
|
451
|
+
frames: await symbolicated(ctx, input, parsedFrames(cause.stack)),
|
|
452
|
+
})));
|
|
453
|
+
}
|
|
419
454
|
/** Ids are content-derived (occurrenceIdFor), NOT random: an exporter
|
|
420
455
|
* that retries a batch — after a 503 from a partially-failed ClickHouse
|
|
421
456
|
* write, or when only our 2xx got lost — must produce the same ids, so
|
|
@@ -424,8 +459,8 @@ export function createIngesterApp(config) {
|
|
|
424
459
|
*
|
|
425
460
|
* `fingerprint` / `fingerprintScheme` stay the legacy identity
|
|
426
461
|
* (message-v1 or code-v1) byte-for-byte; `fingerprints` carries every
|
|
427
|
-
* candidate (manual-v1, code-v1, frames-v3, message-v1) for
|
|
428
|
-
* to resolve the issue from.
|
|
462
|
+
* candidate (manual-v1, code-v1, frames-v3, message-v1, cause-v1) for
|
|
463
|
+
* the platform to resolve the issue from.
|
|
429
464
|
*
|
|
430
465
|
* `batchIndexes` (traces): each occurrence's position among the batch's
|
|
431
466
|
* signals before the per-trace dedupe, so a merged occurrence keeps the
|
|
@@ -439,8 +474,8 @@ export function createIngesterApp(config) {
|
|
|
439
474
|
// through verbatim. Attributes were already sanitised by the
|
|
440
475
|
// normalisers (sanitizeRuntimeContext).
|
|
441
476
|
const input = scrubOccurrence(raw);
|
|
442
|
-
const frames = await framesFor(ctx, input);
|
|
443
|
-
const identity = occurrenceIdentity(input, frames);
|
|
477
|
+
const [frames, causes] = await Promise.all([framesFor(ctx, input), causesFor(ctx, input)]);
|
|
478
|
+
const identity = occurrenceIdentity(input, frames, causes);
|
|
444
479
|
const { fingerprint, scheme } = identity.legacy;
|
|
445
480
|
// Debug ids only locate source maps; they are not stored or forwarded.
|
|
446
481
|
const { debugIds: _debugIds, ...stored } = input;
|
|
@@ -672,6 +707,7 @@ export function createIngesterApp(config) {
|
|
|
672
707
|
countAccepted(ctx, req, [
|
|
673
708
|
{ reason: "spans_over_limit", records: overLimit },
|
|
674
709
|
{ reason: "attributes_truncated", records: truncated },
|
|
710
|
+
{ reason: "caught_limited", records: caughtLimitedRecords(occurrences.map((o) => o.attributes)) },
|
|
675
711
|
]);
|
|
676
712
|
}));
|
|
677
713
|
app.post("/v1/logs", asyncRoute(async (req, res) => {
|
|
@@ -723,6 +759,8 @@ export function createIngesterApp(config) {
|
|
|
723
759
|
countAccepted(ctx, req, [
|
|
724
760
|
{ reason: "log_records_over_limit", records: stats.overLimit },
|
|
725
761
|
{ reason: "attributes_truncated", records: truncated },
|
|
762
|
+
// Every record, promoted or not (the SDK's own count records too).
|
|
763
|
+
{ reason: "caught_limited", records: caughtLimitedRecords(logs.map((row) => row.attributes)) },
|
|
726
764
|
]);
|
|
727
765
|
}));
|
|
728
766
|
app.post("/v1/metrics", asyncRoute(async (req, res) => {
|
package/dist/sink.d.ts
CHANGED
|
@@ -127,7 +127,10 @@ export declare class SinkForwarder {
|
|
|
127
127
|
* keep their legacy meaning. 1.9.0 adds `count`: the occurrences this
|
|
128
128
|
* record stands for, sent only when above 1 (a loop-collapse sample:
|
|
129
129
|
* 1 + collapsedCount, see loop-collapse.ts); consumers add `count ?? 1` to
|
|
130
|
-
* their event counts.
|
|
131
|
-
*
|
|
130
|
+
* their event counts. Unreleased (RF-06): `fingerprints` may end with a
|
|
131
|
+
* `cause-v1` candidate after the legacy one; consumers that do not know the
|
|
132
|
+
* scheme drop it (the platform validates schemes one by one). RF-05: in-app
|
|
133
|
+
* frames' context lines are up to 5 on each side (3 before). Older consumers
|
|
134
|
+
* ignore unknown keys, so the version stays 1.
|
|
132
135
|
*/
|
|
133
136
|
export declare function sinkOccurrence(o: RuntimeOccurrence): Record<string, unknown>;
|