@autter/otlp-ingester 1.4.0 → 1.6.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +51 -0
- package/dist/breadcrumbs.d.ts +41 -0
- package/dist/breadcrumbs.js +151 -0
- package/dist/clickhouse.d.ts +42 -5
- package/dist/clickhouse.js +168 -7
- package/dist/compat-manifest.json +123 -0
- package/dist/compat.d.ts +149 -0
- package/dist/compat.js +221 -0
- package/dist/config.d.ts +23 -0
- package/dist/config.js +19 -0
- package/dist/context.js +24 -11
- package/dist/error-fields.d.ts +41 -0
- package/dist/error-fields.js +103 -0
- package/dist/fingerprint-override.d.ts +42 -0
- package/dist/fingerprint-override.js +76 -0
- package/dist/fingerprint-v3.d.ts +54 -0
- package/dist/fingerprint-v3.js +96 -0
- package/dist/fingerprint.d.ts +54 -0
- package/dist/fingerprint.js +46 -1
- package/dist/index.d.ts +1 -0
- package/dist/index.js +5 -3
- package/dist/llm.js +16 -3
- package/dist/log-promotion.d.ts +55 -0
- package/dist/log-promotion.js +119 -0
- package/dist/logs.d.ts +57 -1
- package/dist/logs.js +148 -11
- package/dist/memory.d.ts +2 -2
- package/dist/migrations.d.ts +13 -0
- package/dist/migrations.js +75 -2
- package/dist/normalize-browser.d.ts +67 -22
- package/dist/normalize-browser.js +46 -19
- package/dist/normalize-otlp.js +18 -2
- package/dist/redact.d.ts +33 -0
- package/dist/redact.js +96 -0
- package/dist/sdk-versions.d.ts +58 -0
- package/dist/sdk-versions.js +167 -0
- package/dist/server.d.ts +3 -0
- package/dist/server.js +219 -15
- package/dist/sink.d.ts +13 -0
- package/dist/sink.js +37 -4
- package/dist/source-maps-lookup.d.ts +31 -0
- package/dist/source-maps-lookup.js +64 -0
- package/dist/source-maps.d.ts +63 -2
- package/dist/source-maps.js +121 -13
- package/dist/stack-fixtures.d.ts +10 -2
- package/dist/stack-fixtures.js +13 -3
- package/dist/stack-frames.d.ts +48 -0
- package/dist/stack-frames.js +441 -0
- package/dist/symbolicate.d.ts +79 -0
- package/dist/symbolicate.js +257 -0
- package/dist/types.d.ts +86 -0
- package/package.json +2 -1
|
@@ -6,6 +6,8 @@ export declare const browserPayloadSchema: z.ZodObject<{
|
|
|
6
6
|
service: z.ZodString;
|
|
7
7
|
environment: z.ZodString;
|
|
8
8
|
release: z.ZodOptional<z.ZodString>;
|
|
9
|
+
/** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
|
|
10
|
+
sdk: z.ZodCatch<z.ZodOptional<z.ZodString>>;
|
|
9
11
|
events: z.ZodArray<z.ZodObject<{
|
|
10
12
|
id: z.ZodOptional<z.ZodString>;
|
|
11
13
|
type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event", "outcome", "request_failure", "csp_violation", "timing"]>;
|
|
@@ -23,82 +25,125 @@ export declare const browserPayloadSchema: z.ZodObject<{
|
|
|
23
25
|
column: z.ZodOptional<z.ZodNumber>;
|
|
24
26
|
/** Path only; query strings are stripped defensively anyway. */
|
|
25
27
|
route: z.ZodOptional<z.ZodString>;
|
|
28
|
+
/**
|
|
29
|
+
* Free-form, scrubbed context. runtime-browser ≥1.4.0 also sends the
|
|
30
|
+
* declared-error keys `autter.error.code|why|fix|link|expected` (from
|
|
31
|
+
* coded errors / `autterErrorFromResponse`) and `autter.request.id` (the
|
|
32
|
+
* `x-request-id` of a failed fetch/XHR). They are validated one by one
|
|
33
|
+
* in normalizeBrowserPayload — a malformed value is dropped on its own
|
|
34
|
+
* instead of rejecting the whole beacon.
|
|
35
|
+
*/
|
|
26
36
|
context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
|
|
37
|
+
/** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
|
|
38
|
+
breadcrumbs: z.ZodCatch<z.ZodOptional<z.ZodArray<z.ZodUnknown, "many">>>;
|
|
39
|
+
/** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
|
|
40
|
+
* event's stack (EI-13). Malformed values are dropped, never the beacon. */
|
|
41
|
+
debugIds: z.ZodCatch<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
|
|
42
|
+
/**
|
|
43
|
+
* Fingerprint override (runtime-browser 1.6.0+): one string or a list.
|
|
44
|
+
* Bounded and scrubbed in fingerprint-override.ts; a malformed value is
|
|
45
|
+
* dropped on its own.
|
|
46
|
+
*/
|
|
47
|
+
fingerprint: z.ZodCatch<z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>>;
|
|
48
|
+
/**
|
|
49
|
+
* runtime-browser ≥1.6.0: W3C trace id of the request behind the event
|
|
50
|
+
* (the `traceparent` it sent), joining it to the server trace. A malformed
|
|
51
|
+
* id is dropped on its own instead of rejecting the beacon.
|
|
52
|
+
*/
|
|
53
|
+
traceId: z.ZodCatch<z.ZodOptional<z.ZodString>>;
|
|
27
54
|
}, "strip", z.ZodTypeAny, {
|
|
28
|
-
type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
|
|
29
55
|
message: string;
|
|
56
|
+
type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
|
|
30
57
|
timestamp: string;
|
|
31
|
-
|
|
58
|
+
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
59
|
+
errorType?: string | undefined;
|
|
60
|
+
stack?: string | undefined;
|
|
32
61
|
route?: string | undefined;
|
|
62
|
+
traceId?: string | undefined;
|
|
63
|
+
breadcrumbs?: unknown[] | undefined;
|
|
64
|
+
debugIds?: Record<string, unknown> | undefined;
|
|
65
|
+
durationMs?: number | undefined;
|
|
33
66
|
filename?: string | undefined;
|
|
34
67
|
id?: string | undefined;
|
|
35
|
-
severity?: "fatal" | "error" | "warning" | "info" | undefined;
|
|
36
68
|
name?: string | undefined;
|
|
37
|
-
stack?: string | undefined;
|
|
38
|
-
errorType?: string | undefined;
|
|
39
69
|
line?: number | undefined;
|
|
40
70
|
column?: number | undefined;
|
|
41
71
|
context?: Record<string, unknown> | undefined;
|
|
72
|
+
fingerprint?: string | string[] | undefined;
|
|
42
73
|
}, {
|
|
43
|
-
type: "
|
|
74
|
+
type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
|
|
44
75
|
timestamp: string;
|
|
45
|
-
durationMs?: number | undefined;
|
|
46
|
-
route?: string | undefined;
|
|
47
76
|
message?: string | undefined;
|
|
77
|
+
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
78
|
+
errorType?: string | undefined;
|
|
79
|
+
stack?: string | undefined;
|
|
80
|
+
route?: string | undefined;
|
|
81
|
+
traceId?: unknown;
|
|
82
|
+
breadcrumbs?: unknown;
|
|
83
|
+
debugIds?: unknown;
|
|
84
|
+
durationMs?: number | undefined;
|
|
48
85
|
filename?: string | undefined;
|
|
49
86
|
id?: string | undefined;
|
|
50
|
-
severity?: "fatal" | "error" | "warning" | "info" | undefined;
|
|
51
87
|
name?: string | undefined;
|
|
52
|
-
stack?: string | undefined;
|
|
53
|
-
errorType?: string | undefined;
|
|
54
88
|
line?: number | undefined;
|
|
55
89
|
column?: number | undefined;
|
|
56
90
|
context?: Record<string, unknown> | undefined;
|
|
91
|
+
fingerprint?: unknown;
|
|
57
92
|
}>, "many">;
|
|
58
93
|
}, "strip", z.ZodTypeAny, {
|
|
59
94
|
service: string;
|
|
60
95
|
environment: string;
|
|
61
96
|
version: 1;
|
|
62
97
|
events: {
|
|
63
|
-
type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
|
|
64
98
|
message: string;
|
|
99
|
+
type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
|
|
65
100
|
timestamp: string;
|
|
66
|
-
|
|
101
|
+
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
102
|
+
errorType?: string | undefined;
|
|
103
|
+
stack?: string | undefined;
|
|
67
104
|
route?: string | undefined;
|
|
105
|
+
traceId?: string | undefined;
|
|
106
|
+
breadcrumbs?: unknown[] | undefined;
|
|
107
|
+
debugIds?: Record<string, unknown> | undefined;
|
|
108
|
+
durationMs?: number | undefined;
|
|
68
109
|
filename?: string | undefined;
|
|
69
110
|
id?: string | undefined;
|
|
70
|
-
severity?: "fatal" | "error" | "warning" | "info" | undefined;
|
|
71
111
|
name?: string | undefined;
|
|
72
|
-
stack?: string | undefined;
|
|
73
|
-
errorType?: string | undefined;
|
|
74
112
|
line?: number | undefined;
|
|
75
113
|
column?: number | undefined;
|
|
76
114
|
context?: Record<string, unknown> | undefined;
|
|
115
|
+
fingerprint?: string | string[] | undefined;
|
|
77
116
|
}[];
|
|
78
117
|
release?: string | undefined;
|
|
79
118
|
sessionId?: string | undefined;
|
|
119
|
+
sdk?: string | undefined;
|
|
80
120
|
}, {
|
|
81
121
|
service: string;
|
|
82
122
|
environment: string;
|
|
83
123
|
version: 1;
|
|
84
124
|
events: {
|
|
85
|
-
type: "
|
|
125
|
+
type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
|
|
86
126
|
timestamp: string;
|
|
87
|
-
durationMs?: number | undefined;
|
|
88
|
-
route?: string | undefined;
|
|
89
127
|
message?: string | undefined;
|
|
128
|
+
severity?: "info" | "warning" | "error" | "fatal" | undefined;
|
|
129
|
+
errorType?: string | undefined;
|
|
130
|
+
stack?: string | undefined;
|
|
131
|
+
route?: string | undefined;
|
|
132
|
+
traceId?: unknown;
|
|
133
|
+
breadcrumbs?: unknown;
|
|
134
|
+
debugIds?: unknown;
|
|
135
|
+
durationMs?: number | undefined;
|
|
90
136
|
filename?: string | undefined;
|
|
91
137
|
id?: string | undefined;
|
|
92
|
-
severity?: "fatal" | "error" | "warning" | "info" | undefined;
|
|
93
138
|
name?: string | undefined;
|
|
94
|
-
stack?: string | undefined;
|
|
95
|
-
errorType?: string | undefined;
|
|
96
139
|
line?: number | undefined;
|
|
97
140
|
column?: number | undefined;
|
|
98
141
|
context?: Record<string, unknown> | undefined;
|
|
142
|
+
fingerprint?: unknown;
|
|
99
143
|
}[];
|
|
100
144
|
release?: string | undefined;
|
|
101
145
|
sessionId?: string | undefined;
|
|
146
|
+
sdk?: unknown;
|
|
102
147
|
}>;
|
|
103
148
|
export type BrowserPayload = z.infer<typeof browserPayloadSchema>;
|
|
104
149
|
export interface NormalizedBrowser {
|
|
@@ -1,5 +1,11 @@
|
|
|
1
1
|
import { z } from "zod";
|
|
2
|
+
import { liftErrorFields } from "./error-fields.js";
|
|
3
|
+
import { fingerprintOverrideOf } from "./fingerprint-override.js";
|
|
4
|
+
import { sanitizeRuntimeContext } from "./context.js";
|
|
2
5
|
import { normalizeRoute } from "./fingerprint.js";
|
|
6
|
+
import { scrubText } from "./redact.js";
|
|
7
|
+
import { breadcrumbsField } from "./breadcrumbs.js";
|
|
8
|
+
import { browserDebugIds } from "./source-maps.js";
|
|
3
9
|
import { asSeverity, } from "./types.js";
|
|
4
10
|
/**
|
|
5
11
|
* The compact browser payload (`version: 1`) emitted by
|
|
@@ -34,7 +40,32 @@ const browserEventSchema = z.object({
|
|
|
34
40
|
column: z.number().int().nonnegative().optional(),
|
|
35
41
|
/** Path only; query strings are stripped defensively anyway. */
|
|
36
42
|
route: z.string().max(1000).optional(),
|
|
43
|
+
/**
|
|
44
|
+
* Free-form, scrubbed context. runtime-browser ≥1.4.0 also sends the
|
|
45
|
+
* declared-error keys `autter.error.code|why|fix|link|expected` (from
|
|
46
|
+
* coded errors / `autterErrorFromResponse`) and `autter.request.id` (the
|
|
47
|
+
* `x-request-id` of a failed fetch/XHR). They are validated one by one
|
|
48
|
+
* in normalizeBrowserPayload — a malformed value is dropped on its own
|
|
49
|
+
* instead of rejecting the whole beacon.
|
|
50
|
+
*/
|
|
37
51
|
context: z.record(z.unknown()).optional(),
|
|
52
|
+
/** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
|
|
53
|
+
breadcrumbs: z.array(z.unknown()).optional().catch(undefined),
|
|
54
|
+
/** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
|
|
55
|
+
* event's stack (EI-13). Malformed values are dropped, never the beacon. */
|
|
56
|
+
debugIds: z.record(z.string().max(1000), z.unknown()).optional().catch(undefined),
|
|
57
|
+
/**
|
|
58
|
+
* Fingerprint override (runtime-browser 1.6.0+): one string or a list.
|
|
59
|
+
* Bounded and scrubbed in fingerprint-override.ts; a malformed value is
|
|
60
|
+
* dropped on its own.
|
|
61
|
+
*/
|
|
62
|
+
fingerprint: z.union([z.string().max(2000), z.array(z.string().max(2000)).max(50)]).optional().catch(undefined),
|
|
63
|
+
/**
|
|
64
|
+
* runtime-browser ≥1.6.0: W3C trace id of the request behind the event
|
|
65
|
+
* (the `traceparent` it sent), joining it to the server trace. A malformed
|
|
66
|
+
* id is dropped on its own instead of rejecting the beacon.
|
|
67
|
+
*/
|
|
68
|
+
traceId: z.string().regex(/^(?!0{32})[0-9a-f]{32}$/).optional().catch(undefined),
|
|
38
69
|
});
|
|
39
70
|
export const browserPayloadSchema = z.object({
|
|
40
71
|
version: z.literal(1),
|
|
@@ -42,6 +73,8 @@ export const browserPayloadSchema = z.object({
|
|
|
42
73
|
service: z.string().min(1).max(200),
|
|
43
74
|
environment: z.string().min(1).max(100),
|
|
44
75
|
release: z.string().max(200).optional(),
|
|
76
|
+
/** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
|
|
77
|
+
sdk: z.string().regex(/^\d{1,6}\.\d{1,6}\.\d{1,6}[0-9A-Za-z.+-]{0,20}$/).optional().catch(undefined),
|
|
45
78
|
events: z.array(browserEventSchema).max(50),
|
|
46
79
|
});
|
|
47
80
|
const TYPE_TO_ERROR_TYPE = {
|
|
@@ -53,25 +86,14 @@ const TYPE_TO_ERROR_TYPE = {
|
|
|
53
86
|
csp_violation: "CspViolation",
|
|
54
87
|
};
|
|
55
88
|
// Content-level gate for the free-form `context` bag. The schema whitelist
|
|
56
|
-
// above is structural; this masks
|
|
57
|
-
//
|
|
58
|
-
//
|
|
59
|
-
//
|
|
60
|
-
const
|
|
61
|
-
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
|
|
89
|
+
// above is structural; this masks secrets/PII inside whatever a (possibly
|
|
90
|
+
// outdated) SDK still sends — sensitive keys and secret-shaped values at any
|
|
91
|
+
// depth, via the same sanitiser as OTLP attributes. Message, stack, and route
|
|
92
|
+
// are scrubbed centrally before fingerprinting (server.ts fingerprintAll).
|
|
93
|
+
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
|
|
62
94
|
const REDACTED = "[redacted]";
|
|
63
95
|
function scrubContext(context) {
|
|
64
|
-
|
|
65
|
-
for (const [key, value] of Object.entries(context)) {
|
|
66
|
-
if (value === undefined || value === null)
|
|
67
|
-
continue;
|
|
68
|
-
out[key] = SENSITIVE_KEY_RE.test(key)
|
|
69
|
-
? REDACTED
|
|
70
|
-
: typeof value === "string"
|
|
71
|
-
? value.replace(EMAIL_VALUE_RE, REDACTED)
|
|
72
|
-
: value;
|
|
73
|
-
}
|
|
74
|
-
return out;
|
|
96
|
+
return sanitizeRuntimeContext(Object.fromEntries(Object.entries(context).filter(([, value]) => value !== undefined && value !== null)));
|
|
75
97
|
}
|
|
76
98
|
/** Default severity per event type when the SDK doesn't say. */
|
|
77
99
|
const TYPE_TO_SEVERITY = {
|
|
@@ -86,6 +108,7 @@ export function normalizeBrowserPayload(payload) {
|
|
|
86
108
|
const occurrences = [];
|
|
87
109
|
const rollups = new Map();
|
|
88
110
|
function bumpRollup(route, occurredAt, counts) {
|
|
111
|
+
route = scrubText(route);
|
|
89
112
|
const bucketAt = new Date(Math.floor(occurredAt.getTime() / 60_000) * 60_000);
|
|
90
113
|
const key = `${route} ${bucketAt.getTime()}`;
|
|
91
114
|
const existing = rollups.get(key);
|
|
@@ -125,7 +148,7 @@ export function normalizeBrowserPayload(payload) {
|
|
|
125
148
|
const name = event.name === "browser.longtask" ? "browser.longtask"
|
|
126
149
|
: `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0].replace(EMAIL_VALUE_RE, REDACTED))}`;
|
|
127
150
|
bumpRollup(name, occurredAt, { requestCount: 1 });
|
|
128
|
-
const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`);
|
|
151
|
+
const point = rollups.get(`${scrubText(name)} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`);
|
|
129
152
|
if (point)
|
|
130
153
|
point.durationSumMs += event.durationMs ?? 0;
|
|
131
154
|
continue;
|
|
@@ -153,7 +176,7 @@ export function normalizeBrowserPayload(payload) {
|
|
|
153
176
|
route: event.route ? (event.route.split("?")[0] ?? null) : null,
|
|
154
177
|
method: null,
|
|
155
178
|
statusCode: null,
|
|
156
|
-
traceId: null,
|
|
179
|
+
traceId: event.traceId ?? null,
|
|
157
180
|
sessionId: payload.sessionId ?? null,
|
|
158
181
|
attributes: {
|
|
159
182
|
...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
|
|
@@ -162,6 +185,10 @@ export function normalizeBrowserPayload(payload) {
|
|
|
162
185
|
...(event.context ? { context: scrubContext(event.context) } : {}),
|
|
163
186
|
},
|
|
164
187
|
occurredAt,
|
|
188
|
+
...liftErrorFields(event.context),
|
|
189
|
+
...breadcrumbsField(event.breadcrumbs),
|
|
190
|
+
...browserDebugIds(event.debugIds),
|
|
191
|
+
...fingerprintOverrideOf(event.fingerprint),
|
|
165
192
|
});
|
|
166
193
|
}
|
|
167
194
|
return { occurrences, metricPoints: [...rollups.values()] };
|
package/dist/normalize-otlp.js
CHANGED
|
@@ -1,8 +1,12 @@
|
|
|
1
1
|
import { decodeOtlpAttributes, sanitizeRuntimeContext } from "./context.js";
|
|
2
|
+
import { liftErrorFields, validRequestId } from "./error-fields.js";
|
|
3
|
+
import { fingerprintOverrideOf } from "./fingerprint-override.js";
|
|
4
|
+
import { scrubText } from "./redact.js";
|
|
2
5
|
import { normalizeRoute } from "./fingerprint.js";
|
|
3
6
|
import { extractLlmCall } from "./llm.js";
|
|
7
|
+
import { BREADCRUMBS_ATTRIBUTE, breadcrumbsField } from "./breadcrumbs.js";
|
|
4
8
|
import { asSeverity, } from "./types.js";
|
|
5
|
-
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]
|
|
9
|
+
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
|
|
6
10
|
function attrMap(attributes) {
|
|
7
11
|
const map = new Map();
|
|
8
12
|
for (const attr of attributes ?? []) {
|
|
@@ -127,7 +131,7 @@ export function normalizeTraces(request) {
|
|
|
127
131
|
traceId: span.traceId ?? "",
|
|
128
132
|
spanId: span.spanId ?? "",
|
|
129
133
|
parentSpanId: span.parentSpanId ?? null,
|
|
130
|
-
name: span.name ?? "unnamed",
|
|
134
|
+
name: scrubText(span.name ?? "unnamed").slice(0, 1000),
|
|
131
135
|
kind,
|
|
132
136
|
status: isError ? "error" : "ok",
|
|
133
137
|
route: route ? normalizeRoute(route) : null,
|
|
@@ -167,6 +171,11 @@ export function normalizeTraces(request) {
|
|
|
167
171
|
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
|
|
168
172
|
...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }),
|
|
169
173
|
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
174
|
+
// Event attributes first: one span can carry several
|
|
175
|
+
// exceptions, each with its own code.
|
|
176
|
+
...liftErrorFields(eventAttrs, attrs),
|
|
177
|
+
...breadcrumbsField(eventAttrs.get(BREADCRUMBS_ATTRIBUTE), attrs.get(BREADCRUMBS_ATTRIBUTE)),
|
|
178
|
+
...fingerprintOverrideOf(eventAttrs.get("autter.fingerprint") ?? attrs.get("autter.fingerprint")),
|
|
170
179
|
});
|
|
171
180
|
}
|
|
172
181
|
const failedOutcomes = (span.events ?? []).filter((event) => {
|
|
@@ -187,6 +196,7 @@ export function normalizeTraces(request) {
|
|
|
187
196
|
traceId: span.traceId ?? null, sessionId: null,
|
|
188
197
|
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }),
|
|
189
198
|
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
199
|
+
...liftErrorFields(outcome, attrs),
|
|
190
200
|
});
|
|
191
201
|
}
|
|
192
202
|
if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) {
|
|
@@ -209,6 +219,7 @@ export function normalizeTraces(request) {
|
|
|
209
219
|
sessionId: null,
|
|
210
220
|
attributes: decodeOtlpAttributes(span.attributes),
|
|
211
221
|
occurredAt: startedAt,
|
|
222
|
+
...liftErrorFields(attrs),
|
|
212
223
|
});
|
|
213
224
|
}
|
|
214
225
|
// LLM provider calls (GenAI semconv, Vercel AI SDK telemetry,
|
|
@@ -305,6 +316,11 @@ function inheritHttpFromParent(occurrences, occurrenceSpanIds, spans) {
|
|
|
305
316
|
continue;
|
|
306
317
|
if (!occurrence.route && http.route)
|
|
307
318
|
occurrence.route = http.route;
|
|
319
|
+
// Request ids live on the request (server) span; child error spans
|
|
320
|
+
// from captureException inherit it so the issue links to the request.
|
|
321
|
+
const requestId = validRequestId(http.attributes?.["autter.request.id"]);
|
|
322
|
+
if (!occurrence.requestId && requestId)
|
|
323
|
+
occurrence.requestId = requestId;
|
|
308
324
|
const method = methodFromSpan(http);
|
|
309
325
|
if (!occurrence.method && method)
|
|
310
326
|
occurrence.method = method;
|
package/dist/redact.d.ts
ADDED
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-side secret/PII scrubbing — the ingester's second line of defence.
|
|
3
|
+
*
|
|
4
|
+
* The SDKs scrub before anything leaves the customer's process, but the
|
|
5
|
+
* ingester also accepts older SDK versions and arbitrary third-party OTLP
|
|
6
|
+
* senders (any OTel SDK in any language) that scrub nothing. Everything this
|
|
7
|
+
* module touches is written to ClickHouse and forwarded to the sink webhook,
|
|
8
|
+
* which is where error context is later read for LLM-generated fixes — so
|
|
9
|
+
* scrubbing here, before storage, covers every downstream consumer.
|
|
10
|
+
*
|
|
11
|
+
* Same patterns as redactText() in @autter/runtime-node; parity is enforced
|
|
12
|
+
* by the shared test-vectors/redaction.json.
|
|
13
|
+
*/
|
|
14
|
+
import type { RuntimeOccurrenceInput } from "./types.js";
|
|
15
|
+
/** Keys whose values are masked wholesale (tested case-insensitively). */
|
|
16
|
+
export declare const SENSITIVE_KEY_RE: RegExp;
|
|
17
|
+
/**
|
|
18
|
+
* Deployment-specific patterns on top of the built-ins (env
|
|
19
|
+
* AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of
|
|
20
|
+
* regex sources). Server-side scrubbing has no off switch: it is the
|
|
21
|
+
* backstop for senders that don't scrub.
|
|
22
|
+
*/
|
|
23
|
+
export declare function configureRedaction(options: {
|
|
24
|
+
redactValuePatterns?: string[];
|
|
25
|
+
redactKeyPatterns?: string[];
|
|
26
|
+
}): void;
|
|
27
|
+
export declare function isSensitiveKey(key: string): boolean;
|
|
28
|
+
export declare function scrubText(value: string): string;
|
|
29
|
+
/** Nullable convenience for optional columns. */
|
|
30
|
+
export declare function scrubOptional(value: string | null): string | null;
|
|
31
|
+
/** Scrub the free-text fields of an occurrence (message, stack, route,
|
|
32
|
+
* error type). Returns a copy. */
|
|
33
|
+
export declare function scrubOccurrence(occurrence: RuntimeOccurrenceInput): RuntimeOccurrenceInput;
|
package/dist/redact.js
ADDED
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Server-side secret/PII scrubbing — the ingester's second line of defence.
|
|
3
|
+
*
|
|
4
|
+
* The SDKs scrub before anything leaves the customer's process, but the
|
|
5
|
+
* ingester also accepts older SDK versions and arbitrary third-party OTLP
|
|
6
|
+
* senders (any OTel SDK in any language) that scrub nothing. Everything this
|
|
7
|
+
* module touches is written to ClickHouse and forwarded to the sink webhook,
|
|
8
|
+
* which is where error context is later read for LLM-generated fixes — so
|
|
9
|
+
* scrubbing here, before storage, covers every downstream consumer.
|
|
10
|
+
*
|
|
11
|
+
* Same patterns as redactText() in @autter/runtime-node; parity is enforced
|
|
12
|
+
* by the shared test-vectors/redaction.json.
|
|
13
|
+
*/
|
|
14
|
+
const MASK = "[redacted]";
|
|
15
|
+
const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g;
|
|
16
|
+
const PREFIXED_SECRET_RE = /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g;
|
|
17
|
+
// Lookbehind, not \b: a \b start at every "eyJ" after a "-" made this
|
|
18
|
+
// quadratic ("eyJ-" x 16k took 3.4 s per string).
|
|
19
|
+
const JWT_RE = /(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g;
|
|
20
|
+
const AUTH_SCHEME_RE = /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi;
|
|
21
|
+
const HEADER_VALUE_RE = /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim;
|
|
22
|
+
const SECRET_ASSIGNMENT_RE = /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi;
|
|
23
|
+
const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi;
|
|
24
|
+
const CARD_RE = /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g;
|
|
25
|
+
const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
|
|
26
|
+
/** Keys whose values are masked wholesale (tested case-insensitively). */
|
|
27
|
+
export const SENSITIVE_KEY_RE = /password|passwd|passphrase|(^|[._-])pwd$|secret|token|credential|authori[sz]ation|^auth(-|_|$)|bearer|cookie|e-?mail|phone|(^|[^a-z])ssn($|[^a-z])|cvv|card[._-]?number|connection[._-]?string|(^|[._-])dsn$|(api|access|private|client|signing|encryption)[._-]?key|apikey|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid|request[._-]?body|response[._-]?body|headers|url[._-]?query/i;
|
|
28
|
+
let extraValuePatterns = [];
|
|
29
|
+
let extraKeyPatterns = [];
|
|
30
|
+
/**
|
|
31
|
+
* Deployment-specific patterns on top of the built-ins (env
|
|
32
|
+
* AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of
|
|
33
|
+
* regex sources). Server-side scrubbing has no off switch: it is the
|
|
34
|
+
* backstop for senders that don't scrub.
|
|
35
|
+
*/
|
|
36
|
+
export function configureRedaction(options) {
|
|
37
|
+
extraValuePatterns = (options.redactValuePatterns ?? []).map((source) => new RegExp(source, "gi"));
|
|
38
|
+
extraKeyPatterns = (options.redactKeyPatterns ?? []).map((source) => new RegExp(source, "i"));
|
|
39
|
+
}
|
|
40
|
+
export function isSensitiveKey(key) {
|
|
41
|
+
return SENSITIVE_KEY_RE.test(key) || extraKeyPatterns.some((re) => re.test(key));
|
|
42
|
+
}
|
|
43
|
+
function luhnValid(candidate) {
|
|
44
|
+
const digits = candidate.replace(/\D/g, "");
|
|
45
|
+
if (digits.length < 13 || digits.length > 19)
|
|
46
|
+
return false;
|
|
47
|
+
let sum = 0;
|
|
48
|
+
for (let i = 0; i < digits.length; i++) {
|
|
49
|
+
let d = digits.charCodeAt(digits.length - 1 - i) - 48;
|
|
50
|
+
if (i % 2 === 1) {
|
|
51
|
+
d *= 2;
|
|
52
|
+
if (d > 9)
|
|
53
|
+
d -= 9;
|
|
54
|
+
}
|
|
55
|
+
sum += d;
|
|
56
|
+
}
|
|
57
|
+
return sum % 10 === 0;
|
|
58
|
+
}
|
|
59
|
+
/** Scrub secrets/PII embedded in one string; the rest of it survives. */
|
|
60
|
+
/** Longest string scanned (same cap as the Node SDK). Callers that store
|
|
61
|
+
* less cut first; anything longer is truncated rather than stored unscanned. */
|
|
62
|
+
const MAX_SCRUB_CHARS = 64 * 1024;
|
|
63
|
+
export function scrubText(value) {
|
|
64
|
+
if (typeof value !== "string" || value === "")
|
|
65
|
+
return value;
|
|
66
|
+
let out = (value.length > MAX_SCRUB_CHARS ? value.slice(0, MAX_SCRUB_CHARS) : value)
|
|
67
|
+
.replace(PRIVATE_KEY_BLOCK_RE, MASK)
|
|
68
|
+
.replace(PREFIXED_SECRET_RE, MASK)
|
|
69
|
+
.replace(JWT_RE, MASK)
|
|
70
|
+
.replace(AUTH_SCHEME_RE, MASK)
|
|
71
|
+
.replace(HEADER_VALUE_RE, (_m, head) => head + MASK)
|
|
72
|
+
.replace(SECRET_ASSIGNMENT_RE, (_m, key, sep, val) => key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + MASK + val[0] : MASK))
|
|
73
|
+
.replace(URL_CREDENTIALS_RE, (_m, scheme) => `${scheme}${MASK}@`)
|
|
74
|
+
.replace(CARD_RE, (m) => (luhnValid(m) ? MASK : m))
|
|
75
|
+
.replace(EMAIL_RE, MASK);
|
|
76
|
+
for (const re of extraValuePatterns) {
|
|
77
|
+
re.lastIndex = 0;
|
|
78
|
+
out = out.replace(re, MASK);
|
|
79
|
+
}
|
|
80
|
+
return out;
|
|
81
|
+
}
|
|
82
|
+
/** Nullable convenience for optional columns. */
|
|
83
|
+
export function scrubOptional(value) {
|
|
84
|
+
return value === null ? null : scrubText(value);
|
|
85
|
+
}
|
|
86
|
+
/** Scrub the free-text fields of an occurrence (message, stack, route,
|
|
87
|
+
* error type). Returns a copy. */
|
|
88
|
+
export function scrubOccurrence(occurrence) {
|
|
89
|
+
return {
|
|
90
|
+
...occurrence,
|
|
91
|
+
errorType: scrubText(occurrence.errorType),
|
|
92
|
+
message: scrubText(occurrence.message),
|
|
93
|
+
stack: scrubOptional(occurrence.stack),
|
|
94
|
+
route: scrubOptional(occurrence.route),
|
|
95
|
+
};
|
|
96
|
+
}
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
import type { IngestContext } from "./types.js";
|
|
2
|
+
/** This ingester's own version (package.json — present in the npm package
|
|
3
|
+
* and the Docker image). */
|
|
4
|
+
export declare function ingesterVersion(): string;
|
|
5
|
+
export interface SdkSighting {
|
|
6
|
+
service: string;
|
|
7
|
+
environment: string;
|
|
8
|
+
sdkName: string;
|
|
9
|
+
sdkVersion: string;
|
|
10
|
+
sdkLanguage: string;
|
|
11
|
+
}
|
|
12
|
+
interface KeyValue {
|
|
13
|
+
key?: string;
|
|
14
|
+
value?: {
|
|
15
|
+
stringValue?: string;
|
|
16
|
+
};
|
|
17
|
+
}
|
|
18
|
+
interface ResourceHolder {
|
|
19
|
+
resource?: {
|
|
20
|
+
attributes?: KeyValue[];
|
|
21
|
+
};
|
|
22
|
+
}
|
|
23
|
+
export interface OtlpResourceRequest {
|
|
24
|
+
resourceSpans?: ResourceHolder[];
|
|
25
|
+
resourceMetrics?: ResourceHolder[];
|
|
26
|
+
resourceLogs?: ResourceHolder[];
|
|
27
|
+
}
|
|
28
|
+
/** SDK identity per OTLP resource. Reads only a handful of resource keys. */
|
|
29
|
+
export declare function sdkSightingsFromOtlp(request: OtlpResourceRequest | null | undefined): SdkSighting[];
|
|
30
|
+
export declare const BROWSER_SDK_NAME = "@autter/runtime-browser";
|
|
31
|
+
export interface SdkVersionRow extends SdkSighting {
|
|
32
|
+
ingesterVersion: string;
|
|
33
|
+
seenAt: Date;
|
|
34
|
+
}
|
|
35
|
+
export interface SdkVersionTrackerOptions {
|
|
36
|
+
/** Re-write an unchanged sighting at most this often. Default 1 h. */
|
|
37
|
+
refreshMs?: number;
|
|
38
|
+
/** Dedupe map bound; cleared (not grown) past this. Default 10 000. */
|
|
39
|
+
maxKeys?: number;
|
|
40
|
+
now?: () => number;
|
|
41
|
+
onError?: (err: unknown) => void;
|
|
42
|
+
}
|
|
43
|
+
export declare class SdkVersionTracker {
|
|
44
|
+
private readonly write;
|
|
45
|
+
private readonly lastWritten;
|
|
46
|
+
private readonly refreshMs;
|
|
47
|
+
private readonly maxKeys;
|
|
48
|
+
private readonly now;
|
|
49
|
+
private readonly onError;
|
|
50
|
+
private lastErrorLog;
|
|
51
|
+
constructor(write: (ctx: IngestContext, rows: SdkVersionRow[]) => Promise<void>, options?: SdkVersionTrackerOptions);
|
|
52
|
+
/** Fire-and-forget. Returns the write promise (resolved) for tests. */
|
|
53
|
+
observe(ctx: IngestContext, sightings: SdkSighting[]): Promise<void>;
|
|
54
|
+
}
|
|
55
|
+
export declare function sdkVersionTableDDL(db: string, ttlDays?: number): string;
|
|
56
|
+
/** One row per ingester version (latest report wins); not tenant data. */
|
|
57
|
+
export declare function ingesterInfoTableDDL(db: string): string;
|
|
58
|
+
export {};
|