@autter/otlp-ingester 1.4.0 → 1.6.0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (52) hide show
  1. package/README.md +51 -0
  2. package/dist/breadcrumbs.d.ts +41 -0
  3. package/dist/breadcrumbs.js +151 -0
  4. package/dist/clickhouse.d.ts +42 -5
  5. package/dist/clickhouse.js +168 -7
  6. package/dist/compat-manifest.json +123 -0
  7. package/dist/compat.d.ts +149 -0
  8. package/dist/compat.js +221 -0
  9. package/dist/config.d.ts +23 -0
  10. package/dist/config.js +19 -0
  11. package/dist/context.js +24 -11
  12. package/dist/error-fields.d.ts +41 -0
  13. package/dist/error-fields.js +103 -0
  14. package/dist/fingerprint-override.d.ts +42 -0
  15. package/dist/fingerprint-override.js +76 -0
  16. package/dist/fingerprint-v3.d.ts +54 -0
  17. package/dist/fingerprint-v3.js +96 -0
  18. package/dist/fingerprint.d.ts +54 -0
  19. package/dist/fingerprint.js +46 -1
  20. package/dist/index.d.ts +1 -0
  21. package/dist/index.js +5 -3
  22. package/dist/llm.js +16 -3
  23. package/dist/log-promotion.d.ts +55 -0
  24. package/dist/log-promotion.js +119 -0
  25. package/dist/logs.d.ts +57 -1
  26. package/dist/logs.js +148 -11
  27. package/dist/memory.d.ts +2 -2
  28. package/dist/migrations.d.ts +13 -0
  29. package/dist/migrations.js +75 -2
  30. package/dist/normalize-browser.d.ts +67 -22
  31. package/dist/normalize-browser.js +46 -19
  32. package/dist/normalize-otlp.js +18 -2
  33. package/dist/redact.d.ts +33 -0
  34. package/dist/redact.js +96 -0
  35. package/dist/sdk-versions.d.ts +58 -0
  36. package/dist/sdk-versions.js +167 -0
  37. package/dist/server.d.ts +3 -0
  38. package/dist/server.js +219 -15
  39. package/dist/sink.d.ts +13 -0
  40. package/dist/sink.js +37 -4
  41. package/dist/source-maps-lookup.d.ts +31 -0
  42. package/dist/source-maps-lookup.js +64 -0
  43. package/dist/source-maps.d.ts +63 -2
  44. package/dist/source-maps.js +121 -13
  45. package/dist/stack-fixtures.d.ts +10 -2
  46. package/dist/stack-fixtures.js +13 -3
  47. package/dist/stack-frames.d.ts +48 -0
  48. package/dist/stack-frames.js +441 -0
  49. package/dist/symbolicate.d.ts +79 -0
  50. package/dist/symbolicate.js +257 -0
  51. package/dist/types.d.ts +86 -0
  52. package/package.json +2 -1
@@ -6,6 +6,8 @@ export declare const browserPayloadSchema: z.ZodObject<{
6
6
  service: z.ZodString;
7
7
  environment: z.ZodString;
8
8
  release: z.ZodOptional<z.ZodString>;
9
+ /** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
10
+ sdk: z.ZodCatch<z.ZodOptional<z.ZodString>>;
9
11
  events: z.ZodArray<z.ZodObject<{
10
12
  id: z.ZodOptional<z.ZodString>;
11
13
  type: z.ZodEnum<["exception", "unhandled_rejection", "message", "session_start", "track_event", "outcome", "request_failure", "csp_violation", "timing"]>;
@@ -23,82 +25,125 @@ export declare const browserPayloadSchema: z.ZodObject<{
23
25
  column: z.ZodOptional<z.ZodNumber>;
24
26
  /** Path only; query strings are stripped defensively anyway. */
25
27
  route: z.ZodOptional<z.ZodString>;
28
+ /**
29
+ * Free-form, scrubbed context. runtime-browser ≥1.4.0 also sends the
30
+ * declared-error keys `autter.error.code|why|fix|link|expected` (from
31
+ * coded errors / `autterErrorFromResponse`) and `autter.request.id` (the
32
+ * `x-request-id` of a failed fetch/XHR). They are validated one by one
33
+ * in normalizeBrowserPayload — a malformed value is dropped on its own
34
+ * instead of rejecting the whole beacon.
35
+ */
26
36
  context: z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>;
37
+ /** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
38
+ breadcrumbs: z.ZodCatch<z.ZodOptional<z.ZodArray<z.ZodUnknown, "many">>>;
39
+ /** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
40
+ * event's stack (EI-13). Malformed values are dropped, never the beacon. */
41
+ debugIds: z.ZodCatch<z.ZodOptional<z.ZodRecord<z.ZodString, z.ZodUnknown>>>;
42
+ /**
43
+ * Fingerprint override (runtime-browser 1.6.0+): one string or a list.
44
+ * Bounded and scrubbed in fingerprint-override.ts; a malformed value is
45
+ * dropped on its own.
46
+ */
47
+ fingerprint: z.ZodCatch<z.ZodOptional<z.ZodUnion<[z.ZodString, z.ZodArray<z.ZodString, "many">]>>>;
48
+ /**
49
+ * runtime-browser ≥1.6.0: W3C trace id of the request behind the event
50
+ * (the `traceparent` it sent), joining it to the server trace. A malformed
51
+ * id is dropped on its own instead of rejecting the beacon.
52
+ */
53
+ traceId: z.ZodCatch<z.ZodOptional<z.ZodString>>;
27
54
  }, "strip", z.ZodTypeAny, {
28
- type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
29
55
  message: string;
56
+ type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
30
57
  timestamp: string;
31
- durationMs?: number | undefined;
58
+ severity?: "info" | "warning" | "error" | "fatal" | undefined;
59
+ errorType?: string | undefined;
60
+ stack?: string | undefined;
32
61
  route?: string | undefined;
62
+ traceId?: string | undefined;
63
+ breadcrumbs?: unknown[] | undefined;
64
+ debugIds?: Record<string, unknown> | undefined;
65
+ durationMs?: number | undefined;
33
66
  filename?: string | undefined;
34
67
  id?: string | undefined;
35
- severity?: "fatal" | "error" | "warning" | "info" | undefined;
36
68
  name?: string | undefined;
37
- stack?: string | undefined;
38
- errorType?: string | undefined;
39
69
  line?: number | undefined;
40
70
  column?: number | undefined;
41
71
  context?: Record<string, unknown> | undefined;
72
+ fingerprint?: string | string[] | undefined;
42
73
  }, {
43
- type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
74
+ type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
44
75
  timestamp: string;
45
- durationMs?: number | undefined;
46
- route?: string | undefined;
47
76
  message?: string | undefined;
77
+ severity?: "info" | "warning" | "error" | "fatal" | undefined;
78
+ errorType?: string | undefined;
79
+ stack?: string | undefined;
80
+ route?: string | undefined;
81
+ traceId?: unknown;
82
+ breadcrumbs?: unknown;
83
+ debugIds?: unknown;
84
+ durationMs?: number | undefined;
48
85
  filename?: string | undefined;
49
86
  id?: string | undefined;
50
- severity?: "fatal" | "error" | "warning" | "info" | undefined;
51
87
  name?: string | undefined;
52
- stack?: string | undefined;
53
- errorType?: string | undefined;
54
88
  line?: number | undefined;
55
89
  column?: number | undefined;
56
90
  context?: Record<string, unknown> | undefined;
91
+ fingerprint?: unknown;
57
92
  }>, "many">;
58
93
  }, "strip", z.ZodTypeAny, {
59
94
  service: string;
60
95
  environment: string;
61
96
  version: 1;
62
97
  events: {
63
- type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
64
98
  message: string;
99
+ type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
65
100
  timestamp: string;
66
- durationMs?: number | undefined;
101
+ severity?: "info" | "warning" | "error" | "fatal" | undefined;
102
+ errorType?: string | undefined;
103
+ stack?: string | undefined;
67
104
  route?: string | undefined;
105
+ traceId?: string | undefined;
106
+ breadcrumbs?: unknown[] | undefined;
107
+ debugIds?: Record<string, unknown> | undefined;
108
+ durationMs?: number | undefined;
68
109
  filename?: string | undefined;
69
110
  id?: string | undefined;
70
- severity?: "fatal" | "error" | "warning" | "info" | undefined;
71
111
  name?: string | undefined;
72
- stack?: string | undefined;
73
- errorType?: string | undefined;
74
112
  line?: number | undefined;
75
113
  column?: number | undefined;
76
114
  context?: Record<string, unknown> | undefined;
115
+ fingerprint?: string | string[] | undefined;
77
116
  }[];
78
117
  release?: string | undefined;
79
118
  sessionId?: string | undefined;
119
+ sdk?: string | undefined;
80
120
  }, {
81
121
  service: string;
82
122
  environment: string;
83
123
  version: 1;
84
124
  events: {
85
- type: "exception" | "message" | "unhandled_rejection" | "session_start" | "track_event" | "outcome" | "request_failure" | "csp_violation" | "timing";
125
+ type: "message" | "exception" | "request_failure" | "timing" | "outcome" | "csp_violation" | "unhandled_rejection" | "session_start" | "track_event";
86
126
  timestamp: string;
87
- durationMs?: number | undefined;
88
- route?: string | undefined;
89
127
  message?: string | undefined;
128
+ severity?: "info" | "warning" | "error" | "fatal" | undefined;
129
+ errorType?: string | undefined;
130
+ stack?: string | undefined;
131
+ route?: string | undefined;
132
+ traceId?: unknown;
133
+ breadcrumbs?: unknown;
134
+ debugIds?: unknown;
135
+ durationMs?: number | undefined;
90
136
  filename?: string | undefined;
91
137
  id?: string | undefined;
92
- severity?: "fatal" | "error" | "warning" | "info" | undefined;
93
138
  name?: string | undefined;
94
- stack?: string | undefined;
95
- errorType?: string | undefined;
96
139
  line?: number | undefined;
97
140
  column?: number | undefined;
98
141
  context?: Record<string, unknown> | undefined;
142
+ fingerprint?: unknown;
99
143
  }[];
100
144
  release?: string | undefined;
101
145
  sessionId?: string | undefined;
146
+ sdk?: unknown;
102
147
  }>;
103
148
  export type BrowserPayload = z.infer<typeof browserPayloadSchema>;
104
149
  export interface NormalizedBrowser {
@@ -1,5 +1,11 @@
1
1
  import { z } from "zod";
2
+ import { liftErrorFields } from "./error-fields.js";
3
+ import { fingerprintOverrideOf } from "./fingerprint-override.js";
4
+ import { sanitizeRuntimeContext } from "./context.js";
2
5
  import { normalizeRoute } from "./fingerprint.js";
6
+ import { scrubText } from "./redact.js";
7
+ import { breadcrumbsField } from "./breadcrumbs.js";
8
+ import { browserDebugIds } from "./source-maps.js";
3
9
  import { asSeverity, } from "./types.js";
4
10
  /**
5
11
  * The compact browser payload (`version: 1`) emitted by
@@ -34,7 +40,32 @@ const browserEventSchema = z.object({
34
40
  column: z.number().int().nonnegative().optional(),
35
41
  /** Path only; query strings are stripped defensively anyway. */
36
42
  route: z.string().max(1000).optional(),
43
+ /**
44
+ * Free-form, scrubbed context. runtime-browser ≥1.4.0 also sends the
45
+ * declared-error keys `autter.error.code|why|fix|link|expected` (from
46
+ * coded errors / `autterErrorFromResponse`) and `autter.request.id` (the
47
+ * `x-request-id` of a failed fetch/XHR). They are validated one by one
48
+ * in normalizeBrowserPayload — a malformed value is dropped on its own
49
+ * instead of rejecting the whole beacon.
50
+ */
37
51
  context: z.record(z.unknown()).optional(),
52
+ /** runtime-browser `/breadcrumbs`: steps before a failure. Validated entry by entry in breadcrumbs.ts; a malformed list is dropped, never the beacon. */
53
+ breadcrumbs: z.array(z.unknown()).optional().catch(undefined),
54
+ /** runtime-browser ≥1.6.0: script URL → debug id for the scripts in this
55
+ * event's stack (EI-13). Malformed values are dropped, never the beacon. */
56
+ debugIds: z.record(z.string().max(1000), z.unknown()).optional().catch(undefined),
57
+ /**
58
+ * Fingerprint override (runtime-browser 1.6.0+): one string or a list.
59
+ * Bounded and scrubbed in fingerprint-override.ts; a malformed value is
60
+ * dropped on its own.
61
+ */
62
+ fingerprint: z.union([z.string().max(2000), z.array(z.string().max(2000)).max(50)]).optional().catch(undefined),
63
+ /**
64
+ * runtime-browser ≥1.6.0: W3C trace id of the request behind the event
65
+ * (the `traceparent` it sent), joining it to the server trace. A malformed
66
+ * id is dropped on its own instead of rejecting the beacon.
67
+ */
68
+ traceId: z.string().regex(/^(?!0{32})[0-9a-f]{32}$/).optional().catch(undefined),
38
69
  });
39
70
  export const browserPayloadSchema = z.object({
40
71
  version: z.literal(1),
@@ -42,6 +73,8 @@ export const browserPayloadSchema = z.object({
42
73
  service: z.string().min(1).max(200),
43
74
  environment: z.string().min(1).max(100),
44
75
  release: z.string().max(200).optional(),
76
+ /** @autter/runtime-browser version (1.4+); recorded for compatibility checks. */
77
+ sdk: z.string().regex(/^\d{1,6}\.\d{1,6}\.\d{1,6}[0-9A-Za-z.+-]{0,20}$/).optional().catch(undefined),
45
78
  events: z.array(browserEventSchema).max(50),
46
79
  });
47
80
  const TYPE_TO_ERROR_TYPE = {
@@ -53,25 +86,14 @@ const TYPE_TO_ERROR_TYPE = {
53
86
  csp_violation: "CspViolation",
54
87
  };
55
88
  // Content-level gate for the free-form `context` bag. The schema whitelist
56
- // above is structural; this masks obvious PII/secrets inside whatever a
57
- // (possibly outdated) SDK still sends: values under sensitive-looking keys
58
- // and email-shaped strings — mirroring redactAttributes() in
59
- // @autter/runtime-node and redactContext() in @autter/runtime-browser.
60
- const SENSITIVE_KEY_RE = /email|pass|token|secret|^auth([-_.]|$)|authorization|bearer|cookie|credential|api[-_.]?key|ssn|cvv|card([-_. ]?(number|num|no))?$/i;
61
- const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
89
+ // above is structural; this masks secrets/PII inside whatever a (possibly
90
+ // outdated) SDK still sends — sensitive keys and secret-shaped values at any
91
+ // depth, via the same sanitiser as OTLP attributes. Message, stack, and route
92
+ // are scrubbed centrally before fingerprinting (server.ts fingerprintAll).
93
+ const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
62
94
  const REDACTED = "[redacted]";
63
95
  function scrubContext(context) {
64
- const out = {};
65
- for (const [key, value] of Object.entries(context)) {
66
- if (value === undefined || value === null)
67
- continue;
68
- out[key] = SENSITIVE_KEY_RE.test(key)
69
- ? REDACTED
70
- : typeof value === "string"
71
- ? value.replace(EMAIL_VALUE_RE, REDACTED)
72
- : value;
73
- }
74
- return out;
96
+ return sanitizeRuntimeContext(Object.fromEntries(Object.entries(context).filter(([, value]) => value !== undefined && value !== null)));
75
97
  }
76
98
  /** Default severity per event type when the SDK doesn't say. */
77
99
  const TYPE_TO_SEVERITY = {
@@ -86,6 +108,7 @@ export function normalizeBrowserPayload(payload) {
86
108
  const occurrences = [];
87
109
  const rollups = new Map();
88
110
  function bumpRollup(route, occurredAt, counts) {
111
+ route = scrubText(route);
89
112
  const bucketAt = new Date(Math.floor(occurredAt.getTime() / 60_000) * 60_000);
90
113
  const key = `${route} ${bucketAt.getTime()}`;
91
114
  const existing = rollups.get(key);
@@ -125,7 +148,7 @@ export function normalizeBrowserPayload(payload) {
125
148
  const name = event.name === "browser.longtask" ? "browser.longtask"
126
149
  : `browser.resource:${normalizeRoute((event.name ?? "").replace(/^browser\.resource:/, "").split("?")[0].replace(EMAIL_VALUE_RE, REDACTED))}`;
127
150
  bumpRollup(name, occurredAt, { requestCount: 1 });
128
- const point = rollups.get(`${name} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`);
151
+ const point = rollups.get(`${scrubText(name)} ${Math.floor(occurredAt.getTime() / 60_000) * 60_000}`);
129
152
  if (point)
130
153
  point.durationSumMs += event.durationMs ?? 0;
131
154
  continue;
@@ -153,7 +176,7 @@ export function normalizeBrowserPayload(payload) {
153
176
  route: event.route ? (event.route.split("?")[0] ?? null) : null,
154
177
  method: null,
155
178
  statusCode: null,
156
- traceId: null,
179
+ traceId: event.traceId ?? null,
157
180
  sessionId: payload.sessionId ?? null,
158
181
  attributes: {
159
182
  ...(event.filename ? { filename: event.filename.split("?")[0] } : {}),
@@ -162,6 +185,10 @@ export function normalizeBrowserPayload(payload) {
162
185
  ...(event.context ? { context: scrubContext(event.context) } : {}),
163
186
  },
164
187
  occurredAt,
188
+ ...liftErrorFields(event.context),
189
+ ...breadcrumbsField(event.breadcrumbs),
190
+ ...browserDebugIds(event.debugIds),
191
+ ...fingerprintOverrideOf(event.fingerprint),
165
192
  });
166
193
  }
167
194
  return { occurrences, metricPoints: [...rollups.values()] };
@@ -1,8 +1,12 @@
1
1
  import { decodeOtlpAttributes, sanitizeRuntimeContext } from "./context.js";
2
+ import { liftErrorFields, validRequestId } from "./error-fields.js";
3
+ import { fingerprintOverrideOf } from "./fingerprint-override.js";
4
+ import { scrubText } from "./redact.js";
2
5
  import { normalizeRoute } from "./fingerprint.js";
3
6
  import { extractLlmCall } from "./llm.js";
7
+ import { BREADCRUMBS_ATTRIBUTE, breadcrumbsField } from "./breadcrumbs.js";
4
8
  import { asSeverity, } from "./types.js";
5
- const EMAIL_VALUE_RE = /[A-Z0-9._%+-]+@[A-Z0-9.-]+\.[A-Z]{2,}/gi;
9
+ const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
6
10
  function attrMap(attributes) {
7
11
  const map = new Map();
8
12
  for (const attr of attributes ?? []) {
@@ -127,7 +131,7 @@ export function normalizeTraces(request) {
127
131
  traceId: span.traceId ?? "",
128
132
  spanId: span.spanId ?? "",
129
133
  parentSpanId: span.parentSpanId ?? null,
130
- name: span.name ?? "unnamed",
134
+ name: scrubText(span.name ?? "unnamed").slice(0, 1000),
131
135
  kind,
132
136
  status: isError ? "error" : "ok",
133
137
  route: route ? normalizeRoute(route) : null,
@@ -167,6 +171,11 @@ export function normalizeTraces(request) {
167
171
  attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
168
172
  ...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }),
169
173
  occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
174
+ // Event attributes first: one span can carry several
175
+ // exceptions, each with its own code.
176
+ ...liftErrorFields(eventAttrs, attrs),
177
+ ...breadcrumbsField(eventAttrs.get(BREADCRUMBS_ATTRIBUTE), attrs.get(BREADCRUMBS_ATTRIBUTE)),
178
+ ...fingerprintOverrideOf(eventAttrs.get("autter.fingerprint") ?? attrs.get("autter.fingerprint")),
170
179
  });
171
180
  }
172
181
  const failedOutcomes = (span.events ?? []).filter((event) => {
@@ -187,6 +196,7 @@ export function normalizeTraces(request) {
187
196
  traceId: span.traceId ?? null, sessionId: null,
188
197
  attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }),
189
198
  occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
199
+ ...liftErrorFields(outcome, attrs),
190
200
  });
191
201
  }
192
202
  if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) {
@@ -209,6 +219,7 @@ export function normalizeTraces(request) {
209
219
  sessionId: null,
210
220
  attributes: decodeOtlpAttributes(span.attributes),
211
221
  occurredAt: startedAt,
222
+ ...liftErrorFields(attrs),
212
223
  });
213
224
  }
214
225
  // LLM provider calls (GenAI semconv, Vercel AI SDK telemetry,
@@ -305,6 +316,11 @@ function inheritHttpFromParent(occurrences, occurrenceSpanIds, spans) {
305
316
  continue;
306
317
  if (!occurrence.route && http.route)
307
318
  occurrence.route = http.route;
319
+ // Request ids live on the request (server) span; child error spans
320
+ // from captureException inherit it so the issue links to the request.
321
+ const requestId = validRequestId(http.attributes?.["autter.request.id"]);
322
+ if (!occurrence.requestId && requestId)
323
+ occurrence.requestId = requestId;
308
324
  const method = methodFromSpan(http);
309
325
  if (!occurrence.method && method)
310
326
  occurrence.method = method;
@@ -0,0 +1,33 @@
1
+ /**
2
+ * Server-side secret/PII scrubbing — the ingester's second line of defence.
3
+ *
4
+ * The SDKs scrub before anything leaves the customer's process, but the
5
+ * ingester also accepts older SDK versions and arbitrary third-party OTLP
6
+ * senders (any OTel SDK in any language) that scrub nothing. Everything this
7
+ * module touches is written to ClickHouse and forwarded to the sink webhook,
8
+ * which is where error context is later read for LLM-generated fixes — so
9
+ * scrubbing here, before storage, covers every downstream consumer.
10
+ *
11
+ * Same patterns as redactText() in @autter/runtime-node; parity is enforced
12
+ * by the shared test-vectors/redaction.json.
13
+ */
14
+ import type { RuntimeOccurrenceInput } from "./types.js";
15
+ /** Keys whose values are masked wholesale (tested case-insensitively). */
16
+ export declare const SENSITIVE_KEY_RE: RegExp;
17
+ /**
18
+ * Deployment-specific patterns on top of the built-ins (env
19
+ * AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of
20
+ * regex sources). Server-side scrubbing has no off switch: it is the
21
+ * backstop for senders that don't scrub.
22
+ */
23
+ export declare function configureRedaction(options: {
24
+ redactValuePatterns?: string[];
25
+ redactKeyPatterns?: string[];
26
+ }): void;
27
+ export declare function isSensitiveKey(key: string): boolean;
28
+ export declare function scrubText(value: string): string;
29
+ /** Nullable convenience for optional columns. */
30
+ export declare function scrubOptional(value: string | null): string | null;
31
+ /** Scrub the free-text fields of an occurrence (message, stack, route,
32
+ * error type). Returns a copy. */
33
+ export declare function scrubOccurrence(occurrence: RuntimeOccurrenceInput): RuntimeOccurrenceInput;
package/dist/redact.js ADDED
@@ -0,0 +1,96 @@
1
+ /**
2
+ * Server-side secret/PII scrubbing — the ingester's second line of defence.
3
+ *
4
+ * The SDKs scrub before anything leaves the customer's process, but the
5
+ * ingester also accepts older SDK versions and arbitrary third-party OTLP
6
+ * senders (any OTel SDK in any language) that scrub nothing. Everything this
7
+ * module touches is written to ClickHouse and forwarded to the sink webhook,
8
+ * which is where error context is later read for LLM-generated fixes — so
9
+ * scrubbing here, before storage, covers every downstream consumer.
10
+ *
11
+ * Same patterns as redactText() in @autter/runtime-node; parity is enforced
12
+ * by the shared test-vectors/redaction.json.
13
+ */
14
+ const MASK = "[redacted]";
15
+ const PRIVATE_KEY_BLOCK_RE = /-----BEGIN [A-Z ]*PRIVATE KEY-----[\s\S]*?(?:-----END [A-Z ]*PRIVATE KEY-----|$)/g;
16
+ const PREFIXED_SECRET_RE = /\b(?:sk-[A-Za-z0-9_-]{20,}|(?:sk|rk)_(?:live|test)_[A-Za-z0-9]{10,}|whsec_[A-Za-z0-9]{10,}|gh[pousr]_[A-Za-z0-9]{20,}|github_pat_[A-Za-z0-9_]{20,}|glpat-[A-Za-z0-9_-]{20,}|xox[abposr]-[A-Za-z0-9-]{10,}|AIza[A-Za-z0-9_-]{30,}|(?:AKIA|ASIA)[0-9A-Z]{16}|npm_[A-Za-z0-9]{36}|autter_(?:rt|pat)_[A-Za-z0-9_-]{10,}|SG\.[A-Za-z0-9_-]{16,}\.[A-Za-z0-9_-]{16,})/g;
17
+ // Lookbehind, not \b: a \b start at every "eyJ" after a "-" made this
18
+ // quadratic ("eyJ-" x 16k took 3.4 s per string).
19
+ const JWT_RE = /(?<![A-Za-z0-9_-])eyJ[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}/g;
20
+ const AUTH_SCHEME_RE = /\b(?:bearer\s+[A-Za-z0-9._~+/=-]{10,}|basic\s+(?=[A-Za-z0-9+/]*[0-9+/=])[A-Za-z0-9+/]{8,}={0,2})/gi;
21
+ const HEADER_VALUE_RE = /((?:^|[^\w-])(?:proxy-)?(?:authorization|(?:set-)?cookie)["']?\s*[:=]\s*["']?)[^"'\r\n]+/gim;
22
+ const SECRET_ASSIGNMENT_RE = /(password|passwd|pwd|passphrase|secret|token|api[_-]?key|apikey|access[_-]?key|private[_-]?key|credential|signature|session[_-]?id|sessionid|ssn)(["']?\s*[:=]\s*)("[^"\r\n]*"|'[^'\r\n]*'|[^\s"'&,;)}\]\[<>]+)/gi;
23
+ const URL_CREDENTIALS_RE = /\b([a-z][a-z0-9+.-]{0,31}:\/\/)(?:[^\s/:@"'<>]*:[^\s/"'<>]*|[^\s/:@"'<>]{16,})@/gi;
24
+ const CARD_RE = /\b(?:4\d{3}|5[1-5]\d{2}|2[2-7]\d{2}|3[47]\d{2}|6(?:011|5\d{2}))(?:[ -]?\d){9,15}\b/g;
25
+ const EMAIL_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
26
+ /** Keys whose values are masked wholesale (tested case-insensitively). */
27
+ export const SENSITIVE_KEY_RE = /password|passwd|passphrase|(^|[._-])pwd$|secret|token|credential|authori[sz]ation|^auth(-|_|$)|bearer|cookie|e-?mail|phone|(^|[^a-z])ssn($|[^a-z])|cvv|card[._-]?number|connection[._-]?string|(^|[._-])dsn$|(api|access|private|client|signing|encryption)[._-]?key|apikey|(^|[._-])session$|^(j|php)?sess(ion)?id$|^sid$|connect\.sid|request[._-]?body|response[._-]?body|headers|url[._-]?query/i;
28
+ let extraValuePatterns = [];
29
+ let extraKeyPatterns = [];
30
+ /**
31
+ * Deployment-specific patterns on top of the built-ins (env
32
+ * AUTTER_REDACT_VALUE_PATTERNS / AUTTER_REDACT_KEY_PATTERNS, JSON arrays of
33
+ * regex sources). Server-side scrubbing has no off switch: it is the
34
+ * backstop for senders that don't scrub.
35
+ */
36
+ export function configureRedaction(options) {
37
+ extraValuePatterns = (options.redactValuePatterns ?? []).map((source) => new RegExp(source, "gi"));
38
+ extraKeyPatterns = (options.redactKeyPatterns ?? []).map((source) => new RegExp(source, "i"));
39
+ }
40
+ export function isSensitiveKey(key) {
41
+ return SENSITIVE_KEY_RE.test(key) || extraKeyPatterns.some((re) => re.test(key));
42
+ }
43
+ function luhnValid(candidate) {
44
+ const digits = candidate.replace(/\D/g, "");
45
+ if (digits.length < 13 || digits.length > 19)
46
+ return false;
47
+ let sum = 0;
48
+ for (let i = 0; i < digits.length; i++) {
49
+ let d = digits.charCodeAt(digits.length - 1 - i) - 48;
50
+ if (i % 2 === 1) {
51
+ d *= 2;
52
+ if (d > 9)
53
+ d -= 9;
54
+ }
55
+ sum += d;
56
+ }
57
+ return sum % 10 === 0;
58
+ }
59
+ /** Scrub secrets/PII embedded in one string; the rest of it survives. */
60
+ /** Longest string scanned (same cap as the Node SDK). Callers that store
61
+ * less cut first; anything longer is truncated rather than stored unscanned. */
62
+ const MAX_SCRUB_CHARS = 64 * 1024;
63
+ export function scrubText(value) {
64
+ if (typeof value !== "string" || value === "")
65
+ return value;
66
+ let out = (value.length > MAX_SCRUB_CHARS ? value.slice(0, MAX_SCRUB_CHARS) : value)
67
+ .replace(PRIVATE_KEY_BLOCK_RE, MASK)
68
+ .replace(PREFIXED_SECRET_RE, MASK)
69
+ .replace(JWT_RE, MASK)
70
+ .replace(AUTH_SCHEME_RE, MASK)
71
+ .replace(HEADER_VALUE_RE, (_m, head) => head + MASK)
72
+ .replace(SECRET_ASSIGNMENT_RE, (_m, key, sep, val) => key + sep + (val[0] === '"' || val[0] === "'" ? val[0] + MASK + val[0] : MASK))
73
+ .replace(URL_CREDENTIALS_RE, (_m, scheme) => `${scheme}${MASK}@`)
74
+ .replace(CARD_RE, (m) => (luhnValid(m) ? MASK : m))
75
+ .replace(EMAIL_RE, MASK);
76
+ for (const re of extraValuePatterns) {
77
+ re.lastIndex = 0;
78
+ out = out.replace(re, MASK);
79
+ }
80
+ return out;
81
+ }
82
+ /** Nullable convenience for optional columns. */
83
+ export function scrubOptional(value) {
84
+ return value === null ? null : scrubText(value);
85
+ }
86
+ /** Scrub the free-text fields of an occurrence (message, stack, route,
87
+ * error type). Returns a copy. */
88
+ export function scrubOccurrence(occurrence) {
89
+ return {
90
+ ...occurrence,
91
+ errorType: scrubText(occurrence.errorType),
92
+ message: scrubText(occurrence.message),
93
+ stack: scrubOptional(occurrence.stack),
94
+ route: scrubOptional(occurrence.route),
95
+ };
96
+ }
@@ -0,0 +1,58 @@
1
+ import type { IngestContext } from "./types.js";
2
+ /** This ingester's own version (package.json — present in the npm package
3
+ * and the Docker image). */
4
+ export declare function ingesterVersion(): string;
5
+ export interface SdkSighting {
6
+ service: string;
7
+ environment: string;
8
+ sdkName: string;
9
+ sdkVersion: string;
10
+ sdkLanguage: string;
11
+ }
12
+ interface KeyValue {
13
+ key?: string;
14
+ value?: {
15
+ stringValue?: string;
16
+ };
17
+ }
18
+ interface ResourceHolder {
19
+ resource?: {
20
+ attributes?: KeyValue[];
21
+ };
22
+ }
23
+ export interface OtlpResourceRequest {
24
+ resourceSpans?: ResourceHolder[];
25
+ resourceMetrics?: ResourceHolder[];
26
+ resourceLogs?: ResourceHolder[];
27
+ }
28
+ /** SDK identity per OTLP resource. Reads only a handful of resource keys. */
29
+ export declare function sdkSightingsFromOtlp(request: OtlpResourceRequest | null | undefined): SdkSighting[];
30
+ export declare const BROWSER_SDK_NAME = "@autter/runtime-browser";
31
+ export interface SdkVersionRow extends SdkSighting {
32
+ ingesterVersion: string;
33
+ seenAt: Date;
34
+ }
35
+ export interface SdkVersionTrackerOptions {
36
+ /** Re-write an unchanged sighting at most this often. Default 1 h. */
37
+ refreshMs?: number;
38
+ /** Dedupe map bound; cleared (not grown) past this. Default 10 000. */
39
+ maxKeys?: number;
40
+ now?: () => number;
41
+ onError?: (err: unknown) => void;
42
+ }
43
+ export declare class SdkVersionTracker {
44
+ private readonly write;
45
+ private readonly lastWritten;
46
+ private readonly refreshMs;
47
+ private readonly maxKeys;
48
+ private readonly now;
49
+ private readonly onError;
50
+ private lastErrorLog;
51
+ constructor(write: (ctx: IngestContext, rows: SdkVersionRow[]) => Promise<void>, options?: SdkVersionTrackerOptions);
52
+ /** Fire-and-forget. Returns the write promise (resolved) for tests. */
53
+ observe(ctx: IngestContext, sightings: SdkSighting[]): Promise<void>;
54
+ }
55
+ export declare function sdkVersionTableDDL(db: string, ttlDays?: number): string;
56
+ /** One row per ingester version (latest report wins); not tenant data. */
57
+ export declare function ingesterInfoTableDDL(db: string): string;
58
+ export {};