@autter/otlp-ingester 1.6.0 → 1.7.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +15 -3
- package/dist/auth.d.ts +1 -1
- package/dist/auth.js +15 -3
- package/dist/clickhouse.js +12 -1
- package/dist/compat-manifest.json +16 -0
- package/dist/config.d.ts +26 -5
- package/dist/config.js +33 -8
- package/dist/normalize-browser.d.ts +26 -26
- package/dist/normalize-otlp.d.ts +6 -0
- package/dist/normalize-otlp.js +131 -84
- package/dist/server.d.ts +7 -0
- package/dist/server.js +78 -30
- package/dist/source-map-mappings.d.ts +57 -0
- package/dist/source-map-mappings.js +245 -0
- package/dist/source-map-scopes.d.ts +71 -0
- package/dist/source-map-scopes.js +190 -0
- package/dist/source-maps-lookup.d.ts +6 -3
- package/dist/source-maps-lookup.js +35 -13
- package/dist/source-maps.d.ts +16 -3
- package/dist/source-maps.js +27 -7
- package/dist/symbolicate.d.ts +46 -8
- package/dist/symbolicate.js +138 -32
- package/dist/trace-dedupe.d.ts +98 -0
- package/dist/trace-dedupe.js +385 -0
- package/dist/types.d.ts +5 -1
- package/package.json +1 -1
package/dist/normalize-otlp.js
CHANGED
|
@@ -5,6 +5,7 @@ import { scrubText } from "./redact.js";
|
|
|
5
5
|
import { normalizeRoute } from "./fingerprint.js";
|
|
6
6
|
import { extractLlmCall } from "./llm.js";
|
|
7
7
|
import { BREADCRUMBS_ATTRIBUTE, breadcrumbsField } from "./breadcrumbs.js";
|
|
8
|
+
import { dedupeTraceSignals, hasAutterKey, hasNextContext } from "./trace-dedupe.js";
|
|
8
9
|
import { asSeverity, } from "./types.js";
|
|
9
10
|
const EMAIL_VALUE_RE = /[A-Z0-9._%+-]{1,64}@[A-Z0-9.-]{1,253}\.[A-Z]{2,63}/gi;
|
|
10
11
|
function attrMap(attributes) {
|
|
@@ -100,8 +101,10 @@ function severityOf(eventAttrs, spanAttrs) {
|
|
|
100
101
|
}
|
|
101
102
|
const MAX_SPANS_PER_REQUEST = 5000;
|
|
102
103
|
export function normalizeTraces(request) {
|
|
103
|
-
const
|
|
104
|
-
|
|
104
|
+
const signals = [];
|
|
105
|
+
// Per signal: whether a server span's request rollup already counts it,
|
|
106
|
+
// and the route its own rollup keys on (before HTTP inheritance).
|
|
107
|
+
const signalRollups = [];
|
|
105
108
|
const spans = [];
|
|
106
109
|
const llmCalls = [];
|
|
107
110
|
const rollups = new Map();
|
|
@@ -143,39 +146,56 @@ export function normalizeTraces(request) {
|
|
|
143
146
|
},
|
|
144
147
|
startedAt,
|
|
145
148
|
});
|
|
146
|
-
// Error
|
|
149
|
+
// Error signals: one per exception event; if the span is
|
|
147
150
|
// errored without exception events, one from the span status.
|
|
148
|
-
|
|
151
|
+
// They become occurrences after the batch-wide dedupe below.
|
|
152
|
+
const traceId = span.traceId ?? "";
|
|
153
|
+
const spanId = span.spanId ?? "";
|
|
154
|
+
const rollup = {
|
|
155
|
+
onRequestRollup: kind === "server" && attrs.get("autter.tail_retained") !== "true",
|
|
156
|
+
route: route ?? "",
|
|
157
|
+
};
|
|
158
|
+
const spanAutter = attrs.has("autter.severity");
|
|
149
159
|
const exceptionEvents = (span.events ?? []).filter((event) => event.name === "exception");
|
|
150
160
|
for (const event of exceptionEvents) {
|
|
151
161
|
const eventAttrs = attrMap(event.attributes);
|
|
152
162
|
const handled = eventAttrs.get("autter.handled") === "true";
|
|
153
|
-
|
|
154
|
-
|
|
155
|
-
|
|
156
|
-
|
|
157
|
-
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
|
|
178
|
-
|
|
163
|
+
const eventAutter = hasAutterKey(eventAttrs);
|
|
164
|
+
signalRollups.push(rollup);
|
|
165
|
+
signals.push({
|
|
166
|
+
traceId,
|
|
167
|
+
spanId,
|
|
168
|
+
signal: "exception",
|
|
169
|
+
autterCapture: eventAutter || spanAutter,
|
|
170
|
+
eventAutter,
|
|
171
|
+
nextContext: hasNextContext(eventAttrs, attrs),
|
|
172
|
+
handled,
|
|
173
|
+
occurrence: {
|
|
174
|
+
source: "server",
|
|
175
|
+
severity: severityOf(eventAttrs, attrs),
|
|
176
|
+
service: resource.service,
|
|
177
|
+
environment: resource.environment,
|
|
178
|
+
release: resource.release,
|
|
179
|
+
errorType: eventAttrs.get("exception.type") ?? "Error",
|
|
180
|
+
message: eventAttrs.get("exception.message") ??
|
|
181
|
+
span.status?.message ??
|
|
182
|
+
span.name ??
|
|
183
|
+
"Unknown error",
|
|
184
|
+
stack: eventAttrs.get("exception.stacktrace") ?? null,
|
|
185
|
+
route,
|
|
186
|
+
method: methodOf(attrs),
|
|
187
|
+
statusCode,
|
|
188
|
+
traceId: span.traceId ?? null,
|
|
189
|
+
sessionId: null,
|
|
190
|
+
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes),
|
|
191
|
+
...(handled ? { "autter.handled": true, "autter.sampled": eventAttrs.get("autter.sampled") === "true" } : {}) }),
|
|
192
|
+
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
193
|
+
// Event attributes first: one span can carry several
|
|
194
|
+
// exceptions, each with its own code.
|
|
195
|
+
...liftErrorFields(eventAttrs, attrs),
|
|
196
|
+
...breadcrumbsField(eventAttrs.get(BREADCRUMBS_ATTRIBUTE), attrs.get(BREADCRUMBS_ATTRIBUTE)),
|
|
197
|
+
...fingerprintOverrideOf(eventAttrs.get("autter.fingerprint") ?? attrs.get("autter.fingerprint")),
|
|
198
|
+
},
|
|
179
199
|
});
|
|
180
200
|
}
|
|
181
201
|
const failedOutcomes = (span.events ?? []).filter((event) => {
|
|
@@ -186,40 +206,58 @@ export function normalizeTraces(request) {
|
|
|
186
206
|
for (const event of failedOutcomes) {
|
|
187
207
|
const outcome = attrMap(event.attributes);
|
|
188
208
|
const name = (outcome.get("autter.outcome.name") ?? "operation").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 200);
|
|
189
|
-
|
|
190
|
-
|
|
191
|
-
|
|
192
|
-
|
|
193
|
-
|
|
194
|
-
|
|
195
|
-
|
|
196
|
-
|
|
197
|
-
|
|
198
|
-
|
|
199
|
-
|
|
209
|
+
signalRollups.push(rollup);
|
|
210
|
+
signals.push({
|
|
211
|
+
traceId,
|
|
212
|
+
spanId,
|
|
213
|
+
signal: "outcome",
|
|
214
|
+
autterCapture: true,
|
|
215
|
+
eventAutter: true,
|
|
216
|
+
nextContext: hasNextContext(outcome, attrs),
|
|
217
|
+
handled: false,
|
|
218
|
+
occurrence: {
|
|
219
|
+
source: "server", severity: "error", service: resource.service,
|
|
220
|
+
environment: resource.environment, release: resource.release,
|
|
221
|
+
errorType: "OutcomeFailure",
|
|
222
|
+
message: `${name}: ${(outcome.get("autter.outcome.message") ?? "failed").replace(EMAIL_VALUE_RE, "[redacted]").slice(0, 1000)}`,
|
|
223
|
+
stack: String(decodeOtlpAttributes(event.attributes)["autter.outcome.stack"] ?? "") || null, route, method: methodOf(attrs), statusCode,
|
|
224
|
+
traceId: span.traceId ?? null, sessionId: null,
|
|
225
|
+
attributes: sanitizeRuntimeContext({ ...decodeOtlpAttributes(span.attributes), ...decodeOtlpAttributes(event.attributes) }),
|
|
226
|
+
occurredAt: nanosToDate(event.timeUnixNano ?? span.startTimeUnixNano),
|
|
227
|
+
...liftErrorFields(outcome, attrs),
|
|
228
|
+
},
|
|
200
229
|
});
|
|
201
230
|
}
|
|
202
231
|
if (exceptionEvents.length === 0 && failedOutcomes.length === 0 && isError) {
|
|
203
|
-
|
|
204
|
-
|
|
205
|
-
|
|
206
|
-
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
211
|
-
|
|
212
|
-
|
|
213
|
-
:
|
|
214
|
-
|
|
215
|
-
|
|
216
|
-
|
|
217
|
-
|
|
218
|
-
|
|
219
|
-
|
|
220
|
-
|
|
221
|
-
|
|
222
|
-
|
|
232
|
+
signalRollups.push(rollup);
|
|
233
|
+
signals.push({
|
|
234
|
+
traceId,
|
|
235
|
+
spanId,
|
|
236
|
+
signal: "status",
|
|
237
|
+
autterCapture: spanAutter,
|
|
238
|
+
eventAutter: false,
|
|
239
|
+
nextContext: hasNextContext(attrs),
|
|
240
|
+
handled: false,
|
|
241
|
+
occurrence: {
|
|
242
|
+
source: "server",
|
|
243
|
+
severity: severityOf(new Map(), attrs),
|
|
244
|
+
service: resource.service,
|
|
245
|
+
environment: resource.environment,
|
|
246
|
+
release: resource.release,
|
|
247
|
+
errorType: statusCode !== null && statusCode >= 500 ? "HttpServerError" : "SpanError",
|
|
248
|
+
message: span.status?.message || (statusCode !== null && statusCode >= 500
|
|
249
|
+
? `${methodOf(attrs) ?? "HTTP"} ${route ?? span.name ?? "request"} returned ${statusCode}`
|
|
250
|
+
: `${span.name ?? "span"} failed`),
|
|
251
|
+
stack: null,
|
|
252
|
+
route,
|
|
253
|
+
method: methodOf(attrs),
|
|
254
|
+
statusCode,
|
|
255
|
+
traceId: span.traceId ?? null,
|
|
256
|
+
sessionId: null,
|
|
257
|
+
attributes: decodeOtlpAttributes(span.attributes),
|
|
258
|
+
occurredAt: startedAt,
|
|
259
|
+
...liftErrorFields(attrs),
|
|
260
|
+
},
|
|
223
261
|
});
|
|
224
262
|
}
|
|
225
263
|
// LLM provider calls (GenAI semconv, Vercel AI SDK telemetry,
|
|
@@ -249,7 +287,7 @@ export function normalizeTraces(request) {
|
|
|
249
287
|
// ships those at ~100% alongside a metrics pipeline that already
|
|
250
288
|
// counts every request, so folding them in would double-count
|
|
251
289
|
// erroring routes.
|
|
252
|
-
if (
|
|
290
|
+
if (rollup.onRequestRollup) {
|
|
253
291
|
addToRollup(rollups, {
|
|
254
292
|
service: resource.service,
|
|
255
293
|
environment: resource.environment,
|
|
@@ -262,33 +300,42 @@ export function normalizeTraces(request) {
|
|
|
262
300
|
sessionCount: 0,
|
|
263
301
|
});
|
|
264
302
|
}
|
|
265
|
-
else {
|
|
266
|
-
// Occurrences on non-server spans (captureException's internal
|
|
267
|
-
// error spans, workers, consumers) have no request rollup to
|
|
268
|
-
// ride — count each as one event / one error event, or a
|
|
269
|
-
// service whose errors arrive outside HTTP handlers groups
|
|
270
|
-
// issues while every event counter stays 0. Server spans are
|
|
271
|
-
// excluded: their request rollup above already represents them.
|
|
272
|
-
for (const occ of occurrences.slice(occurrencesBefore)) {
|
|
273
|
-
addToRollup(rollups, {
|
|
274
|
-
service: occ.service,
|
|
275
|
-
environment: occ.environment,
|
|
276
|
-
release: occ.release,
|
|
277
|
-
route: occ.route ?? "",
|
|
278
|
-
bucketAt: minuteBucket(occ.occurredAt),
|
|
279
|
-
requestCount: 1,
|
|
280
|
-
errorCount: occ.severity === "fatal" || occ.severity === "error" ? 1 : 0,
|
|
281
|
-
durationSumMs: 0,
|
|
282
|
-
sessionCount: 0,
|
|
283
|
-
});
|
|
284
|
-
}
|
|
285
|
-
}
|
|
286
303
|
}
|
|
287
304
|
}
|
|
288
305
|
}
|
|
289
|
-
|
|
306
|
+
// One occurrence per thrown error: recordings of one error on several
|
|
307
|
+
// spans of a trace merge, and status-only occurrences an error on their
|
|
308
|
+
// span chain explains are dropped (trace-dedupe.ts).
|
|
309
|
+
const deduped = dedupeTraceSignals(signals, spans);
|
|
310
|
+
const occurrences = deduped.occurrences;
|
|
311
|
+
// Occurrences on non-server spans (captureException's internal error
|
|
312
|
+
// spans, workers, consumers) have no request rollup to ride — count each
|
|
313
|
+
// as one event / one error event, or a service whose errors arrive
|
|
314
|
+
// outside HTTP handlers groups issues while every event counter stays 0.
|
|
315
|
+
// Server spans are excluded: their request rollup above already
|
|
316
|
+
// represents them, and so does it for a merged error any of whose
|
|
317
|
+
// recordings sits on one. Counted after the dedupe, so each thrown error
|
|
318
|
+
// is one event; a dropped status-only occurrence is none.
|
|
319
|
+
deduped.members.forEach((members, i) => {
|
|
320
|
+
if (members.some((m) => signalRollups[m].onRequestRollup))
|
|
321
|
+
return;
|
|
322
|
+
const occ = occurrences[i];
|
|
323
|
+
addToRollup(rollups, {
|
|
324
|
+
service: occ.service,
|
|
325
|
+
environment: occ.environment,
|
|
326
|
+
release: occ.release,
|
|
327
|
+
route: signalRollups[deduped.sourceIndexes[i]].route,
|
|
328
|
+
bucketAt: minuteBucket(occ.occurredAt),
|
|
329
|
+
requestCount: 1,
|
|
330
|
+
errorCount: occ.severity === "fatal" || occ.severity === "error" ? 1 : 0,
|
|
331
|
+
durationSumMs: 0,
|
|
332
|
+
sessionCount: 0,
|
|
333
|
+
});
|
|
334
|
+
});
|
|
335
|
+
inheritHttpFromParent(occurrences, deduped.sourceIndexes.map((index) => signals[index].spanId), spans);
|
|
290
336
|
return {
|
|
291
337
|
occurrences,
|
|
338
|
+
occurrenceBatchIndexes: deduped.sourceIndexes,
|
|
292
339
|
spans,
|
|
293
340
|
metricPoints: [...rollups.values()],
|
|
294
341
|
llmCalls,
|
package/dist/server.d.ts
CHANGED
|
@@ -11,4 +11,11 @@ export interface IngesterApp {
|
|
|
11
11
|
* Autter dashboard). Called after the boot schema warm-up; throttled. */
|
|
12
12
|
reportIngesterInfo: () => Promise<void>;
|
|
13
13
|
}
|
|
14
|
+
/** The only routes an upload key (`autter_rtu_…`, scope "upload") may call. */
|
|
15
|
+
export declare const UPLOAD_KEY_ROUTES: readonly ["/v1/sourcemaps", "/v1/sourcemaps/check"];
|
|
16
|
+
/** The 403 an upload key gets on every other authenticated route. */
|
|
17
|
+
export declare const UPLOAD_KEY_NOT_ALLOWED: {
|
|
18
|
+
readonly error: "upload_key_not_allowed";
|
|
19
|
+
readonly detail: string;
|
|
20
|
+
};
|
|
14
21
|
export declare function createIngesterApp(config: IngesterConfig): IngesterApp;
|
package/dist/server.js
CHANGED
|
@@ -13,12 +13,20 @@ import { decodeMetricsRequest, decodeTraceRequest, decodeLogsRequest } from "./o
|
|
|
13
13
|
import { normalizeLogs } from "./logs.js";
|
|
14
14
|
import { decodeProfile } from "./profiles.js";
|
|
15
15
|
import { normalizeMemoryMetrics, normalizePlatformEvent, platformEventSchema } from "./memory.js";
|
|
16
|
-
import { validateSourceMap, validateSourceMapCheck } from "./source-maps.js";
|
|
16
|
+
import { SOURCE_MAP_MAX_BODY_BYTES, validateSourceMap, validateSourceMapCheck } from "./source-maps.js";
|
|
17
17
|
import { SinkForwarder } from "./sink.js";
|
|
18
18
|
import { configureRedaction, scrubOccurrence } from "./redact.js";
|
|
19
19
|
import { buildCompatReport, evaluateCompat, INGESTER_VERSION_HEADER, } from "./compat.js";
|
|
20
20
|
import { MIGRATIONS } from "./migrations.js";
|
|
21
21
|
import { BROWSER_SDK_NAME, ingesterVersion, SdkVersionTracker, sdkSightingsFromOtlp, } from "./sdk-versions.js";
|
|
22
|
+
/** The only routes an upload key (`autter_rtu_…`, scope "upload") may call. */
|
|
23
|
+
export const UPLOAD_KEY_ROUTES = ["/v1/sourcemaps", "/v1/sourcemaps/check"];
|
|
24
|
+
/** The 403 an upload key gets on every other authenticated route. */
|
|
25
|
+
export const UPLOAD_KEY_NOT_ALLOWED = {
|
|
26
|
+
error: "upload_key_not_allowed",
|
|
27
|
+
detail: "autter_rtu_ keys can only upload source maps (POST /v1/sourcemaps and /v1/sourcemaps/check); " +
|
|
28
|
+
"telemetry needs a server key (autter_rt_…), or a client key (autter_rtc_…) for /v1/browser",
|
|
29
|
+
};
|
|
22
30
|
/** Re-report the ingester version at most this often (keeps the row inside
|
|
23
31
|
* its TTL for long-running ingesters). */
|
|
24
32
|
const INGESTER_INFO_INTERVAL_MS = 6 * 60 * 60 * 1000;
|
|
@@ -84,7 +92,40 @@ export function createIngesterApp(config) {
|
|
|
84
92
|
next();
|
|
85
93
|
});
|
|
86
94
|
app.use(express.raw({ limit: "1mb", type: ["application/x-pprof"] }));
|
|
87
|
-
|
|
95
|
+
// A check names at most 500 maps: well under 1 MB, whatever MAX_BODY_BYTES is.
|
|
96
|
+
app.use("/v1/sourcemaps/check", express.json({ limit: "1mb" }));
|
|
97
|
+
// Source map uploads (up to 32 MiB of JSON, usually gzipped) are routed
|
|
98
|
+
// before the small-body parsers below, and read their body only once the
|
|
99
|
+
// key is checked: an anonymous client can't make the ingester inflate and
|
|
100
|
+
// parse one. body-parser inflates gzip/deflate and applies the limit to
|
|
101
|
+
// the inflated bytes, so a small compressed body can't expand past it.
|
|
102
|
+
const sourceMapBody = express.json({ limit: SOURCE_MAP_MAX_BODY_BYTES });
|
|
103
|
+
app.post("/v1/sourcemaps", async (req, res, next) => {
|
|
104
|
+
// On failure the body is never read (Node discards it).
|
|
105
|
+
const ctx = await authenticate(req, res, "sourcemaps");
|
|
106
|
+
if (!ctx)
|
|
107
|
+
return;
|
|
108
|
+
sourceMapBody(req, res, async (err) => {
|
|
109
|
+
if (err) {
|
|
110
|
+
next(err);
|
|
111
|
+
return;
|
|
112
|
+
}
|
|
113
|
+
const sourceMap = validateSourceMap(req.body);
|
|
114
|
+
if (!sourceMap) {
|
|
115
|
+
res.status(400).json({ error: "invalid source map" });
|
|
116
|
+
return;
|
|
117
|
+
}
|
|
118
|
+
// Idempotent: the same release + path replaces the earlier row
|
|
119
|
+
// (ReplacingMergeTree), so CI retries and re-runs are harmless.
|
|
120
|
+
try {
|
|
121
|
+
await store.insertSourceMap(ctx, sourceMap);
|
|
122
|
+
res.status(202).json({ accepted: true });
|
|
123
|
+
}
|
|
124
|
+
catch (storeErr) {
|
|
125
|
+
storageError(res, storeErr);
|
|
126
|
+
}
|
|
127
|
+
});
|
|
128
|
+
});
|
|
88
129
|
app.use(express.json({
|
|
89
130
|
limit: config.maxBodyBytes,
|
|
90
131
|
type: ["application/json"],
|
|
@@ -164,7 +205,12 @@ export function createIngesterApp(config) {
|
|
|
164
205
|
res.setHeader("cache-control", "no-store");
|
|
165
206
|
res.status(200).json(body);
|
|
166
207
|
});
|
|
167
|
-
/**
|
|
208
|
+
/**
|
|
209
|
+
* Auth + scope + rate limit; returns null (response sent) on failure.
|
|
210
|
+
* Server keys pass everywhere; client keys only on "browser"; upload
|
|
211
|
+
* keys only on "sourcemaps". A new route that passes "otlp" is closed to
|
|
212
|
+
* client and upload keys without further changes.
|
|
213
|
+
*/
|
|
168
214
|
async function authenticate(req, res, surface) {
|
|
169
215
|
// A storage-less ingester must refuse, not accept-and-drop: exporters
|
|
170
216
|
// retry on 503, so telemetry survives a misconfigured deploy.
|
|
@@ -182,12 +228,22 @@ export function createIngesterApp(config) {
|
|
|
182
228
|
res.status(401).json({ error: "invalid ingest key" });
|
|
183
229
|
return null;
|
|
184
230
|
}
|
|
185
|
-
if (ctx.scope === "
|
|
231
|
+
if (ctx.scope === "upload") {
|
|
232
|
+
// CI keys: source map upload and check only. Server rate window
|
|
233
|
+
// (each key has its own).
|
|
234
|
+
if (surface !== "sourcemaps") {
|
|
235
|
+
res.status(403).json(UPLOAD_KEY_NOT_ALLOWED);
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
}
|
|
239
|
+
else if (ctx.scope === "client") {
|
|
186
240
|
// Publishable keys: browser surface only, origin allow-list, and
|
|
187
241
|
// the tighter rate window.
|
|
188
242
|
if (surface !== "browser") {
|
|
189
243
|
res.status(403).json({
|
|
190
|
-
error:
|
|
244
|
+
error: surface === "sourcemaps"
|
|
245
|
+
? "client keys cannot upload source maps — use an upload key (autter_rtu_…) or a server key"
|
|
246
|
+
: "client keys cannot send OTLP — use a server key",
|
|
191
247
|
});
|
|
192
248
|
return null;
|
|
193
249
|
}
|
|
@@ -203,6 +259,12 @@ export function createIngesterApp(config) {
|
|
|
203
259
|
}
|
|
204
260
|
return ctx;
|
|
205
261
|
}
|
|
262
|
+
else if (ctx.scope !== "server") {
|
|
263
|
+
// KeyResolver never yields another scope; refuse rather than
|
|
264
|
+
// treat an unknown one as a server key.
|
|
265
|
+
res.status(403).json({ error: "key scope not allowed" });
|
|
266
|
+
return null;
|
|
267
|
+
}
|
|
206
268
|
if (!serverRateLimiter.allow(key)) {
|
|
207
269
|
res.status(429).json({ error: "rate limit exceeded" });
|
|
208
270
|
return null;
|
|
@@ -242,8 +304,13 @@ export function createIngesterApp(config) {
|
|
|
242
304
|
* `fingerprint` / `fingerprintScheme` stay the legacy identity
|
|
243
305
|
* (message-v1 or code-v1) byte-for-byte; `fingerprints` carries every
|
|
244
306
|
* candidate (manual-v1, code-v1, frames-v3, message-v1) for the platform
|
|
245
|
-
* to resolve the issue from.
|
|
246
|
-
|
|
307
|
+
* to resolve the issue from.
|
|
308
|
+
*
|
|
309
|
+
* `batchIndexes` (traces): each occurrence's position among the batch's
|
|
310
|
+
* signals before the per-trace dedupe, so a merged occurrence keeps the
|
|
311
|
+
* id of the recording it kept and a drop never shifts another's id.
|
|
312
|
+
* Defaults to the position in `inputs`. */
|
|
313
|
+
async function fingerprintAll(ctx, inputs, batchIndexes) {
|
|
247
314
|
return Promise.all(inputs.map(async (raw, index) => {
|
|
248
315
|
// Second line of defence, before fingerprinting, storage, and the
|
|
249
316
|
// sink (which feeds LLM fix generation): scrub secrets from the
|
|
@@ -258,7 +325,7 @@ export function createIngesterApp(config) {
|
|
|
258
325
|
const { debugIds: _debugIds, ...stored } = input;
|
|
259
326
|
return {
|
|
260
327
|
...stored,
|
|
261
|
-
occurrenceId: occurrenceIdFor(ctx, input, fingerprint, index),
|
|
328
|
+
occurrenceId: occurrenceIdFor(ctx, input, fingerprint, batchIndexes?.[index] ?? index),
|
|
262
329
|
fingerprint,
|
|
263
330
|
fingerprintScheme: scheme,
|
|
264
331
|
...deriveFields(input),
|
|
@@ -341,29 +408,10 @@ export function createIngesterApp(config) {
|
|
|
341
408
|
storageError(res, err);
|
|
342
409
|
}
|
|
343
410
|
});
|
|
344
|
-
app.post("/v1/sourcemaps", async (req, res) => {
|
|
345
|
-
const ctx = await authenticate(req, res, "otlp");
|
|
346
|
-
if (!ctx)
|
|
347
|
-
return;
|
|
348
|
-
const sourceMap = validateSourceMap(req.body);
|
|
349
|
-
if (!sourceMap) {
|
|
350
|
-
res.status(400).json({ error: "invalid source map" });
|
|
351
|
-
return;
|
|
352
|
-
}
|
|
353
|
-
// Idempotent: the same release + path replaces the earlier row
|
|
354
|
-
// (ReplacingMergeTree), so CI retries and re-runs are harmless.
|
|
355
|
-
try {
|
|
356
|
-
await store.insertSourceMap(ctx, sourceMap);
|
|
357
|
-
res.status(202).json({ accepted: true });
|
|
358
|
-
}
|
|
359
|
-
catch (err) {
|
|
360
|
-
storageError(res, err);
|
|
361
|
-
}
|
|
362
|
-
});
|
|
363
411
|
/** Which of these maps does this tenant already have? Lets
|
|
364
412
|
* `autter-runtime sourcemaps upload` skip unchanged maps (1.6.0+). */
|
|
365
413
|
app.post("/v1/sourcemaps/check", async (req, res) => {
|
|
366
|
-
const ctx = await authenticate(req, res, "
|
|
414
|
+
const ctx = await authenticate(req, res, "sourcemaps");
|
|
367
415
|
if (!ctx)
|
|
368
416
|
return;
|
|
369
417
|
const check = validateSourceMapCheck(req.body);
|
|
@@ -418,8 +466,8 @@ export function createIngesterApp(config) {
|
|
|
418
466
|
else {
|
|
419
467
|
request = req.body;
|
|
420
468
|
}
|
|
421
|
-
const { occurrences, spans, metricPoints, llmCalls } = normalizeTraces(request);
|
|
422
|
-
const fingerprinted = await fingerprintAll(ctx, occurrences);
|
|
469
|
+
const { occurrences, occurrenceBatchIndexes, spans, metricPoints, llmCalls } = normalizeTraces(request);
|
|
470
|
+
const fingerprinted = await fingerprintAll(ctx, occurrences, occurrenceBatchIndexes);
|
|
423
471
|
// ClickHouse has no cross-table transaction, so these four inserts can
|
|
424
472
|
// partially commit. Recovery boundary: any failure → 503 → the exporter
|
|
425
473
|
// retries the whole batch. Deterministic occurrence ids make the retry
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Position lookups in a source map's `mappings`, without the memory of a
|
|
3
|
+
* general-purpose decoder.
|
|
4
|
+
*
|
|
5
|
+
* @jridgewell/trace-mapping decodes `mappings` into one small array per
|
|
6
|
+
* segment: about 14 bytes of heap per byte of JSON (a 5 MB map takes 67 MB).
|
|
7
|
+
* Symbolication only needs "which original position is at or before this
|
|
8
|
+
* generated one", so a map that is kept in the cache is decoded into one
|
|
9
|
+
* Int32Array (5 ints per segment: about 3 bytes per byte of JSON), and a map
|
|
10
|
+
* too big to keep is not decoded at all: its frames are resolved in one
|
|
11
|
+
* streaming pass over the string.
|
|
12
|
+
*
|
|
13
|
+
* Lookup semantics match trace-mapping's `originalPositionFor` (greatest
|
|
14
|
+
* lower bound): the first segment that starts at the frame's column, else
|
|
15
|
+
* the last segment before it on the same line; a segment without a source
|
|
16
|
+
* maps to nothing.
|
|
17
|
+
*/
|
|
18
|
+
/** An original position. Lines and columns are 0-based; `name` is -1 when the segment has none. */
|
|
19
|
+
export interface MappedSegment {
|
|
20
|
+
source: number;
|
|
21
|
+
line: number;
|
|
22
|
+
column: number;
|
|
23
|
+
name: number;
|
|
24
|
+
}
|
|
25
|
+
/** A 0-based generated (line, column). */
|
|
26
|
+
export interface GeneratedPosition {
|
|
27
|
+
line: number;
|
|
28
|
+
column: number;
|
|
29
|
+
}
|
|
30
|
+
/**
|
|
31
|
+
* Walk a mappings-style string — `;` between generated lines, `,` between
|
|
32
|
+
* groups of base64 VLQ values — calling `visit(line, values, count)` with
|
|
33
|
+
* each group's raw (relative) values. `values` is reused between calls and
|
|
34
|
+
* holds the first 6; `count` is how many the group had. Throws on
|
|
35
|
+
* malformed VLQ. Also reads `x_autter_scopes` (source-map-scopes.ts).
|
|
36
|
+
*/
|
|
37
|
+
export declare function forEachVlqGroup(text: string, visit: (line: number, values: readonly number[], count: number) => void): void;
|
|
38
|
+
/** Heap a DecodedMappings of `mappings` would take, without decoding it. */
|
|
39
|
+
export declare function decodedMappingsBytes(mappings: string): number;
|
|
40
|
+
export declare class DecodedMappings {
|
|
41
|
+
/** FIELDS ints per segment, each line's segments sorted by generated column. */
|
|
42
|
+
private readonly segments;
|
|
43
|
+
/** Index of each line's first segment; one extra entry closes the last line. */
|
|
44
|
+
private readonly lineStarts;
|
|
45
|
+
private constructor();
|
|
46
|
+
static decode(mappings: string): DecodedMappings;
|
|
47
|
+
get bytes(): number;
|
|
48
|
+
/** Generators almost always emit a line's segments in column order; sort the rare line that isn't (stable, as trace-mapping does). */
|
|
49
|
+
private sortLine;
|
|
50
|
+
/** The original position at a 0-based generated (line, column), or null. */
|
|
51
|
+
find(line: number, column: number): MappedSegment | null;
|
|
52
|
+
}
|
|
53
|
+
/**
|
|
54
|
+
* Resolve several positions in one pass over `mappings` without decoding
|
|
55
|
+
* it — for maps too big to keep. Same results as DecodedMappings.find.
|
|
56
|
+
*/
|
|
57
|
+
export declare function findInMappings(mappings: string, positions: readonly GeneratedPosition[]): Array<MappedSegment | null>;
|