@obsunified/collector 2.3.0 → 2.4.4
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +7 -3
- package/dist/auth.d.ts +2 -1
- package/dist/auth.js +1 -1
- package/dist/{chunk-2FALTDCE.js → chunk-VSOZJO6Y.js} +12 -1
- package/dist/chunk-VSOZJO6Y.js.map +1 -0
- package/dist/index.d.ts +271 -62
- package/dist/index.js +1077 -137
- package/dist/index.js.map +1 -1
- package/dist/{ingest-auth-BmP5-f96.d.ts → ingest-auth-BTowieER.d.ts} +66 -2
- package/package.json +3 -3
- package/src/framework/collector.ts +22 -0
- package/src/framework/env.ts +9 -1
- package/src/framework/ingest-auth-warning.test.ts +71 -0
- package/src/index.ts +15 -0
- package/src/lib/alert-email.ts +104 -0
- package/src/lib/alerts-store.test.ts +147 -2
- package/src/lib/alerts-store.ts +438 -63
- package/src/lib/metric-alerts.test.ts +769 -0
- package/src/lib/metric-alerts.ts +640 -0
- package/src/lib/metrics-store.ts +9 -2
- package/src/lib/projects-store.test.ts +44 -1
- package/src/lib/projects-store.ts +21 -0
- package/src/lib/sql-db-postgres.integration.test.ts +247 -0
- package/src/lib/store/trace-queries.ts +1 -1
- package/src/lib/webhook-url.test.ts +49 -0
- package/src/lib/webhook-url.ts +72 -14
- package/src/migrations/054_alert_rules_pending_secs.sql +11 -0
- package/src/migrations/055_alert_state_pending_since.sql +12 -0
- package/src/migrations/056_metric_point_series_received.sql +9 -0
- package/src/migrations/057_alert_evaluations_outcome.sql +11 -0
- package/src/migrations/058_alert_state_pending_seen_at.sql +12 -0
- package/src/migrations/059_alert_evaluations_error.sql +11 -0
- package/src/migrations-postgres/054_alert_rules_pending_secs.sql +7 -0
- package/src/migrations-postgres/055_alert_state_pending_since.sql +6 -0
- package/src/migrations-postgres/056_metric_point_series_received.sql +7 -0
- package/src/migrations-postgres/057_alert_evaluations_outcome.sql +6 -0
- package/src/migrations-postgres/058_alert_state_pending_seen_at.sql +6 -0
- package/src/migrations-postgres/059_alert_evaluations_error.sql +6 -0
- package/src/migrations-postgres/README.md +13 -0
- package/src/otlp/decode/logs.ts +30 -2
- package/src/otlp/decode/metrics.ts +67 -2
- package/src/otlp/decode/traces.ts +57 -4
- package/src/otlp/decode/values.ts +64 -10
- package/src/otlp/decode.test.ts +305 -0
- package/src/otlp/decode.ts +9 -3
- package/src/plugins/alerts-evaluator-pending.test.ts +322 -0
- package/src/plugins/alerts-evaluator-retention.test.ts +2 -0
- package/src/plugins/alerts-evaluator.test.ts +10 -2
- package/src/plugins/alerts-evaluator.ts +239 -40
- package/src/plugins/alerts-routes.ts +17 -2
- package/src/plugins/autosre-routes.ts +9 -0
- package/src/plugins/logs-receiver.ts +8 -8
- package/src/plugins/metrics-receiver.ts +10 -9
- package/src/plugins/otlp-receiver.ts +10 -26
- package/src/plugins/profile-routes.test.ts +22 -0
- package/src/plugins/profile-routes.ts +11 -2
- package/src/plugins/replay-query-routes.test.ts +63 -1
- package/src/plugins/replay-query-routes.ts +40 -9
- package/dist/chunk-2FALTDCE.js.map +0 -1
package/src/otlp/decode/logs.ts
CHANGED
|
@@ -41,7 +41,26 @@ export interface DecodedLogRecord {
|
|
|
41
41
|
* storage. Resource `service.name` and scope name are denormalized onto each
|
|
42
42
|
* record so downstream code doesn't need to traverse the hierarchy.
|
|
43
43
|
*/
|
|
44
|
-
export const decodeLogsRequest = (body: ReadBodyResult): DecodedLogRecord[] =>
|
|
44
|
+
export const decodeLogsRequest = (body: ReadBodyResult): DecodedLogRecord[] =>
|
|
45
|
+
decodeLogsRequestCapped(body, Number.POSITIVE_INFINITY).records;
|
|
46
|
+
|
|
47
|
+
export interface CappedLogsRequest {
|
|
48
|
+
/** First `maxRecords` records, in wire order. */
|
|
49
|
+
records: DecodedLogRecord[];
|
|
50
|
+
/** Records beyond the cap that were counted but never adapted. */
|
|
51
|
+
rejected: number;
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Like {@link decodeLogsRequest}, but stops adapting once `maxRecords`
|
|
56
|
+
* records have been produced, so an oversized batch never gets a second,
|
|
57
|
+
* adapted in-memory copy. Output is identical to decoding everything and
|
|
58
|
+
* slicing the first `maxRecords`.
|
|
59
|
+
*/
|
|
60
|
+
export const decodeLogsRequestCapped = (
|
|
61
|
+
body: ReadBodyResult,
|
|
62
|
+
maxRecords: number,
|
|
63
|
+
): CappedLogsRequest => {
|
|
45
64
|
let msg: ExportLogsServiceRequest;
|
|
46
65
|
try {
|
|
47
66
|
msg =
|
|
@@ -56,16 +75,25 @@ export const decodeLogsRequest = (body: ReadBodyResult): DecodedLogRecord[] => {
|
|
|
56
75
|
}
|
|
57
76
|
|
|
58
77
|
const out: DecodedLogRecord[] = [];
|
|
78
|
+
let rejected = 0;
|
|
59
79
|
for (const rl of msg.resourceLogs) {
|
|
80
|
+
if (out.length >= maxRecords) {
|
|
81
|
+
for (const sl of rl.scopeLogs) rejected += sl.logRecords.length;
|
|
82
|
+
continue;
|
|
83
|
+
}
|
|
60
84
|
const serviceName = extractServiceName(rl.resource?.attributes);
|
|
61
85
|
for (const sl of rl.scopeLogs) {
|
|
62
86
|
const loggerName = sl.scope?.name || null;
|
|
63
87
|
for (const log of sl.logRecords) {
|
|
88
|
+
if (out.length >= maxRecords) {
|
|
89
|
+
rejected++;
|
|
90
|
+
continue;
|
|
91
|
+
}
|
|
64
92
|
out.push(adaptLogRecord(log, serviceName, loggerName));
|
|
65
93
|
}
|
|
66
94
|
}
|
|
67
95
|
}
|
|
68
|
-
return out;
|
|
96
|
+
return { records: out, rejected };
|
|
69
97
|
};
|
|
70
98
|
|
|
71
99
|
const adaptLogRecord = (
|
|
@@ -78,7 +78,26 @@ export interface DecodedMetricPoint {
|
|
|
78
78
|
|
|
79
79
|
export const decodeMetricsRequest = (
|
|
80
80
|
body: ReadBodyResult,
|
|
81
|
-
): DecodedMetricPoint[] =>
|
|
81
|
+
): DecodedMetricPoint[] =>
|
|
82
|
+
decodeMetricsRequestCapped(body, Number.POSITIVE_INFINITY).points;
|
|
83
|
+
|
|
84
|
+
export interface CappedMetricsRequest {
|
|
85
|
+
/** First `maxPoints` points, in wire order. */
|
|
86
|
+
points: DecodedMetricPoint[];
|
|
87
|
+
/** Points beyond the cap that were counted but never adapted. */
|
|
88
|
+
rejected: number;
|
|
89
|
+
}
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Like {@link decodeMetricsRequest}, but stops adapting once `maxPoints`
|
|
93
|
+
* points have been produced, so an oversized batch never gets a second,
|
|
94
|
+
* adapted in-memory copy. Output is identical to decoding everything and
|
|
95
|
+
* slicing the first `maxPoints`.
|
|
96
|
+
*/
|
|
97
|
+
export const decodeMetricsRequestCapped = (
|
|
98
|
+
body: ReadBodyResult,
|
|
99
|
+
maxPoints: number,
|
|
100
|
+
): CappedMetricsRequest => {
|
|
82
101
|
let msg: ExportMetricsServiceRequest;
|
|
83
102
|
try {
|
|
84
103
|
msg =
|
|
@@ -96,7 +115,20 @@ export const decodeMetricsRequest = (
|
|
|
96
115
|
}
|
|
97
116
|
|
|
98
117
|
const out: DecodedMetricPoint[] = [];
|
|
118
|
+
let rejected = 0;
|
|
99
119
|
for (const rm of msg.resourceMetrics) {
|
|
120
|
+
if (out.length >= maxPoints) {
|
|
121
|
+
for (const sm of rm.scopeMetrics) rejected += countPoints(sm);
|
|
122
|
+
continue;
|
|
123
|
+
}
|
|
124
|
+
const remaining = maxPoints - out.length;
|
|
125
|
+
const total = rm.scopeMetrics.reduce((n, sm) => n + countPoints(sm), 0);
|
|
126
|
+
if (total > remaining) {
|
|
127
|
+
// Trim this resource's data points in place (wire order) so the
|
|
128
|
+
// adapters below only ever build `remaining` points.
|
|
129
|
+
rejected += total - remaining;
|
|
130
|
+
trimScopeMetrics(rm.scopeMetrics, remaining);
|
|
131
|
+
}
|
|
100
132
|
const resourceAttrs = rm.resource?.attributes ?? [];
|
|
101
133
|
const resourceAttrsJson = resourceAttrs.length
|
|
102
134
|
? JSON.stringify(keyValuesToRecord(resourceAttrs))
|
|
@@ -106,7 +138,40 @@ export const decodeMetricsRequest = (
|
|
|
106
138
|
addMetrics(out, rm, sm, serviceName, resourceAttrsJson);
|
|
107
139
|
}
|
|
108
140
|
}
|
|
109
|
-
return out;
|
|
141
|
+
return { points: out, rejected };
|
|
142
|
+
};
|
|
143
|
+
|
|
144
|
+
/** Data-point count of the metric kinds `addMetrics` emits. */
|
|
145
|
+
const metricPointCount = (metric: Metric): number => {
|
|
146
|
+
switch (metric.data.case) {
|
|
147
|
+
case "gauge":
|
|
148
|
+
case "sum":
|
|
149
|
+
case "histogram":
|
|
150
|
+
case "exponentialHistogram":
|
|
151
|
+
case "summary":
|
|
152
|
+
return metric.data.value.dataPoints.length;
|
|
153
|
+
default:
|
|
154
|
+
return 0;
|
|
155
|
+
}
|
|
156
|
+
};
|
|
157
|
+
|
|
158
|
+
const countPoints = (sm: ScopeMetrics): number =>
|
|
159
|
+
sm.metrics.reduce((n, m) => n + metricPointCount(m), 0);
|
|
160
|
+
|
|
161
|
+
/** Keep only the first `budget` emitted data points, in wire order. */
|
|
162
|
+
const trimScopeMetrics = (scopeMetrics: ScopeMetrics[], budget: number) => {
|
|
163
|
+
let left = budget;
|
|
164
|
+
for (const sm of scopeMetrics) {
|
|
165
|
+
for (const metric of sm.metrics) {
|
|
166
|
+
const n = metricPointCount(metric);
|
|
167
|
+
if (n === 0) continue;
|
|
168
|
+
if (n > left) {
|
|
169
|
+
const data = metric.data.value as { dataPoints: unknown[] };
|
|
170
|
+
data.dataPoints = data.dataPoints.slice(0, left);
|
|
171
|
+
}
|
|
172
|
+
left -= Math.min(n, left);
|
|
173
|
+
}
|
|
174
|
+
}
|
|
110
175
|
};
|
|
111
176
|
|
|
112
177
|
const addMetrics = (
|
|
@@ -7,7 +7,10 @@ import {
|
|
|
7
7
|
type ExportTraceServiceRequest,
|
|
8
8
|
ExportTraceServiceRequestSchema,
|
|
9
9
|
} from "../gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.js";
|
|
10
|
-
import type {
|
|
10
|
+
import type {
|
|
11
|
+
ResourceSpans,
|
|
12
|
+
Span,
|
|
13
|
+
} from "../gen/opentelemetry/proto/trace/v1/trace_pb.js";
|
|
11
14
|
import { decodeJsonBody, OtlpDecodeError, type ReadBodyResult } from "./body";
|
|
12
15
|
import { adaptKeyValue, bigintToString, bytesToHex } from "./values";
|
|
13
16
|
|
|
@@ -17,7 +20,57 @@ import { adaptKeyValue, bigintToString, bytesToHex } from "./values";
|
|
|
17
20
|
*/
|
|
18
21
|
export const decodeTraceRequest = (
|
|
19
22
|
body: ReadBodyResult,
|
|
20
|
-
): OtlpTraceExportRequest =>
|
|
23
|
+
): OtlpTraceExportRequest =>
|
|
24
|
+
decodeTraceRequestCapped(body, Number.POSITIVE_INFINITY).request;
|
|
25
|
+
|
|
26
|
+
export interface CappedTraceRequest {
|
|
27
|
+
/** Adapted request holding at most `maxSpans` spans. */
|
|
28
|
+
request: OtlpTraceExportRequest;
|
|
29
|
+
/** Spans kept (≤ maxSpans). */
|
|
30
|
+
accepted: number;
|
|
31
|
+
/** Well-formed spans dropped because the cap was reached. */
|
|
32
|
+
rejected: number;
|
|
33
|
+
}
|
|
34
|
+
|
|
35
|
+
/**
|
|
36
|
+
* Like {@link decodeTraceRequest}, but applies the per-request span cap to
|
|
37
|
+
* the raw decoded message *before* adapting, so an oversized batch never
|
|
38
|
+
* gets a second, adapted in-memory copy. Spans with malformed trace/span
|
|
39
|
+
* ID lengths are skipped without counting against the cap — exactly as
|
|
40
|
+
* the adapter drops them — so the kept set is identical to adapting
|
|
41
|
+
* everything and keeping the first `maxSpans`.
|
|
42
|
+
*/
|
|
43
|
+
export const decodeTraceRequestCapped = (
|
|
44
|
+
body: ReadBodyResult,
|
|
45
|
+
maxSpans: number,
|
|
46
|
+
): CappedTraceRequest => {
|
|
47
|
+
const msg = parseTraceMessage(body);
|
|
48
|
+
let accepted = 0;
|
|
49
|
+
let rejected = 0;
|
|
50
|
+
const resourceSpans: OtlpResourceSpans[] = [];
|
|
51
|
+
for (const rs of msg.resourceSpans) {
|
|
52
|
+
for (const ss of rs.scopeSpans) {
|
|
53
|
+
const kept: Span[] = [];
|
|
54
|
+
for (const s of ss.spans) {
|
|
55
|
+
if (!hasValidIds(s)) continue;
|
|
56
|
+
if (accepted >= maxSpans) {
|
|
57
|
+
rejected++;
|
|
58
|
+
} else {
|
|
59
|
+
kept.push(s);
|
|
60
|
+
accepted++;
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
ss.spans = kept;
|
|
64
|
+
}
|
|
65
|
+
resourceSpans.push(adaptResourceSpans(rs));
|
|
66
|
+
}
|
|
67
|
+
return { request: { resourceSpans }, accepted, rejected };
|
|
68
|
+
};
|
|
69
|
+
|
|
70
|
+
const hasValidIds = (s: Span): boolean =>
|
|
71
|
+
s.traceId.length === 16 && s.spanId.length === 8;
|
|
72
|
+
|
|
73
|
+
const parseTraceMessage = (body: ReadBodyResult): ExportTraceServiceRequest => {
|
|
21
74
|
let msg: ExportTraceServiceRequest;
|
|
22
75
|
try {
|
|
23
76
|
msg =
|
|
@@ -30,7 +83,7 @@ export const decodeTraceRequest = (
|
|
|
30
83
|
400,
|
|
31
84
|
);
|
|
32
85
|
}
|
|
33
|
-
return
|
|
86
|
+
return msg;
|
|
34
87
|
};
|
|
35
88
|
|
|
36
89
|
// ── proto-native → legacy shape adapters ─────────────────────────────
|
|
@@ -44,7 +97,7 @@ const adaptResourceSpans = (rs: ResourceSpans): OtlpResourceSpans => ({
|
|
|
44
97
|
? { name: ss.scope.name, version: ss.scope.version }
|
|
45
98
|
: undefined,
|
|
46
99
|
spans: ss.spans.flatMap((s) => {
|
|
47
|
-
if (s
|
|
100
|
+
if (!hasValidIds(s)) return [];
|
|
48
101
|
return [
|
|
49
102
|
{
|
|
50
103
|
traceId: bytesToHex(s.traceId),
|
|
@@ -55,23 +55,63 @@ export const anyValueToString = (v: AnyValue | undefined): string => {
|
|
|
55
55
|
return bytesToBase64(v.value.value);
|
|
56
56
|
case "arrayValue":
|
|
57
57
|
case "kvlistValue":
|
|
58
|
-
return JSON.stringify(
|
|
58
|
+
return JSON.stringify(adaptAnyValueAt(v, 1));
|
|
59
59
|
default:
|
|
60
60
|
return "";
|
|
61
61
|
}
|
|
62
62
|
};
|
|
63
63
|
|
|
64
|
+
/**
|
|
65
|
+
* Max nesting depth for `arrayValue` / `kvlistValue` AnyValues. Real OTel
|
|
66
|
+
* attributes are a handful of levels deep at most; the cap keeps recursion
|
|
67
|
+
* bounded (no stack overflow on hostile payloads) and keeps stored JSON far
|
|
68
|
+
* below SQLite's 1000-level JSON depth limit. Containers nested deeper than
|
|
69
|
+
* this are replaced by {@link TRUNCATED_DEPTH_MARKER}.
|
|
70
|
+
*/
|
|
71
|
+
export const MAX_ANY_VALUE_DEPTH = 64;
|
|
72
|
+
export const TRUNCATED_DEPTH_MARKER = `[truncated: nested deeper than ${MAX_ANY_VALUE_DEPTH} levels]`;
|
|
73
|
+
|
|
74
|
+
/**
|
|
75
|
+
* Assign as an own data property. Plain `out[key] = v` would treat a
|
|
76
|
+
* `__proto__` attribute key as a prototype setter and silently drop it;
|
|
77
|
+
* defineProperty stores it like any other key (and JSON.stringify emits it).
|
|
78
|
+
*/
|
|
79
|
+
const setOwn = (
|
|
80
|
+
out: Record<string, ObsJsonValue>,
|
|
81
|
+
key: string,
|
|
82
|
+
value: ObsJsonValue,
|
|
83
|
+
): void => {
|
|
84
|
+
if (key === "__proto__") {
|
|
85
|
+
Object.defineProperty(out, key, {
|
|
86
|
+
value,
|
|
87
|
+
enumerable: true,
|
|
88
|
+
writable: true,
|
|
89
|
+
configurable: true,
|
|
90
|
+
});
|
|
91
|
+
} else {
|
|
92
|
+
out[key] = value;
|
|
93
|
+
}
|
|
94
|
+
};
|
|
95
|
+
|
|
64
96
|
export const keyValuesToRecord = (
|
|
65
97
|
kvs: KeyValue[],
|
|
98
|
+
): Record<string, ObsJsonValue> => keyValuesToRecordAt(kvs, 0);
|
|
99
|
+
|
|
100
|
+
const keyValuesToRecordAt = (
|
|
101
|
+
kvs: KeyValue[],
|
|
102
|
+
depth: number,
|
|
66
103
|
): Record<string, ObsJsonValue> => {
|
|
67
104
|
const out: Record<string, ObsJsonValue> = {};
|
|
68
105
|
for (const kv of kvs) {
|
|
69
|
-
out
|
|
106
|
+
setOwn(out, kv.key, anyValueToJson(kv.value, depth + 1));
|
|
70
107
|
}
|
|
71
108
|
return out;
|
|
72
109
|
};
|
|
73
110
|
|
|
74
|
-
const anyValueToJson = (
|
|
111
|
+
const anyValueToJson = (
|
|
112
|
+
v: AnyValue | undefined,
|
|
113
|
+
depth: number,
|
|
114
|
+
): ObsJsonValue => {
|
|
75
115
|
if (!v) return null;
|
|
76
116
|
switch (v.value.case) {
|
|
77
117
|
case "stringValue":
|
|
@@ -86,20 +126,28 @@ const anyValueToJson = (v: AnyValue | undefined): ObsJsonValue => {
|
|
|
86
126
|
case "bytesValue":
|
|
87
127
|
return bytesToBase64(v.value.value);
|
|
88
128
|
case "arrayValue":
|
|
89
|
-
|
|
129
|
+
if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
|
|
130
|
+
return v.value.value.values.map((x) => anyValueToJson(x, depth + 1));
|
|
90
131
|
case "kvlistValue":
|
|
91
|
-
|
|
132
|
+
if (depth > MAX_ANY_VALUE_DEPTH) return TRUNCATED_DEPTH_MARKER;
|
|
133
|
+
return keyValuesToRecordAt(v.value.value.values, depth);
|
|
92
134
|
default:
|
|
93
135
|
return null;
|
|
94
136
|
}
|
|
95
137
|
};
|
|
96
138
|
|
|
97
|
-
export const adaptKeyValue = (kv: KeyValue): OtlpKeyValue =>
|
|
139
|
+
export const adaptKeyValue = (kv: KeyValue): OtlpKeyValue =>
|
|
140
|
+
adaptKeyValueAt(kv, 0);
|
|
141
|
+
|
|
142
|
+
const adaptKeyValueAt = (kv: KeyValue, depth: number): OtlpKeyValue => ({
|
|
98
143
|
key: kv.key,
|
|
99
|
-
value: kv.value ?
|
|
144
|
+
value: kv.value ? adaptAnyValueAt(kv.value, depth + 1) : undefined,
|
|
100
145
|
});
|
|
101
146
|
|
|
102
|
-
export const adaptAnyValue = (v: AnyValue): OtlpAnyValue =>
|
|
147
|
+
export const adaptAnyValue = (v: AnyValue): OtlpAnyValue =>
|
|
148
|
+
adaptAnyValueAt(v, 1);
|
|
149
|
+
|
|
150
|
+
const adaptAnyValueAt = (v: AnyValue, depth: number): OtlpAnyValue => {
|
|
103
151
|
switch (v.value.case) {
|
|
104
152
|
case "stringValue":
|
|
105
153
|
return { stringValue: v.value.value };
|
|
@@ -110,15 +158,21 @@ export const adaptAnyValue = (v: AnyValue): OtlpAnyValue => {
|
|
|
110
158
|
case "doubleValue":
|
|
111
159
|
return { doubleValue: v.value.value };
|
|
112
160
|
case "arrayValue":
|
|
161
|
+
if (depth > MAX_ANY_VALUE_DEPTH)
|
|
162
|
+
return { stringValue: TRUNCATED_DEPTH_MARKER };
|
|
113
163
|
return {
|
|
114
164
|
arrayValue: {
|
|
115
|
-
values: v.value.value.values.map((x) =>
|
|
165
|
+
values: v.value.value.values.map((x) =>
|
|
166
|
+
adaptAnyValueAt(x, depth + 1),
|
|
167
|
+
),
|
|
116
168
|
},
|
|
117
169
|
};
|
|
118
170
|
case "kvlistValue":
|
|
171
|
+
if (depth > MAX_ANY_VALUE_DEPTH)
|
|
172
|
+
return { stringValue: TRUNCATED_DEPTH_MARKER };
|
|
119
173
|
return {
|
|
120
174
|
kvlistValue: {
|
|
121
|
-
values: v.value.value.values.map(
|
|
175
|
+
values: v.value.value.values.map((kv) => adaptKeyValueAt(kv, depth)),
|
|
122
176
|
},
|
|
123
177
|
};
|
|
124
178
|
case "bytesValue":
|
package/src/otlp/decode.test.ts
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import { create, toBinary, toJson } from "@bufbuild/protobuf";
|
|
2
2
|
import { describe, expect, it } from "vitest";
|
|
3
|
+
import { toStoredSpans } from "../lib/otlp";
|
|
3
4
|
import {
|
|
4
5
|
decodeLogsRequest,
|
|
6
|
+
decodeLogsRequestCapped,
|
|
5
7
|
decodeMetricsRequest,
|
|
8
|
+
decodeMetricsRequestCapped,
|
|
6
9
|
decodeTraceRequest,
|
|
10
|
+
decodeTraceRequestCapped,
|
|
7
11
|
OtlpDecodeError,
|
|
8
12
|
readOtlpBody,
|
|
9
13
|
} from "./decode";
|
|
14
|
+
import { MAX_ANY_VALUE_DEPTH, TRUNCATED_DEPTH_MARKER } from "./decode/values";
|
|
10
15
|
import { ExportLogsServiceRequestSchema } from "./gen/opentelemetry/proto/collector/logs/v1/logs_service_pb.js";
|
|
11
16
|
import { ExportMetricsServiceRequestSchema } from "./gen/opentelemetry/proto/collector/metrics/v1/metrics_service_pb.js";
|
|
12
17
|
import { ExportTraceServiceRequestSchema } from "./gen/opentelemetry/proto/collector/trace/v1/trace_service_pb.js";
|
|
@@ -798,3 +803,303 @@ describe("decodeMetricsRequest", () => {
|
|
|
798
803
|
]);
|
|
799
804
|
});
|
|
800
805
|
});
|
|
806
|
+
|
|
807
|
+
// ── Security hardening: pre-adaptation caps, depth cap, __proto__ keys ──
|
|
808
|
+
|
|
809
|
+
const strAttr = (key: string, value: string) =>
|
|
810
|
+
create(KeyValueSchema, {
|
|
811
|
+
key,
|
|
812
|
+
value: create(AnyValueSchema, { value: { case: "stringValue", value } }),
|
|
813
|
+
});
|
|
814
|
+
|
|
815
|
+
/** AnyValue nested `levels` kvlists deep, innermost holding a string. */
|
|
816
|
+
const nestedKvlist = (levels: number) => {
|
|
817
|
+
let v = create(AnyValueSchema, {
|
|
818
|
+
value: { case: "stringValue", value: "leaf" },
|
|
819
|
+
});
|
|
820
|
+
for (let i = 0; i < levels; i++) {
|
|
821
|
+
v = create(AnyValueSchema, {
|
|
822
|
+
value: {
|
|
823
|
+
case: "kvlistValue",
|
|
824
|
+
value: { values: [create(KeyValueSchema, { key: "k", value: v })] },
|
|
825
|
+
},
|
|
826
|
+
});
|
|
827
|
+
}
|
|
828
|
+
return v;
|
|
829
|
+
};
|
|
830
|
+
|
|
831
|
+
const jsonDepth = (v: unknown): number => {
|
|
832
|
+
if (Array.isArray(v)) return 1 + Math.max(0, ...v.map(jsonDepth));
|
|
833
|
+
if (v && typeof v === "object")
|
|
834
|
+
return 1 + Math.max(0, ...Object.values(v).map(jsonDepth));
|
|
835
|
+
return 0;
|
|
836
|
+
};
|
|
837
|
+
|
|
838
|
+
const traceBatch = (spanCounts: number[], invalidEvery = 0) =>
|
|
839
|
+
create(ExportTraceServiceRequestSchema, {
|
|
840
|
+
resourceSpans: spanCounts.map((count, r) =>
|
|
841
|
+
create(ResourceSpansSchema, {
|
|
842
|
+
resource: create(ResourceSchema, {
|
|
843
|
+
attributes: [strAttr("service.name", `svc-${r}`)],
|
|
844
|
+
}),
|
|
845
|
+
scopeSpans: [
|
|
846
|
+
create(ScopeSpansSchema, {
|
|
847
|
+
spans: Array.from({ length: count }, (_, i) =>
|
|
848
|
+
create(SpanSchema, {
|
|
849
|
+
traceId:
|
|
850
|
+
invalidEvery && i % invalidEvery === 0
|
|
851
|
+
? new Uint8Array(3)
|
|
852
|
+
: new Uint8Array(16).fill((i % 250) + 1),
|
|
853
|
+
spanId: new Uint8Array(8).fill((r % 250) + 1),
|
|
854
|
+
name: `op-${r}-${i}`,
|
|
855
|
+
}),
|
|
856
|
+
),
|
|
857
|
+
}),
|
|
858
|
+
],
|
|
859
|
+
}),
|
|
860
|
+
),
|
|
861
|
+
});
|
|
862
|
+
|
|
863
|
+
describe("decodeTraceRequestCapped", () => {
|
|
864
|
+
it("keeps exactly the spans the post-adapt cap kept, and counts the rest", () => {
|
|
865
|
+
const bytes = toBinary(
|
|
866
|
+
ExportTraceServiceRequestSchema,
|
|
867
|
+
traceBatch([300, 400], 7),
|
|
868
|
+
);
|
|
869
|
+
const full = decodeTraceRequest({ bytes, wireFormat: "protobuf" });
|
|
870
|
+
const fullSpans = (full.resourceSpans ?? []).flatMap((rs) =>
|
|
871
|
+
(rs.scopeSpans ?? []).flatMap((ss) => ss.spans ?? []),
|
|
872
|
+
);
|
|
873
|
+
const capped = decodeTraceRequestCapped(
|
|
874
|
+
{ bytes, wireFormat: "protobuf" },
|
|
875
|
+
500,
|
|
876
|
+
);
|
|
877
|
+
const cappedSpans = capped.request.resourceSpans?.flatMap((rs) =>
|
|
878
|
+
(rs.scopeSpans ?? []).flatMap((ss) => ss.spans ?? []),
|
|
879
|
+
);
|
|
880
|
+
expect(capped.accepted).toBe(500);
|
|
881
|
+
expect(capped.rejected).toBe(fullSpans.length - 500);
|
|
882
|
+
expect(cappedSpans).toEqual(fullSpans.slice(0, 500));
|
|
883
|
+
// Resource/scope structure is preserved even when all its spans drop.
|
|
884
|
+
expect(capped.request.resourceSpans).toHaveLength(2);
|
|
885
|
+
});
|
|
886
|
+
|
|
887
|
+
it("does not reject anything under the cap (JSON)", () => {
|
|
888
|
+
const json = toJson(ExportTraceServiceRequestSchema, traceBatch([10, 5]));
|
|
889
|
+
const bytes = new TextEncoder().encode(JSON.stringify(json));
|
|
890
|
+
const capped = decodeTraceRequestCapped({ bytes, wireFormat: "json" }, 500);
|
|
891
|
+
expect(capped).toMatchObject({ accepted: 15, rejected: 0 });
|
|
892
|
+
expect(capped.request).toEqual(
|
|
893
|
+
decodeTraceRequest({ bytes, wireFormat: "json" }),
|
|
894
|
+
);
|
|
895
|
+
});
|
|
896
|
+
|
|
897
|
+
it("still surfaces malformed bodies as OtlpDecodeError 400", () => {
|
|
898
|
+
expect(() =>
|
|
899
|
+
decodeTraceRequestCapped(
|
|
900
|
+
{ bytes: new Uint8Array([0xff, 0xff]), wireFormat: "protobuf" },
|
|
901
|
+
500,
|
|
902
|
+
),
|
|
903
|
+
).toThrow(OtlpDecodeError);
|
|
904
|
+
});
|
|
905
|
+
});
|
|
906
|
+
|
|
907
|
+
describe("decodeLogsRequestCapped", () => {
|
|
908
|
+
const logsBatch = (counts: number[]) =>
|
|
909
|
+
create(ExportLogsServiceRequestSchema, {
|
|
910
|
+
resourceLogs: counts.map((count, r) =>
|
|
911
|
+
create(ResourceLogsSchema, {
|
|
912
|
+
resource: create(ResourceSchema, {
|
|
913
|
+
attributes: [strAttr("service.name", `svc-${r}`)],
|
|
914
|
+
}),
|
|
915
|
+
scopeLogs: [
|
|
916
|
+
create(ScopeLogsSchema, {
|
|
917
|
+
logRecords: Array.from({ length: count }, (_, i) =>
|
|
918
|
+
create(LogRecordSchema, {
|
|
919
|
+
timeUnixNano: 1_700_000_000_000_000_000n,
|
|
920
|
+
body: create(AnyValueSchema, {
|
|
921
|
+
value: { case: "stringValue", value: `m-${r}-${i}` },
|
|
922
|
+
}),
|
|
923
|
+
}),
|
|
924
|
+
),
|
|
925
|
+
}),
|
|
926
|
+
],
|
|
927
|
+
}),
|
|
928
|
+
),
|
|
929
|
+
});
|
|
930
|
+
|
|
931
|
+
it("equals decode-then-slice and reports the same rejected count", () => {
|
|
932
|
+
const bytes = toBinary(
|
|
933
|
+
ExportLogsServiceRequestSchema,
|
|
934
|
+
logsBatch([600, 300, 400]),
|
|
935
|
+
);
|
|
936
|
+
const full = decodeLogsRequest({ bytes, wireFormat: "protobuf" });
|
|
937
|
+
const capped = decodeLogsRequestCapped(
|
|
938
|
+
{ bytes, wireFormat: "protobuf" },
|
|
939
|
+
1000,
|
|
940
|
+
);
|
|
941
|
+
expect(capped.rejected).toBe(full.length - 1000);
|
|
942
|
+
expect(capped.records.map((r) => r.message)).toEqual(
|
|
943
|
+
full.slice(0, 1000).map((r) => r.message),
|
|
944
|
+
);
|
|
945
|
+
});
|
|
946
|
+
});
|
|
947
|
+
|
|
948
|
+
describe("decodeMetricsRequestCapped", () => {
|
|
949
|
+
const gauge = (name: string, n: number) =>
|
|
950
|
+
create(MetricSchema, {
|
|
951
|
+
name,
|
|
952
|
+
data: {
|
|
953
|
+
case: "gauge",
|
|
954
|
+
value: create(GaugeSchema, {
|
|
955
|
+
dataPoints: Array.from({ length: n }, (_, i) =>
|
|
956
|
+
create(NumberDataPointSchema, {
|
|
957
|
+
timeUnixNano: 1_700_000_000_000_000_000n + BigInt(i),
|
|
958
|
+
value: { case: "asDouble", value: i },
|
|
959
|
+
}),
|
|
960
|
+
),
|
|
961
|
+
}),
|
|
962
|
+
},
|
|
963
|
+
});
|
|
964
|
+
const metricsBatch = () =>
|
|
965
|
+
create(ExportMetricsServiceRequestSchema, {
|
|
966
|
+
resourceMetrics: [
|
|
967
|
+
create(ResourceMetricsSchema, {
|
|
968
|
+
scopeMetrics: [
|
|
969
|
+
create(ScopeMetricsSchema, {
|
|
970
|
+
metrics: [gauge("a", 1500), gauge("b", 800)],
|
|
971
|
+
}),
|
|
972
|
+
],
|
|
973
|
+
}),
|
|
974
|
+
create(ResourceMetricsSchema, {
|
|
975
|
+
scopeMetrics: [
|
|
976
|
+
create(ScopeMetricsSchema, { metrics: [gauge("c", 50)] }),
|
|
977
|
+
],
|
|
978
|
+
}),
|
|
979
|
+
],
|
|
980
|
+
});
|
|
981
|
+
|
|
982
|
+
it("equals decode-then-slice and reports the same rejected count", () => {
|
|
983
|
+
const bytes = toBinary(ExportMetricsServiceRequestSchema, metricsBatch());
|
|
984
|
+
const full = decodeMetricsRequest({ bytes, wireFormat: "protobuf" });
|
|
985
|
+
const capped = decodeMetricsRequestCapped(
|
|
986
|
+
{ bytes, wireFormat: "protobuf" },
|
|
987
|
+
2000,
|
|
988
|
+
);
|
|
989
|
+
expect(full).toHaveLength(2350);
|
|
990
|
+
expect(capped.rejected).toBe(350);
|
|
991
|
+
expect(capped.points).toEqual(full.slice(0, 2000));
|
|
992
|
+
});
|
|
993
|
+
});
|
|
994
|
+
|
|
995
|
+
describe("AnyValue nesting depth cap", () => {
|
|
996
|
+
const logWithAttr = (value: ReturnType<typeof nestedKvlist>) =>
|
|
997
|
+
toBinary(
|
|
998
|
+
ExportLogsServiceRequestSchema,
|
|
999
|
+
create(ExportLogsServiceRequestSchema, {
|
|
1000
|
+
resourceLogs: [
|
|
1001
|
+
create(ResourceLogsSchema, {
|
|
1002
|
+
scopeLogs: [
|
|
1003
|
+
create(ScopeLogsSchema, {
|
|
1004
|
+
logRecords: [
|
|
1005
|
+
create(LogRecordSchema, {
|
|
1006
|
+
attributes: [
|
|
1007
|
+
create(KeyValueSchema, { key: "deep", value }),
|
|
1008
|
+
],
|
|
1009
|
+
}),
|
|
1010
|
+
],
|
|
1011
|
+
}),
|
|
1012
|
+
],
|
|
1013
|
+
}),
|
|
1014
|
+
],
|
|
1015
|
+
}),
|
|
1016
|
+
);
|
|
1017
|
+
|
|
1018
|
+
it("leaves realistic nesting untouched", () => {
|
|
1019
|
+
const [log] = decodeLogsRequest({
|
|
1020
|
+
bytes: logWithAttr(nestedKvlist(MAX_ANY_VALUE_DEPTH)),
|
|
1021
|
+
wireFormat: "protobuf",
|
|
1022
|
+
});
|
|
1023
|
+
expect(JSON.stringify(log?.attributes)).not.toContain("truncated");
|
|
1024
|
+
expect(jsonDepth(log?.attributes)).toBe(MAX_ANY_VALUE_DEPTH + 1);
|
|
1025
|
+
});
|
|
1026
|
+
|
|
1027
|
+
it("truncates attributes nested past the cap (logs/metrics path)", () => {
|
|
1028
|
+
const [log] = decodeLogsRequest({
|
|
1029
|
+
bytes: logWithAttr(nestedKvlist(400)),
|
|
1030
|
+
wireFormat: "protobuf",
|
|
1031
|
+
});
|
|
1032
|
+
const serialized = JSON.stringify(log?.attributes);
|
|
1033
|
+
expect(serialized).toContain(TRUNCATED_DEPTH_MARKER);
|
|
1034
|
+
expect(jsonDepth(log?.attributes)).toBeLessThanOrEqual(
|
|
1035
|
+
MAX_ANY_VALUE_DEPTH + 1,
|
|
1036
|
+
);
|
|
1037
|
+
});
|
|
1038
|
+
|
|
1039
|
+
it("truncates span attributes nested past the cap (traces path)", () => {
|
|
1040
|
+
const msg = traceBatch([1]);
|
|
1041
|
+
const span = msg.resourceSpans[0]?.scopeSpans[0]?.spans[0];
|
|
1042
|
+
if (!span) throw new Error("fixture");
|
|
1043
|
+
span.attributes = [
|
|
1044
|
+
create(KeyValueSchema, { key: "deep", value: nestedKvlist(400) }),
|
|
1045
|
+
];
|
|
1046
|
+
const bytes = toBinary(ExportTraceServiceRequestSchema, msg);
|
|
1047
|
+
const stored = toStoredSpans(
|
|
1048
|
+
decodeTraceRequest({ bytes, wireFormat: "protobuf" }),
|
|
1049
|
+
"p",
|
|
1050
|
+
new Date(),
|
|
1051
|
+
72,
|
|
1052
|
+
);
|
|
1053
|
+
const attrs = JSON.parse(stored[0]?.attributesJson ?? "{}");
|
|
1054
|
+
expect(stored[0]?.attributesJson).toContain(TRUNCATED_DEPTH_MARKER);
|
|
1055
|
+
expect(jsonDepth(attrs)).toBeLessThanOrEqual(MAX_ANY_VALUE_DEPTH + 1);
|
|
1056
|
+
});
|
|
1057
|
+
});
|
|
1058
|
+
|
|
1059
|
+
describe("__proto__ attribute keys", () => {
|
|
1060
|
+
it("are stored as normal own keys, not dropped or used as a prototype", () => {
|
|
1061
|
+
const bytes = toBinary(
|
|
1062
|
+
ExportLogsServiceRequestSchema,
|
|
1063
|
+
create(ExportLogsServiceRequestSchema, {
|
|
1064
|
+
resourceLogs: [
|
|
1065
|
+
create(ResourceLogsSchema, {
|
|
1066
|
+
scopeLogs: [
|
|
1067
|
+
create(ScopeLogsSchema, {
|
|
1068
|
+
logRecords: [
|
|
1069
|
+
create(LogRecordSchema, {
|
|
1070
|
+
attributes: [
|
|
1071
|
+
strAttr("__proto__", "x"),
|
|
1072
|
+
create(KeyValueSchema, {
|
|
1073
|
+
key: "nested",
|
|
1074
|
+
value: create(AnyValueSchema, {
|
|
1075
|
+
value: {
|
|
1076
|
+
case: "kvlistValue",
|
|
1077
|
+
value: {
|
|
1078
|
+
values: [
|
|
1079
|
+
create(KeyValueSchema, {
|
|
1080
|
+
key: "__proto__",
|
|
1081
|
+
value: nestedKvlist(1),
|
|
1082
|
+
}),
|
|
1083
|
+
],
|
|
1084
|
+
},
|
|
1085
|
+
},
|
|
1086
|
+
}),
|
|
1087
|
+
}),
|
|
1088
|
+
],
|
|
1089
|
+
}),
|
|
1090
|
+
],
|
|
1091
|
+
}),
|
|
1092
|
+
],
|
|
1093
|
+
}),
|
|
1094
|
+
],
|
|
1095
|
+
}),
|
|
1096
|
+
);
|
|
1097
|
+
const [log] = decodeLogsRequest({ bytes, wireFormat: "protobuf" });
|
|
1098
|
+
const attrs = log?.attributes ?? {};
|
|
1099
|
+
expect(Object.keys(attrs)).toEqual(["__proto__", "nested"]);
|
|
1100
|
+
expect(Object.getPrototypeOf(attrs)).toBe(Object.prototype);
|
|
1101
|
+
expect(JSON.stringify(attrs)).toBe(
|
|
1102
|
+
'{"__proto__":"x","nested":{"__proto__":{"k":"leaf"}}}',
|
|
1103
|
+
);
|
|
1104
|
+
});
|
|
1105
|
+
});
|
package/src/otlp/decode.ts
CHANGED
|
@@ -1,11 +1,17 @@
|
|
|
1
1
|
export type { OtlpWireFormat, ReadBodyResult } from "./decode/body";
|
|
2
2
|
export { OtlpDecodeError, readOtlpBody } from "./decode/body";
|
|
3
3
|
export type { DecodedLogRecord } from "./decode/logs";
|
|
4
|
-
export { decodeLogsRequest } from "./decode/logs";
|
|
4
|
+
export { decodeLogsRequest, decodeLogsRequestCapped } from "./decode/logs";
|
|
5
5
|
export type {
|
|
6
6
|
DecodedExemplar,
|
|
7
7
|
DecodedMetricPoint,
|
|
8
8
|
MetricType,
|
|
9
9
|
} from "./decode/metrics";
|
|
10
|
-
export {
|
|
11
|
-
|
|
10
|
+
export {
|
|
11
|
+
decodeMetricsRequest,
|
|
12
|
+
decodeMetricsRequestCapped,
|
|
13
|
+
} from "./decode/metrics";
|
|
14
|
+
export {
|
|
15
|
+
decodeTraceRequest,
|
|
16
|
+
decodeTraceRequestCapped,
|
|
17
|
+
} from "./decode/traces";
|