otelturbine 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +453 -0
- package/dist/index.js +25966 -0
- package/index.ts +27 -0
- package/package.json +59 -0
- package/src/adapters/bun.ts +62 -0
- package/src/adapters/elysia.ts +35 -0
- package/src/compress/snappy.ts +9 -0
- package/src/core/Compat.ts +92 -0
- package/src/core/OtelTurbine.ts +121 -0
- package/src/core/Pipeline.ts +169 -0
- package/src/core/RemoteWriteConfig.ts +8 -0
- package/src/proto/writeRequest.ts +133 -0
- package/src/tests/bunHandler.test.ts +134 -0
- package/src/tests/compat.test.ts +120 -0
- package/src/tests/otlpToTimeSeries.test.ts +184 -0
- package/src/tests/pipeline.test.ts +140 -0
- package/src/tests/proto.test.ts +102 -0
- package/src/tests/schemaEngine.test.ts +222 -0
- package/src/tests/varint.test.ts +53 -0
- package/src/transform/SchemaEngine.ts +174 -0
- package/src/transform/otlpToTimeSeries.ts +156 -0
- package/src/types/otlp.ts +83 -0
- package/src/types/prometheus.ts +23 -0
- package/src/types/schema.ts +36 -0
- package/src/util/varint.ts +80 -0
|
@@ -0,0 +1,140 @@
|
|
|
1
|
+
import { describe, it, expect, mock, beforeEach, afterEach } from 'bun:test';
|
|
2
|
+
import { Pipeline } from '../core/Pipeline.ts';
|
|
3
|
+
import { compileSchemas } from '../transform/SchemaEngine.ts';
|
|
4
|
+
|
|
5
|
+
const validOtlpPayload = JSON.stringify({
|
|
6
|
+
resourceMetrics: [
|
|
7
|
+
{
|
|
8
|
+
resource: {},
|
|
9
|
+
scopeMetrics: [
|
|
10
|
+
{
|
|
11
|
+
metrics: [
|
|
12
|
+
{
|
|
13
|
+
name: 'test_metric',
|
|
14
|
+
gauge: {
|
|
15
|
+
dataPoints: [
|
|
16
|
+
{
|
|
17
|
+
timeUnixNano: '1700000000000000000',
|
|
18
|
+
asDouble: 42.0,
|
|
19
|
+
},
|
|
20
|
+
],
|
|
21
|
+
},
|
|
22
|
+
},
|
|
23
|
+
],
|
|
24
|
+
},
|
|
25
|
+
],
|
|
26
|
+
},
|
|
27
|
+
],
|
|
28
|
+
});
|
|
29
|
+
|
|
30
|
+
const emptyOtlpPayload = JSON.stringify({
|
|
31
|
+
resourceMetrics: [],
|
|
32
|
+
});
|
|
33
|
+
|
|
34
|
+
describe('Pipeline', () => {
|
|
35
|
+
let originalFetch: typeof globalThis.fetch;
|
|
36
|
+
|
|
37
|
+
beforeEach(() => {
|
|
38
|
+
originalFetch = globalThis.fetch;
|
|
39
|
+
});
|
|
40
|
+
|
|
41
|
+
afterEach(() => {
|
|
42
|
+
globalThis.fetch = originalFetch;
|
|
43
|
+
});
|
|
44
|
+
|
|
45
|
+
function makePipeline(defaultAction: 'pass' | 'drop' = 'pass') {
|
|
46
|
+
return new Pipeline(
|
|
47
|
+
{ url: 'http://localhost:9090/api/v1/write', timeout: 5000 },
|
|
48
|
+
[],
|
|
49
|
+
defaultAction
|
|
50
|
+
);
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
it('returns 415 for application/x-protobuf content type', async () => {
|
|
54
|
+
const pipeline = makePipeline();
|
|
55
|
+
const result = await pipeline.process('', 'application/x-protobuf');
|
|
56
|
+
expect(result.status).toBe(415);
|
|
57
|
+
});
|
|
58
|
+
|
|
59
|
+
it('returns 415 for unsupported content types', async () => {
|
|
60
|
+
const pipeline = makePipeline();
|
|
61
|
+
const result = await pipeline.process('', 'text/plain');
|
|
62
|
+
expect(result.status).toBe(415);
|
|
63
|
+
});
|
|
64
|
+
|
|
65
|
+
it('returns 400 for invalid JSON', async () => {
|
|
66
|
+
const pipeline = makePipeline();
|
|
67
|
+
const result = await pipeline.process('not json at all {{{', 'application/json');
|
|
68
|
+
expect(result.status).toBe(400);
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('returns 400 for valid JSON without resourceMetrics', async () => {
|
|
72
|
+
const pipeline = makePipeline();
|
|
73
|
+
const result = await pipeline.process('{"foo": "bar"}', 'application/json');
|
|
74
|
+
expect(result.status).toBe(400);
|
|
75
|
+
});
|
|
76
|
+
|
|
77
|
+
it('returns 204 for empty resourceMetrics', async () => {
|
|
78
|
+
const pipeline = makePipeline();
|
|
79
|
+
const result = await pipeline.process(emptyOtlpPayload, 'application/json');
|
|
80
|
+
expect(result.status).toBe(204);
|
|
81
|
+
});
|
|
82
|
+
|
|
83
|
+
it('returns 204 when all metrics are dropped', async () => {
|
|
84
|
+
const pipeline = new Pipeline(
|
|
85
|
+
{ url: 'http://localhost:9090/api/v1/write', timeout: 5000 },
|
|
86
|
+
[],
|
|
87
|
+
'drop'
|
|
88
|
+
);
|
|
89
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json');
|
|
90
|
+
expect(result.status).toBe(204);
|
|
91
|
+
});
|
|
92
|
+
|
|
93
|
+
it('returns 200 on successful remote write', async () => {
|
|
94
|
+
globalThis.fetch = mock(async () => new Response(null, { status: 204 })) as unknown as typeof fetch;
|
|
95
|
+
const pipeline = makePipeline();
|
|
96
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json');
|
|
97
|
+
expect(result.status).toBe(200);
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('returns 502 when remote write fails with non-2xx', async () => {
|
|
101
|
+
globalThis.fetch = mock(async () => new Response('Internal Server Error', { status: 500 })) as unknown as typeof fetch;
|
|
102
|
+
const pipeline = makePipeline();
|
|
103
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json');
|
|
104
|
+
expect(result.status).toBe(502);
|
|
105
|
+
expect(result.message).toContain('500');
|
|
106
|
+
});
|
|
107
|
+
|
|
108
|
+
it('returns 502 when remote write throws (network error)', async () => {
|
|
109
|
+
globalThis.fetch = mock(async () => { throw new Error('Connection refused'); }) as unknown as typeof fetch;
|
|
110
|
+
const pipeline = makePipeline();
|
|
111
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json');
|
|
112
|
+
expect(result.status).toBe(502);
|
|
113
|
+
expect(result.message).toContain('Connection refused');
|
|
114
|
+
});
|
|
115
|
+
|
|
116
|
+
it('accepts content-type with charset parameter', async () => {
|
|
117
|
+
globalThis.fetch = mock(async () => new Response(null, { status: 204 })) as unknown as typeof fetch;
|
|
118
|
+
const pipeline = makePipeline();
|
|
119
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json; charset=utf-8');
|
|
120
|
+
expect(result.status).toBe(200);
|
|
121
|
+
});
|
|
122
|
+
|
|
123
|
+
it('applies schemas before sending', async () => {
|
|
124
|
+
let capturedBody: Uint8Array | undefined;
|
|
125
|
+
globalThis.fetch = mock(async (_: Request, init: RequestInit) => {
|
|
126
|
+
capturedBody = init.body as Uint8Array;
|
|
127
|
+
return new Response(null, { status: 204 });
|
|
128
|
+
}) as unknown as typeof fetch;
|
|
129
|
+
|
|
130
|
+
const schemas = compileSchemas([{ name: 'test_metric' }]);
|
|
131
|
+
const pipeline = new Pipeline(
|
|
132
|
+
{ url: 'http://localhost:9090/api/v1/write', timeout: 5000 },
|
|
133
|
+
schemas,
|
|
134
|
+
'drop'
|
|
135
|
+
);
|
|
136
|
+
const result = await pipeline.process(validOtlpPayload, 'application/json');
|
|
137
|
+
expect(result.status).toBe(200);
|
|
138
|
+
expect(capturedBody).toBeDefined();
|
|
139
|
+
});
|
|
140
|
+
});
|
|
@@ -0,0 +1,102 @@
|
|
|
1
|
+
import { describe, it, expect } from 'bun:test';
|
|
2
|
+
import { encodeWriteRequest } from '../proto/writeRequest.ts';
|
|
3
|
+
import type { WriteRequest, TimeSeries } from '../types/prometheus.ts';
|
|
4
|
+
|
|
5
|
+
describe('protobuf encoding', () => {
|
|
6
|
+
it('encodes an empty WriteRequest', () => {
|
|
7
|
+
const req: WriteRequest = { timeseries: [] };
|
|
8
|
+
const encoded = encodeWriteRequest(req);
|
|
9
|
+
expect(encoded).toBeInstanceOf(Uint8Array);
|
|
10
|
+
expect(encoded.length).toBe(0);
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('encodes a single TimeSeries with one label and one sample', () => {
|
|
14
|
+
const req: WriteRequest = {
|
|
15
|
+
timeseries: [
|
|
16
|
+
{
|
|
17
|
+
labels: [
|
|
18
|
+
{ name: '__name__', value: 'test_metric' },
|
|
19
|
+
],
|
|
20
|
+
samples: [
|
|
21
|
+
{ value: 42.0, timestamp: 1000n },
|
|
22
|
+
],
|
|
23
|
+
},
|
|
24
|
+
],
|
|
25
|
+
};
|
|
26
|
+
const encoded = encodeWriteRequest(req);
|
|
27
|
+
expect(encoded).toBeInstanceOf(Uint8Array);
|
|
28
|
+
expect(encoded.length).toBeGreaterThan(0);
|
|
29
|
+
// Field tag 0x0A = timeseries field 1
|
|
30
|
+
expect(encoded[0]).toBe(0x0a);
|
|
31
|
+
});
|
|
32
|
+
|
|
33
|
+
it('produces non-zero output for valid series', () => {
|
|
34
|
+
const ts: TimeSeries = {
|
|
35
|
+
labels: [
|
|
36
|
+
{ name: '__name__', value: 'http_requests_total' },
|
|
37
|
+
{ name: 'method', value: 'GET' },
|
|
38
|
+
{ name: 'status', value: '200' },
|
|
39
|
+
],
|
|
40
|
+
samples: [
|
|
41
|
+
{ value: 1234.5, timestamp: 1700000000000n },
|
|
42
|
+
],
|
|
43
|
+
};
|
|
44
|
+
const encoded = encodeWriteRequest({ timeseries: [ts] });
|
|
45
|
+
expect(encoded.length).toBeGreaterThan(20);
|
|
46
|
+
});
|
|
47
|
+
|
|
48
|
+
it('double value is encoded as 8 bytes LE', () => {
|
|
49
|
+
// Encode a known double: 1.0
|
|
50
|
+
const req: WriteRequest = {
|
|
51
|
+
timeseries: [
|
|
52
|
+
{
|
|
53
|
+
labels: [{ name: 'n', value: 'v' }],
|
|
54
|
+
samples: [{ value: 1.0, timestamp: 0n }],
|
|
55
|
+
},
|
|
56
|
+
],
|
|
57
|
+
};
|
|
58
|
+
const encoded = encodeWriteRequest(req);
|
|
59
|
+
// Find the double bytes for 1.0: 3F F0 00 00 00 00 00 00 in BE, reversed for LE
|
|
60
|
+
const buf = Buffer.from(encoded);
|
|
61
|
+
// Search for 0x3F 0xF0 sequence (last 6 bytes would be 00s)
|
|
62
|
+
let found = false;
|
|
63
|
+
for (let i = 0; i < buf.length - 7; i++) {
|
|
64
|
+
if (buf[i + 7] === 0x3f && buf[i + 6] === 0xf0) {
|
|
65
|
+
found = true;
|
|
66
|
+
break;
|
|
67
|
+
}
|
|
68
|
+
}
|
|
69
|
+
expect(found).toBe(true);
|
|
70
|
+
});
|
|
71
|
+
|
|
72
|
+
it('encodes multiple time series', () => {
|
|
73
|
+
const series: TimeSeries[] = Array.from({ length: 5 }, (_, i) => ({
|
|
74
|
+
labels: [{ name: '__name__', value: `metric_${i}` }],
|
|
75
|
+
samples: [{ value: i * 10.0, timestamp: BigInt(i * 1000) }],
|
|
76
|
+
}));
|
|
77
|
+
const encoded = encodeWriteRequest({ timeseries: series });
|
|
78
|
+
expect(encoded.length).toBeGreaterThan(0);
|
|
79
|
+
// Should have 5 repeated field 1 entries (each starting with 0x0A)
|
|
80
|
+
let count = 0;
|
|
81
|
+
// Count top-level 0x0A bytes (approximation)
|
|
82
|
+
let i = 0;
|
|
83
|
+
while (i < encoded.length) {
|
|
84
|
+
const tag = encoded[i]!;
|
|
85
|
+
i++;
|
|
86
|
+
if (tag === 0x0a) {
|
|
87
|
+
count++;
|
|
88
|
+
// Read length varint
|
|
89
|
+
let len = 0;
|
|
90
|
+
let shift = 0;
|
|
91
|
+
while (i < encoded.length) {
|
|
92
|
+
const b = encoded[i++]!;
|
|
93
|
+
len |= (b & 0x7f) << shift;
|
|
94
|
+
shift += 7;
|
|
95
|
+
if ((b & 0x80) === 0) break;
|
|
96
|
+
}
|
|
97
|
+
i += len;
|
|
98
|
+
}
|
|
99
|
+
}
|
|
100
|
+
expect(count).toBe(5);
|
|
101
|
+
});
|
|
102
|
+
});
|
|
@@ -0,0 +1,222 @@
|
|
|
1
|
+
import { describe, it, expect } from 'bun:test';
|
|
2
|
+
import { compileSchemas, applySchemas } from '../transform/SchemaEngine.ts';
|
|
3
|
+
import type { TimeSeries } from '../types/prometheus.ts';
|
|
4
|
+
import type { MetricSchema } from '../types/schema.ts';
|
|
5
|
+
|
|
6
|
+
function makeSeries(name: string, labels: Record<string, string> = {}): TimeSeries {
|
|
7
|
+
const allLabels = [
|
|
8
|
+
{ name: '__name__', value: name },
|
|
9
|
+
...Object.entries(labels).map(([k, v]) => ({ name: k, value: v })),
|
|
10
|
+
].sort((a, b) => a.name.localeCompare(b.name));
|
|
11
|
+
return {
|
|
12
|
+
labels: allLabels,
|
|
13
|
+
samples: [{ value: 1.0, timestamp: 1000n }],
|
|
14
|
+
};
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
describe('SchemaEngine', () => {
|
|
18
|
+
describe('name matching', () => {
|
|
19
|
+
it('matches exact string name', () => {
|
|
20
|
+
const schemas = compileSchemas([{ name: 'cpu_usage' }]);
|
|
21
|
+
const series = [makeSeries('cpu_usage'), makeSeries('mem_usage')];
|
|
22
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
23
|
+
expect(result).toHaveLength(1);
|
|
24
|
+
expect(result[0]!.labels.find(l => l.name === '__name__')?.value).toBe('cpu_usage');
|
|
25
|
+
});
|
|
26
|
+
|
|
27
|
+
it('matches regexp name', () => {
|
|
28
|
+
const schemas = compileSchemas([{ name: /^http_/ }]);
|
|
29
|
+
const series = [
|
|
30
|
+
makeSeries('http_requests_total'),
|
|
31
|
+
makeSeries('http_errors_total'),
|
|
32
|
+
makeSeries('grpc_requests_total'),
|
|
33
|
+
];
|
|
34
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
35
|
+
expect(result).toHaveLength(2);
|
|
36
|
+
});
|
|
37
|
+
|
|
38
|
+
it('first matching schema wins', () => {
|
|
39
|
+
const schemas: MetricSchema[] = [
|
|
40
|
+
{ name: /^http_/, inject: { matched: 'first' } },
|
|
41
|
+
{ name: /^http_requests/, inject: { matched: 'second' } },
|
|
42
|
+
];
|
|
43
|
+
const compiled = compileSchemas(schemas);
|
|
44
|
+
const series = [makeSeries('http_requests_total', { method: 'GET' })];
|
|
45
|
+
const result = applySchemas(series, compiled, 'drop');
|
|
46
|
+
expect(result[0]!.labels.find(l => l.name === 'matched')?.value).toBe('first');
|
|
47
|
+
});
|
|
48
|
+
});
|
|
49
|
+
|
|
50
|
+
describe('defaultAction', () => {
|
|
51
|
+
it('passes through unmatched metrics when defaultAction=pass', () => {
|
|
52
|
+
const schemas = compileSchemas([{ name: 'cpu_usage' }]);
|
|
53
|
+
const series = [makeSeries('mem_usage')];
|
|
54
|
+
const result = applySchemas(series, schemas, 'pass');
|
|
55
|
+
expect(result).toHaveLength(1);
|
|
56
|
+
});
|
|
57
|
+
|
|
58
|
+
it('drops unmatched metrics when defaultAction=drop', () => {
|
|
59
|
+
const schemas = compileSchemas([{ name: 'cpu_usage' }]);
|
|
60
|
+
const series = [makeSeries('mem_usage')];
|
|
61
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
62
|
+
expect(result).toHaveLength(0);
|
|
63
|
+
});
|
|
64
|
+
});
|
|
65
|
+
|
|
66
|
+
describe('explicit label matching', () => {
|
|
67
|
+
it('keeps series when explicit label matches', () => {
|
|
68
|
+
const schemas = compileSchemas([
|
|
69
|
+
{ name: 'http_req', labels: { method: /^GET$/ } },
|
|
70
|
+
]);
|
|
71
|
+
const series = [makeSeries('http_req', { method: 'GET', status: '200' })];
|
|
72
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
73
|
+
expect(result).toHaveLength(1);
|
|
74
|
+
});
|
|
75
|
+
|
|
76
|
+
it('drops series when explicit label value does not match', () => {
|
|
77
|
+
const schemas = compileSchemas([
|
|
78
|
+
{ name: 'http_req', labels: { method: /^GET$/ } },
|
|
79
|
+
]);
|
|
80
|
+
const series = [makeSeries('http_req', { method: 'POST' })];
|
|
81
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
82
|
+
expect(result).toHaveLength(0);
|
|
83
|
+
});
|
|
84
|
+
|
|
85
|
+
it('drops series when required explicit label is missing', () => {
|
|
86
|
+
const schemas = compileSchemas([
|
|
87
|
+
{ name: 'http_req', labels: { method: /^GET$/ } },
|
|
88
|
+
]);
|
|
89
|
+
const series = [makeSeries('http_req', { status: '200' })];
|
|
90
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
91
|
+
expect(result).toHaveLength(0);
|
|
92
|
+
});
|
|
93
|
+
|
|
94
|
+
it('drops unlisted labels when no wildcard', () => {
|
|
95
|
+
const schemas = compileSchemas([
|
|
96
|
+
{ name: 'http_req', labels: { method: /^GET$/ } },
|
|
97
|
+
]);
|
|
98
|
+
const series = [makeSeries('http_req', { method: 'GET', status: '200', extra: 'val' })];
|
|
99
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
100
|
+
expect(result).toHaveLength(1);
|
|
101
|
+
const labelNames = result[0]!.labels.map(l => l.name);
|
|
102
|
+
expect(labelNames).toContain('__name__');
|
|
103
|
+
expect(labelNames).toContain('method');
|
|
104
|
+
expect(labelNames).not.toContain('status');
|
|
105
|
+
expect(labelNames).not.toContain('extra');
|
|
106
|
+
});
|
|
107
|
+
});
|
|
108
|
+
|
|
109
|
+
describe('wildcard label matching', () => {
|
|
110
|
+
it('keeps unlisted labels when wildcard matches', () => {
|
|
111
|
+
const schemas = compileSchemas([
|
|
112
|
+
{
|
|
113
|
+
name: 'http_req',
|
|
114
|
+
labels: {
|
|
115
|
+
method: /^GET$/,
|
|
116
|
+
'*': /.*/,
|
|
117
|
+
},
|
|
118
|
+
},
|
|
119
|
+
]);
|
|
120
|
+
const series = [makeSeries('http_req', { method: 'GET', status: '200', host: 'api.example.com' })];
|
|
121
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
122
|
+
expect(result).toHaveLength(1);
|
|
123
|
+
const labelNames = result[0]!.labels.map(l => l.name);
|
|
124
|
+
expect(labelNames).toContain('status');
|
|
125
|
+
expect(labelNames).toContain('host');
|
|
126
|
+
});
|
|
127
|
+
|
|
128
|
+
it('drops unlisted labels when wildcard value does not match', () => {
|
|
129
|
+
// Wildcard matches label VALUES, not names
|
|
130
|
+
// labels: { a: 'allowed_val', b: 'blocked_val' }
|
|
131
|
+
// wildcard /^allowed_/ → keep 'a' (value matches), drop 'b' (value doesn't)
|
|
132
|
+
const schemas = compileSchemas([
|
|
133
|
+
{
|
|
134
|
+
name: 'http_req',
|
|
135
|
+
labels: {
|
|
136
|
+
'*': /^allowed_/,
|
|
137
|
+
},
|
|
138
|
+
},
|
|
139
|
+
]);
|
|
140
|
+
const series = [makeSeries('http_req', { a: 'allowed_val', b: 'blocked_val' })];
|
|
141
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
142
|
+
const labelNames = result[0]!.labels.map(l => l.name);
|
|
143
|
+
expect(labelNames).toContain('a');
|
|
144
|
+
expect(labelNames).not.toContain('b');
|
|
145
|
+
});
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
describe('inject', () => {
|
|
149
|
+
it('adds new labels via inject', () => {
|
|
150
|
+
const schemas = compileSchemas([
|
|
151
|
+
{ name: 'test_metric', inject: { env: 'prod', region: 'us-east' } },
|
|
152
|
+
]);
|
|
153
|
+
const series = [makeSeries('test_metric')];
|
|
154
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
155
|
+
expect(result[0]!.labels.find(l => l.name === 'env')?.value).toBe('prod');
|
|
156
|
+
expect(result[0]!.labels.find(l => l.name === 'region')?.value).toBe('us-east');
|
|
157
|
+
});
|
|
158
|
+
|
|
159
|
+
it('overrides existing labels via inject', () => {
|
|
160
|
+
const schemas = compileSchemas([
|
|
161
|
+
{
|
|
162
|
+
name: 'test_metric',
|
|
163
|
+
labels: { env: /.*/ },
|
|
164
|
+
inject: { env: 'overridden' },
|
|
165
|
+
},
|
|
166
|
+
]);
|
|
167
|
+
const series = [makeSeries('test_metric', { env: 'original' })];
|
|
168
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
169
|
+
expect(result[0]!.labels.find(l => l.name === 'env')?.value).toBe('overridden');
|
|
170
|
+
});
|
|
171
|
+
|
|
172
|
+
it('inject does not cause series drop', () => {
|
|
173
|
+
const schemas = compileSchemas([
|
|
174
|
+
{ name: 'test_metric', inject: { always: 'here' } },
|
|
175
|
+
]);
|
|
176
|
+
const series = [makeSeries('test_metric')];
|
|
177
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
178
|
+
expect(result).toHaveLength(1);
|
|
179
|
+
});
|
|
180
|
+
});
|
|
181
|
+
|
|
182
|
+
describe('maxLabels', () => {
|
|
183
|
+
it('caps total labels excluding __name__', () => {
|
|
184
|
+
const schemas = compileSchemas([
|
|
185
|
+
{
|
|
186
|
+
name: 'test_metric',
|
|
187
|
+
labels: { '*': /.*/ },
|
|
188
|
+
maxLabels: 2,
|
|
189
|
+
},
|
|
190
|
+
]);
|
|
191
|
+
const series = [makeSeries('test_metric', { a: '1', b: '2', c: '3', d: '4' })];
|
|
192
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
193
|
+
const labelNames = result[0]!.labels.map(l => l.name).filter(n => n !== '__name__');
|
|
194
|
+
expect(labelNames).toHaveLength(2);
|
|
195
|
+
});
|
|
196
|
+
|
|
197
|
+
it('always preserves __name__', () => {
|
|
198
|
+
const schemas = compileSchemas([
|
|
199
|
+
{
|
|
200
|
+
name: 'test_metric',
|
|
201
|
+
labels: { '*': /.*/ },
|
|
202
|
+
maxLabels: 1,
|
|
203
|
+
},
|
|
204
|
+
]);
|
|
205
|
+
const series = [makeSeries('test_metric', { a: '1', b: '2', c: '3' })];
|
|
206
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
207
|
+
expect(result[0]!.labels.find(l => l.name === '__name__')).toBeDefined();
|
|
208
|
+
});
|
|
209
|
+
});
|
|
210
|
+
|
|
211
|
+
describe('output sorted', () => {
|
|
212
|
+
it('output labels are sorted alphabetically', () => {
|
|
213
|
+
const schemas = compileSchemas([
|
|
214
|
+
{ name: 'test_metric', labels: { '*': /.*/ } },
|
|
215
|
+
]);
|
|
216
|
+
const series = [makeSeries('test_metric', { z: '1', a: '2', m: '3' })];
|
|
217
|
+
const result = applySchemas(series, schemas, 'drop');
|
|
218
|
+
const names = result[0]!.labels.map(l => l.name);
|
|
219
|
+
expect(names).toEqual([...names].sort());
|
|
220
|
+
});
|
|
221
|
+
});
|
|
222
|
+
});
|
|
@@ -0,0 +1,53 @@
|
|
|
1
|
+
import { describe, it, expect } from 'bun:test';
|
|
2
|
+
import { encodeVarint, writeVarint, varintSize } from '../util/varint.ts';
|
|
3
|
+
|
|
4
|
+
describe('varint encoding', () => {
|
|
5
|
+
it('encodes 0', () => {
|
|
6
|
+
expect(encodeVarint(0n)).toEqual(new Uint8Array([0x00]));
|
|
7
|
+
});
|
|
8
|
+
|
|
9
|
+
it('encodes 1', () => {
|
|
10
|
+
expect(encodeVarint(1n)).toEqual(new Uint8Array([0x01]));
|
|
11
|
+
});
|
|
12
|
+
|
|
13
|
+
it('encodes 127 as single byte', () => {
|
|
14
|
+
expect(encodeVarint(127n)).toEqual(new Uint8Array([0x7f]));
|
|
15
|
+
});
|
|
16
|
+
|
|
17
|
+
it('encodes 128 as two bytes', () => {
|
|
18
|
+
expect(encodeVarint(128n)).toEqual(new Uint8Array([0x80, 0x01]));
|
|
19
|
+
});
|
|
20
|
+
|
|
21
|
+
it('encodes 300', () => {
|
|
22
|
+
// 300 = 0x12C → varint: 0xAC 0x02
|
|
23
|
+
expect(encodeVarint(300n)).toEqual(new Uint8Array([0xac, 0x02]));
|
|
24
|
+
});
|
|
25
|
+
|
|
26
|
+
it('encodes large value (2^32)', () => {
|
|
27
|
+
const val = 4294967296n; // 2^32
|
|
28
|
+
const encoded = encodeVarint(val);
|
|
29
|
+
// Decode manually to verify
|
|
30
|
+
let result = 0n;
|
|
31
|
+
let shift = 0n;
|
|
32
|
+
for (const byte of encoded) {
|
|
33
|
+
result |= BigInt(byte & 0x7f) << shift;
|
|
34
|
+
shift += 7n;
|
|
35
|
+
}
|
|
36
|
+
expect(result).toBe(val);
|
|
37
|
+
});
|
|
38
|
+
|
|
39
|
+
it('varintSize matches encodeVarint length', () => {
|
|
40
|
+
const testValues = [0n, 1n, 127n, 128n, 300n, 16383n, 16384n, 2097151n, 2097152n];
|
|
41
|
+
for (const v of testValues) {
|
|
42
|
+
expect(varintSize(v)).toBe(encodeVarint(v).length);
|
|
43
|
+
}
|
|
44
|
+
});
|
|
45
|
+
|
|
46
|
+
it('writeVarint writes correctly', () => {
|
|
47
|
+
const buf = new Uint8Array(10);
|
|
48
|
+
const written = writeVarint(buf, 0, 300n);
|
|
49
|
+
expect(written).toBe(2);
|
|
50
|
+
expect(buf[0]).toBe(0xac);
|
|
51
|
+
expect(buf[1]).toBe(0x02);
|
|
52
|
+
});
|
|
53
|
+
});
|
|
@@ -0,0 +1,174 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Schema compilation and application engine.
|
|
3
|
+
*
|
|
4
|
+
* Compilation (once at build()):
|
|
5
|
+
* - string name → exact-match RegExp (anchored)
|
|
6
|
+
* - LabelPattern → FastMatcher (any / exact / regex)
|
|
7
|
+
* - "*" key extracted as wildcardMatcher
|
|
8
|
+
* - inject entries pre-computed as Array<[string, string]>
|
|
9
|
+
*
|
|
10
|
+
* Application (per-series):
|
|
11
|
+
* 1. Match series name against compiled schemas (first match wins)
|
|
12
|
+
* 2. Single pass over ts.labels:
|
|
13
|
+
* - explicit key: value must match FastMatcher; else DROP series
|
|
14
|
+
* - count how many explicit labels were seen; drop if any missing
|
|
15
|
+
* - wildcard: keep unlisted labels whose value matches; drop otherwise
|
|
16
|
+
* 3. Apply inject (binary search on sorted outLabels)
|
|
17
|
+
* 4. Apply maxLabels cap
|
|
18
|
+
* 5. Re-sort labels
|
|
19
|
+
*/
|
|
20
|
+
|
|
21
|
+
import type { MetricSchema, CompiledSchema, FastMatcher, LabelPattern, DefaultAction } from '../types/schema.ts';
|
|
22
|
+
import type { TimeSeries, Label } from '../types/prometheus.ts';
|
|
23
|
+
|
|
24
|
+
// ─── Compilation ────────────────────────────────────────────────────────────
|
|
25
|
+
|
|
26
|
+
function compileFastMatcher(p: LabelPattern): FastMatcher {
|
|
27
|
+
if (typeof p === 'string') {
|
|
28
|
+
return { type: 'exact', value: p };
|
|
29
|
+
}
|
|
30
|
+
const src = p.source;
|
|
31
|
+
// /.*/ with any flags → always matches
|
|
32
|
+
if (src === '.*' || src === '^.*$') return { type: 'any' };
|
|
33
|
+
// /^exact$/ where inner part has no regex special chars → equality
|
|
34
|
+
const exactInner = src.match(/^\^([^.*+?[\](){}\\|^$]+)\$$/)?.[1];
|
|
35
|
+
if (exactInner !== undefined) return { type: 'exact', value: exactInner };
|
|
36
|
+
return { type: 'regex', re: p };
|
|
37
|
+
}
|
|
38
|
+
|
|
39
|
+
function nameToRegExp(n: string | RegExp): RegExp {
|
|
40
|
+
if (n instanceof RegExp) return n;
|
|
41
|
+
return new RegExp(`^${n.replace(/[.*+?^${}()|[\]\\]/g, '\\$&')}$`);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
export function compileSchemas(schemas: MetricSchema[]): CompiledSchema[] {
|
|
45
|
+
return schemas.map((s) => {
|
|
46
|
+
const labelMatchers = new Map<string, FastMatcher>();
|
|
47
|
+
let wildcardMatcher: FastMatcher | undefined;
|
|
48
|
+
|
|
49
|
+
if (s.labels) {
|
|
50
|
+
for (const [key, pattern] of Object.entries(s.labels)) {
|
|
51
|
+
if (key === '*') {
|
|
52
|
+
wildcardMatcher = compileFastMatcher(pattern);
|
|
53
|
+
} else {
|
|
54
|
+
labelMatchers.set(key, compileFastMatcher(pattern));
|
|
55
|
+
}
|
|
56
|
+
}
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
const inject = s.inject ?? {};
|
|
60
|
+
return {
|
|
61
|
+
namePattern: nameToRegExp(s.name),
|
|
62
|
+
labelMatchers,
|
|
63
|
+
wildcardMatcher,
|
|
64
|
+
inject,
|
|
65
|
+
injectEntries: Object.entries(inject) as Array<[string, string]>,
|
|
66
|
+
maxLabels: s.maxLabels,
|
|
67
|
+
};
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
// ─── Fast matcher ────────────────────────────────────────────────────────────
|
|
72
|
+
|
|
73
|
+
function testFastMatcher(m: FastMatcher, value: string): boolean {
|
|
74
|
+
switch (m.type) {
|
|
75
|
+
case 'any': return true;
|
|
76
|
+
case 'exact': return value === m.value;
|
|
77
|
+
case 'regex': return m.re.test(value);
|
|
78
|
+
}
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
// ─── Binary search over sorted Label[] ──────────────────────────────────────
|
|
82
|
+
|
|
83
|
+
function bsearchLabel(labels: Label[], name: string): number {
|
|
84
|
+
let lo = 0, hi = labels.length - 1;
|
|
85
|
+
while (lo <= hi) {
|
|
86
|
+
const mid = (lo + hi) >>> 1;
|
|
87
|
+
const n = labels[mid]!.name;
|
|
88
|
+
if (n === name) return mid;
|
|
89
|
+
if (n < name) lo = mid + 1;
|
|
90
|
+
else hi = mid - 1;
|
|
91
|
+
}
|
|
92
|
+
return -1;
|
|
93
|
+
}
|
|
94
|
+
|
|
95
|
+
// ─── Application ─────────────────────────────────────────────────────────────
|
|
96
|
+
|
|
97
|
+
export function applySchemas(
|
|
98
|
+
series: TimeSeries[],
|
|
99
|
+
schemas: CompiledSchema[],
|
|
100
|
+
defaultAction: DefaultAction
|
|
101
|
+
): TimeSeries[] {
|
|
102
|
+
const result: TimeSeries[] = [];
|
|
103
|
+
for (const ts of series) {
|
|
104
|
+
const nameLabel = ts.labels[bsearchLabel(ts.labels, '__name__')];
|
|
105
|
+
const metricName = nameLabel?.value ?? '';
|
|
106
|
+
|
|
107
|
+
const schema = schemas.find((s) => s.namePattern.test(metricName));
|
|
108
|
+
if (!schema) {
|
|
109
|
+
if (defaultAction === 'pass') result.push(ts);
|
|
110
|
+
continue;
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
const transformed = applySchema(ts, schema);
|
|
114
|
+
if (transformed !== null) result.push(transformed);
|
|
115
|
+
}
|
|
116
|
+
return result;
|
|
117
|
+
}
|
|
118
|
+
|
|
119
|
+
function applySchema(ts: TimeSeries, schema: CompiledSchema): TimeSeries | null {
|
|
120
|
+
const outLabels: Label[] = [];
|
|
121
|
+
// Track how many of the required explicit labels we've seen
|
|
122
|
+
let seenExplicit = 0;
|
|
123
|
+
const explicitCount = schema.labelMatchers.size;
|
|
124
|
+
|
|
125
|
+
for (const label of ts.labels) {
|
|
126
|
+
const name = label.name;
|
|
127
|
+
|
|
128
|
+
if (name === '__name__') {
|
|
129
|
+
outLabels.push(label);
|
|
130
|
+
continue;
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
const matcher = schema.labelMatchers.get(name);
|
|
134
|
+
if (matcher !== undefined) {
|
|
135
|
+
if (!testFastMatcher(matcher, label.value)) return null; // value mismatch → drop series
|
|
136
|
+
outLabels.push(label);
|
|
137
|
+
seenExplicit++;
|
|
138
|
+
} else if (schema.wildcardMatcher !== undefined) {
|
|
139
|
+
if (testFastMatcher(schema.wildcardMatcher, label.value)) {
|
|
140
|
+
outLabels.push(label);
|
|
141
|
+
}
|
|
142
|
+
// else: label value doesn't match wildcard → drop this label (not the series)
|
|
143
|
+
}
|
|
144
|
+
// No wildcard → drop unlisted label
|
|
145
|
+
}
|
|
146
|
+
|
|
147
|
+
// All explicitly required labels must have been present in the series
|
|
148
|
+
if (seenExplicit < explicitCount) return null;
|
|
149
|
+
|
|
150
|
+
// Apply inject labels (outLabels is still sorted here)
|
|
151
|
+
for (const [k, v] of schema.injectEntries) {
|
|
152
|
+
const idx = bsearchLabel(outLabels, k);
|
|
153
|
+
if (idx >= 0) {
|
|
154
|
+
outLabels[idx] = { name: k, value: v };
|
|
155
|
+
} else {
|
|
156
|
+
outLabels.push({ name: k, value: v });
|
|
157
|
+
}
|
|
158
|
+
}
|
|
159
|
+
|
|
160
|
+
// Apply maxLabels cap
|
|
161
|
+
if (schema.maxLabels !== undefined) {
|
|
162
|
+
const nameIdx = outLabels.findIndex((l) => l.name === '__name__');
|
|
163
|
+
const nameLabel = nameIdx >= 0 ? outLabels.splice(nameIdx, 1)[0]! : undefined;
|
|
164
|
+
outLabels.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
165
|
+
const trimmed = outLabels.slice(0, schema.maxLabels);
|
|
166
|
+
if (nameLabel) trimmed.push(nameLabel);
|
|
167
|
+
trimmed.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
168
|
+
return { labels: trimmed, samples: ts.samples };
|
|
169
|
+
}
|
|
170
|
+
|
|
171
|
+
// Re-sort (inject may have added new labels at the end)
|
|
172
|
+
outLabels.sort((a, b) => (a.name < b.name ? -1 : a.name > b.name ? 1 : 0));
|
|
173
|
+
return { labels: outLabels, samples: ts.samples };
|
|
174
|
+
}
|