@aztec/telemetry-client 0.0.0-test.1 → 0.0.1-commit.017a351
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/attributes.d.ts +45 -10
- package/dest/attributes.d.ts.map +1 -1
- package/dest/attributes.js +28 -9
- package/dest/bench.d.ts +6 -1
- package/dest/bench.d.ts.map +1 -1
- package/dest/bench.js +31 -14
- package/dest/config.d.ts +9 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +44 -8
- package/dest/index.d.ts +2 -1
- package/dest/index.d.ts.map +1 -1
- package/dest/index.js +1 -0
- package/dest/l1_metrics.d.ts +17 -0
- package/dest/l1_metrics.d.ts.map +1 -0
- package/dest/l1_metrics.js +54 -0
- package/dest/lmdb_metrics.d.ts +5 -3
- package/dest/lmdb_metrics.d.ts.map +1 -1
- package/dest/lmdb_metrics.js +7 -15
- package/dest/metric-utils.d.ts +24 -0
- package/dest/metric-utils.d.ts.map +1 -0
- package/dest/metric-utils.js +59 -0
- package/dest/metrics.d.ts +312 -122
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +1595 -120
- package/dest/monitored_batch_span_processor.d.ts +29 -0
- package/dest/monitored_batch_span_processor.d.ts.map +1 -0
- package/dest/monitored_batch_span_processor.js +70 -0
- package/dest/nodejs_metrics_monitor.d.ts +19 -0
- package/dest/nodejs_metrics_monitor.d.ts.map +1 -0
- package/dest/nodejs_metrics_monitor.js +115 -0
- package/dest/noop.d.ts +9 -4
- package/dest/noop.d.ts.map +1 -1
- package/dest/noop.js +34 -1
- package/dest/otel.d.ts +13 -7
- package/dest/otel.d.ts.map +1 -1
- package/dest/otel.js +204 -24
- package/dest/otel_filter_metric_exporter.d.ts +12 -4
- package/dest/otel_filter_metric_exporter.d.ts.map +1 -1
- package/dest/otel_filter_metric_exporter.js +38 -4
- package/dest/otel_logger_provider.d.ts +1 -1
- package/dest/otel_propagation.d.ts +3 -1
- package/dest/otel_propagation.d.ts.map +1 -1
- package/dest/otel_propagation.js +49 -1
- package/dest/otel_resource.d.ts +1 -1
- package/dest/otel_resource.d.ts.map +1 -1
- package/dest/otel_resource.js +43 -2
- package/dest/prom_otel_adapter.d.ts +61 -12
- package/dest/prom_otel_adapter.d.ts.map +1 -1
- package/dest/prom_otel_adapter.js +157 -48
- package/dest/start.d.ts +3 -2
- package/dest/start.d.ts.map +1 -1
- package/dest/start.js +8 -7
- package/dest/telemetry.d.ts +63 -34
- package/dest/telemetry.d.ts.map +1 -1
- package/dest/telemetry.js +47 -24
- package/dest/vendor/attributes.d.ts +1 -1
- package/dest/vendor/otel-pino-stream.d.ts +1 -2
- package/dest/vendor/otel-pino-stream.d.ts.map +1 -1
- package/dest/vendor/otel-pino-stream.js +2 -2
- package/dest/with_tracer.d.ts +1 -1
- package/dest/with_tracer.d.ts.map +1 -1
- package/dest/wrappers/fetch.d.ts +3 -3
- package/dest/wrappers/fetch.d.ts.map +1 -1
- package/dest/wrappers/fetch.js +9 -6
- package/dest/wrappers/index.d.ts +1 -1
- package/dest/wrappers/json_rpc_server.d.ts +2 -2
- package/dest/wrappers/json_rpc_server.d.ts.map +1 -1
- package/dest/wrappers/l2_block_stream.d.ts +4 -3
- package/dest/wrappers/l2_block_stream.d.ts.map +1 -1
- package/dest/wrappers/l2_block_stream.js +385 -11
- package/package.json +21 -15
- package/src/attributes.ts +59 -12
- package/src/bench.ts +37 -15
- package/src/config.ts +76 -9
- package/src/index.ts +1 -0
- package/src/l1_metrics.ts +65 -0
- package/src/lmdb_metrics.ts +24 -24
- package/src/metric-utils.ts +75 -0
- package/src/metrics.ts +1679 -145
- package/src/monitored_batch_span_processor.ts +93 -0
- package/src/nodejs_metrics_monitor.ts +135 -0
- package/src/noop.ts +65 -4
- package/src/otel.ts +193 -28
- package/src/otel_filter_metric_exporter.ts +47 -5
- package/src/otel_propagation.ts +42 -1
- package/src/otel_resource.ts +57 -2
- package/src/prom_otel_adapter.ts +195 -70
- package/src/start.ts +17 -8
- package/src/telemetry.ts +152 -76
- package/src/vendor/otel-pino-stream.ts +1 -4
- package/src/wrappers/fetch.ts +32 -33
- package/src/wrappers/json_rpc_server.ts +1 -1
- package/src/wrappers/l2_block_stream.ts +3 -2
- package/dest/event_loop_monitor.d.ts +0 -18
- package/dest/event_loop_monitor.d.ts.map +0 -1
- package/dest/event_loop_monitor.js +0 -93
- package/src/event_loop_monitor.ts +0 -119
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
|
|
3
|
+
import { type Context, SpanStatusCode } from '@opentelemetry/api';
|
|
4
|
+
import { hrTimeToMilliseconds } from '@opentelemetry/core';
|
|
5
|
+
import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
|
|
6
|
+
import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
|
|
7
|
+
|
|
8
|
+
/** Minimum interval between drop warnings to avoid log spam. */
|
|
9
|
+
const DROP_WARNING_INTERVAL_MS = 30_000;
|
|
10
|
+
|
|
11
|
+
const DEFAULT_MIN_TRACE_DURATION_MS = 10;
|
|
12
|
+
|
|
13
|
+
export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
|
|
14
|
+
minTraceDurationMs?: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
19
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
20
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
21
|
+
*/
|
|
22
|
+
export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
23
|
+
private readonly maxQueueSize: number;
|
|
24
|
+
private readonly minTraceDurationMs: number;
|
|
25
|
+
private readonly log: Logger;
|
|
26
|
+
|
|
27
|
+
private approxQueueSize = 0;
|
|
28
|
+
private droppedSinceLastWarning = 0;
|
|
29
|
+
private totalDropped = 0;
|
|
30
|
+
private lastWarningTime = 0;
|
|
31
|
+
|
|
32
|
+
constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
|
|
33
|
+
const maxQueueSize = config?.maxQueueSize ?? 2048;
|
|
34
|
+
super(exporter, { ...config, maxQueueSize });
|
|
35
|
+
this.maxQueueSize = maxQueueSize;
|
|
36
|
+
this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
|
|
37
|
+
this.log = log;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
override onStart(span: Span, parentContext: Context): void {
|
|
41
|
+
super.onStart(span, parentContext);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
override onEnd(span: ReadableSpan): void {
|
|
45
|
+
if (this.shouldDropShortSpan(span)) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (this.approxQueueSize >= this.maxQueueSize) {
|
|
50
|
+
this.droppedSinceLastWarning++;
|
|
51
|
+
this.totalDropped++;
|
|
52
|
+
this.maybeLogDropWarning();
|
|
53
|
+
} else {
|
|
54
|
+
this.approxQueueSize++;
|
|
55
|
+
}
|
|
56
|
+
super.onEnd(span);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
override async forceFlush(): Promise<void> {
|
|
60
|
+
await super.forceFlush();
|
|
61
|
+
this.approxQueueSize = 0;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
override async shutdown(): Promise<void> {
|
|
65
|
+
if (this.totalDropped > 0) {
|
|
66
|
+
this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
|
|
67
|
+
totalDropped: this.totalDropped,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
await super.shutdown();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private shouldDropShortSpan(span: ReadableSpan): boolean {
|
|
74
|
+
return (
|
|
75
|
+
this.minTraceDurationMs > 0 &&
|
|
76
|
+
span.status.code !== SpanStatusCode.ERROR &&
|
|
77
|
+
hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private maybeLogDropWarning(): void {
|
|
82
|
+
const now = Date.now();
|
|
83
|
+
if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
|
|
84
|
+
this.log.warn(
|
|
85
|
+
`BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` +
|
|
86
|
+
`${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
|
|
87
|
+
{ droppedSinceLastWarning: this.droppedSinceLastWarning, totalDropped: this.totalDropped },
|
|
88
|
+
);
|
|
89
|
+
this.droppedSinceLastWarning = 0;
|
|
90
|
+
this.lastWarningTime = now;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -0,0 +1,135 @@
|
|
|
1
|
+
import type { Observable } from '@opentelemetry/api';
|
|
2
|
+
import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
|
3
|
+
|
|
4
|
+
import * as Attributes from './attributes.js';
|
|
5
|
+
import { createUpDownCounterWithDefault } from './metric-utils.js';
|
|
6
|
+
import * as Metrics from './metrics.js';
|
|
7
|
+
import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
|
|
8
|
+
|
|
9
|
+
/** Monitors Node.js runtime metrics */
|
|
10
|
+
export class NodejsMetricsMonitor {
|
|
11
|
+
private eventLoopDelayGauges: {
|
|
12
|
+
min: ObservableGauge;
|
|
13
|
+
max: ObservableGauge;
|
|
14
|
+
mean: ObservableGauge;
|
|
15
|
+
stddev: ObservableGauge;
|
|
16
|
+
p50: ObservableGauge;
|
|
17
|
+
p90: ObservableGauge;
|
|
18
|
+
p99: ObservableGauge;
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
// skip `rss` because that's already tracked by @opentelemetry/host-metrics
|
|
22
|
+
// description of each field here https://nodejs.org/api/process.html#processmemoryusage
|
|
23
|
+
private memoryGauges: Record<Exclude<keyof NodeJS.MemoryUsage, 'rss'>, ObservableGauge>;
|
|
24
|
+
|
|
25
|
+
private eventLoopUilization: ObservableGauge;
|
|
26
|
+
private eventLoopTime: UpDownCounter;
|
|
27
|
+
|
|
28
|
+
private started = false;
|
|
29
|
+
|
|
30
|
+
private lastELU: EventLoopUtilization | undefined;
|
|
31
|
+
private eventLoopDelay: IntervalHistogram;
|
|
32
|
+
|
|
33
|
+
constructor(private meter: Meter) {
|
|
34
|
+
this.eventLoopDelayGauges = {
|
|
35
|
+
min: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN),
|
|
36
|
+
mean: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN),
|
|
37
|
+
max: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX),
|
|
38
|
+
stddev: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV),
|
|
39
|
+
p50: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50),
|
|
40
|
+
p90: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90),
|
|
41
|
+
p99: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99),
|
|
42
|
+
};
|
|
43
|
+
|
|
44
|
+
this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
|
|
45
|
+
this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
|
|
46
|
+
[Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
|
|
47
|
+
});
|
|
48
|
+
this.eventLoopDelay = monitorEventLoopDelay();
|
|
49
|
+
|
|
50
|
+
this.memoryGauges = {
|
|
51
|
+
heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE),
|
|
52
|
+
heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL),
|
|
53
|
+
arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE),
|
|
54
|
+
external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE),
|
|
55
|
+
};
|
|
56
|
+
}
|
|
57
|
+
|
|
58
|
+
start(): void {
|
|
59
|
+
if (this.started) {
|
|
60
|
+
return;
|
|
61
|
+
}
|
|
62
|
+
|
|
63
|
+
this.lastELU = performance.eventLoopUtilization();
|
|
64
|
+
this.eventLoopDelay.enable();
|
|
65
|
+
this.meter.addBatchObservableCallback(this.measure, [
|
|
66
|
+
this.eventLoopUilization,
|
|
67
|
+
...Object.values(this.eventLoopDelayGauges),
|
|
68
|
+
...Object.values(this.memoryGauges),
|
|
69
|
+
]);
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
stop(): void {
|
|
73
|
+
if (!this.started) {
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
this.meter.removeBatchObservableCallback(this.measure, [
|
|
77
|
+
this.eventLoopUilization,
|
|
78
|
+
...Object.values(this.eventLoopDelayGauges),
|
|
79
|
+
...Object.values(this.memoryGauges),
|
|
80
|
+
]);
|
|
81
|
+
this.eventLoopDelay.disable();
|
|
82
|
+
this.eventLoopDelay.reset();
|
|
83
|
+
this.lastELU = undefined;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
private measure = (obs: BatchObservableResult): void => {
|
|
87
|
+
this.measureMemoryUsage(obs);
|
|
88
|
+
this.measureEventLoopDelay(obs);
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
private measureMemoryUsage = (observer: BatchObservableResult) => {
|
|
92
|
+
const mem = process.memoryUsage();
|
|
93
|
+
|
|
94
|
+
observer.observe(this.memoryGauges.heapUsed, mem.heapUsed);
|
|
95
|
+
observer.observe(this.memoryGauges.heapTotal, mem.heapTotal);
|
|
96
|
+
observer.observe(this.memoryGauges.arrayBuffers, mem.arrayBuffers);
|
|
97
|
+
observer.observe(this.memoryGauges.external, mem.external);
|
|
98
|
+
};
|
|
99
|
+
|
|
100
|
+
private measureEventLoopDelay = (obs: BatchObservableResult): void => {
|
|
101
|
+
const newELU = performance.eventLoopUtilization();
|
|
102
|
+
const delta = performance.eventLoopUtilization(newELU, this.lastELU);
|
|
103
|
+
this.lastELU = newELU;
|
|
104
|
+
|
|
105
|
+
// `utilization` [0,1] represents how much the event loop is busy vs waiting for new events to come in
|
|
106
|
+
// This should be corelated with CPU usage to gauge the performance characteristics of services
|
|
107
|
+
// 100% utilization leads to high latency because the event loop is _always_ busy, there's no breathing room for events to be processed quickly.
|
|
108
|
+
// Docs and examples:
|
|
109
|
+
// - https://nodesource.com/blog/event-loop-utilization-nodejs
|
|
110
|
+
// - https://youtu.be/WetXnEPraYM
|
|
111
|
+
obs.observe(this.eventLoopUilization, delta.utilization);
|
|
112
|
+
|
|
113
|
+
this.eventLoopTime.add(Math.trunc(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
|
|
114
|
+
this.eventLoopTime.add(Math.trunc(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
|
|
115
|
+
|
|
116
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.min, this.eventLoopDelay.min);
|
|
117
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.mean, this.eventLoopDelay.mean);
|
|
118
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.max, this.eventLoopDelay.max);
|
|
119
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.stddev, this.eventLoopDelay.stddev);
|
|
120
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.p50, this.eventLoopDelay.percentile(50));
|
|
121
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.p90, this.eventLoopDelay.percentile(90));
|
|
122
|
+
safeObserveInt(obs, this.eventLoopDelayGauges.p99, this.eventLoopDelay.percentile(99));
|
|
123
|
+
|
|
124
|
+
this.eventLoopDelay.reset();
|
|
125
|
+
};
|
|
126
|
+
}
|
|
127
|
+
|
|
128
|
+
function safeObserveInt(observer: BatchObservableResult, metric: Observable, value: number, attrs?: object) {
|
|
129
|
+
// discard NaN, Infinity, -Infinity
|
|
130
|
+
if (!Number.isFinite(value)) {
|
|
131
|
+
return;
|
|
132
|
+
}
|
|
133
|
+
|
|
134
|
+
observer.observe(metric, Math.trunc(value), attrs);
|
|
135
|
+
}
|
package/src/noop.ts
CHANGED
|
@@ -1,10 +1,63 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type Context, type Span, type SpanContext, type Tracer, createNoopMeter } from '@opentelemetry/api';
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type { MetricDefinition } from './metrics.js';
|
|
4
|
+
import type {
|
|
5
|
+
Gauge,
|
|
6
|
+
Histogram,
|
|
7
|
+
Meter,
|
|
8
|
+
ObservableGauge,
|
|
9
|
+
ObservableUpDownCounter,
|
|
10
|
+
TelemetryClient,
|
|
11
|
+
UpDownCounter,
|
|
12
|
+
} from './telemetry.js';
|
|
13
|
+
|
|
14
|
+
/** A no-op meter that implements our custom Meter interface */
|
|
15
|
+
class NoopMeter implements Meter {
|
|
16
|
+
private otelMeter = createNoopMeter();
|
|
17
|
+
|
|
18
|
+
createGauge(_metric: MetricDefinition): Gauge {
|
|
19
|
+
return this.otelMeter.createGauge('');
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
createObservableGauge(_metric: MetricDefinition): ObservableGauge {
|
|
23
|
+
return this.otelMeter.createObservableGauge('');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
createHistogram(_metric: MetricDefinition, _extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
|
|
27
|
+
return this.otelMeter.createHistogram('');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
createUpDownCounter(_metric: MetricDefinition): UpDownCounter {
|
|
31
|
+
return this.otelMeter.createUpDownCounter('');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
createObservableUpDownCounter(_metric: MetricDefinition): ObservableUpDownCounter {
|
|
35
|
+
return this.otelMeter.createObservableUpDownCounter('');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
addBatchObservableCallback(
|
|
39
|
+
callback: Parameters<Meter['addBatchObservableCallback']>[0],
|
|
40
|
+
observables: Parameters<Meter['addBatchObservableCallback']>[1],
|
|
41
|
+
): void {
|
|
42
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
removeBatchObservableCallback(
|
|
46
|
+
callback: Parameters<Meter['removeBatchObservableCallback']>[0],
|
|
47
|
+
observables: Parameters<Meter['removeBatchObservableCallback']>[1],
|
|
48
|
+
): void {
|
|
49
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
4
52
|
|
|
5
53
|
export class NoopTelemetryClient implements TelemetryClient {
|
|
54
|
+
private meter = new NoopMeter();
|
|
55
|
+
|
|
56
|
+
setExportedPublicTelemetry(_prefixes: string[]): void {}
|
|
57
|
+
setPublicTelemetryCollectFrom(_roles: string[]): void {}
|
|
58
|
+
|
|
6
59
|
getMeter(): Meter {
|
|
7
|
-
return
|
|
60
|
+
return this.meter;
|
|
8
61
|
}
|
|
9
62
|
|
|
10
63
|
getTracer(): Tracer {
|
|
@@ -22,6 +75,14 @@ export class NoopTelemetryClient implements TelemetryClient {
|
|
|
22
75
|
isEnabled() {
|
|
23
76
|
return false;
|
|
24
77
|
}
|
|
78
|
+
|
|
79
|
+
getTraceContext(): string | undefined {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
extractPropagatedContext(_traceContext: string): Context | undefined {
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
25
86
|
}
|
|
26
87
|
|
|
27
88
|
// @opentelemetry/api internally uses NoopTracer and NoopSpan but they're not exported
|
|
@@ -32,7 +93,7 @@ export class NoopTracer implements Tracer {
|
|
|
32
93
|
return new NoopSpan();
|
|
33
94
|
}
|
|
34
95
|
|
|
35
|
-
startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args:
|
|
96
|
+
startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args: unknown[]): ReturnType<F> {
|
|
36
97
|
// there are three different signatures for startActiveSpan, grab the function, we don't care about the rest
|
|
37
98
|
const fn = args.find(arg => typeof arg === 'function') as F;
|
|
38
99
|
return fn(new NoopSpan());
|
package/src/otel.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { type LogData, type Logger, addLogDataHandler } from '@aztec/foundation/log';
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
|
+
type Context,
|
|
4
5
|
DiagConsoleLogger,
|
|
5
6
|
DiagLogLevel,
|
|
6
|
-
type Meter,
|
|
7
|
+
type Meter as OtelMeter,
|
|
8
|
+
ROOT_CONTEXT,
|
|
7
9
|
type Tracer,
|
|
8
10
|
type TracerProvider,
|
|
9
11
|
context,
|
|
10
12
|
diag,
|
|
11
13
|
isSpanContextValid,
|
|
14
|
+
propagation,
|
|
12
15
|
trace,
|
|
13
16
|
} from '@opentelemetry/api';
|
|
17
|
+
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
|
14
18
|
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
15
19
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
16
20
|
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
@@ -24,22 +28,72 @@ import {
|
|
|
24
28
|
type PeriodicExportingMetricReaderOptions,
|
|
25
29
|
View,
|
|
26
30
|
} from '@opentelemetry/sdk-metrics';
|
|
27
|
-
import {
|
|
31
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
28
32
|
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
29
33
|
|
|
30
34
|
import type { TelemetryClientConfig } from './config.js';
|
|
31
|
-
import {
|
|
32
|
-
import {
|
|
35
|
+
import { toMetricOptions } from './metric-utils.js';
|
|
36
|
+
import type { MetricDefinition } from './metrics.js';
|
|
37
|
+
import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
|
|
38
|
+
import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
|
|
39
|
+
import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
33
40
|
import { registerOtelLoggerProvider } from './otel_logger_provider.js';
|
|
34
41
|
import { getOtelResource } from './otel_resource.js';
|
|
35
|
-
import type {
|
|
42
|
+
import type {
|
|
43
|
+
Gauge,
|
|
44
|
+
Histogram,
|
|
45
|
+
Meter,
|
|
46
|
+
ObservableGauge,
|
|
47
|
+
ObservableUpDownCounter,
|
|
48
|
+
TelemetryClient,
|
|
49
|
+
UpDownCounter,
|
|
50
|
+
} from './telemetry.js';
|
|
51
|
+
|
|
52
|
+
/** Wraps an OpenTelemetry Meter to implement our custom Meter interface */
|
|
53
|
+
class WrappedMeter implements Meter {
|
|
54
|
+
constructor(private otelMeter: OtelMeter) {}
|
|
55
|
+
|
|
56
|
+
createGauge(metric: MetricDefinition): Gauge {
|
|
57
|
+
return this.otelMeter.createGauge(metric.name, toMetricOptions(metric));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
createObservableGauge(metric: MetricDefinition): ObservableGauge {
|
|
61
|
+
return this.otelMeter.createObservableGauge(metric.name, toMetricOptions(metric));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
createHistogram(metric: MetricDefinition, extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
|
|
65
|
+
return this.otelMeter.createHistogram(metric.name, { ...toMetricOptions(metric), ...extraOptions });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
createUpDownCounter(metric: MetricDefinition): UpDownCounter {
|
|
69
|
+
return this.otelMeter.createUpDownCounter(metric.name, toMetricOptions(metric));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
createObservableUpDownCounter(metric: MetricDefinition): ObservableUpDownCounter {
|
|
73
|
+
return this.otelMeter.createObservableUpDownCounter(metric.name, toMetricOptions(metric));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
addBatchObservableCallback(
|
|
77
|
+
callback: Parameters<Meter['addBatchObservableCallback']>[0],
|
|
78
|
+
observables: Parameters<Meter['addBatchObservableCallback']>[1],
|
|
79
|
+
): void {
|
|
80
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
removeBatchObservableCallback(
|
|
84
|
+
callback: Parameters<Meter['removeBatchObservableCallback']>[0],
|
|
85
|
+
observables: Parameters<Meter['removeBatchObservableCallback']>[1],
|
|
86
|
+
): void {
|
|
87
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
36
90
|
|
|
37
91
|
export type OpenTelemetryClientFactory = (resource: IResource, log: Logger) => OpenTelemetryClient;
|
|
38
92
|
|
|
39
93
|
export class OpenTelemetryClient implements TelemetryClient {
|
|
40
94
|
hostMetrics: HostMetrics | undefined;
|
|
41
|
-
|
|
42
|
-
private meters: Map<string,
|
|
95
|
+
nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
|
|
96
|
+
private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
|
|
43
97
|
private tracers: Map<string, Tracer> = new Map<string, Tracer>();
|
|
44
98
|
|
|
45
99
|
protected constructor(
|
|
@@ -47,13 +101,23 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
47
101
|
private meterProvider: MeterProvider,
|
|
48
102
|
private traceProvider: TracerProvider,
|
|
49
103
|
private loggerProvider: LoggerProvider | undefined,
|
|
104
|
+
private publicMetricExporter: PublicOtelFilterMetricExporter | undefined,
|
|
50
105
|
private log: Logger,
|
|
51
106
|
) {}
|
|
52
107
|
|
|
108
|
+
setExportedPublicTelemetry(metrics: string[]): void {
|
|
109
|
+
this.publicMetricExporter?.setMetricPrefixes(metrics);
|
|
110
|
+
}
|
|
111
|
+
|
|
112
|
+
setPublicTelemetryCollectFrom(roles: string[]): void {
|
|
113
|
+
this.publicMetricExporter?.setAllowedRoles(roles);
|
|
114
|
+
}
|
|
115
|
+
|
|
53
116
|
getMeter(name: string): Meter {
|
|
54
117
|
let meter = this.meters.get(name);
|
|
55
118
|
if (!meter) {
|
|
56
|
-
|
|
119
|
+
const otelMeter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
|
|
120
|
+
meter = new WrappedMeter(otelMeter);
|
|
57
121
|
this.meters.set(name, meter);
|
|
58
122
|
}
|
|
59
123
|
return meter;
|
|
@@ -91,12 +155,13 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
91
155
|
meterProvider: this.meterProvider,
|
|
92
156
|
});
|
|
93
157
|
|
|
94
|
-
|
|
158
|
+
const nodejsMeter = new WrappedMeter(
|
|
95
159
|
this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME] as string),
|
|
96
160
|
);
|
|
161
|
+
this.nodejsMetricsMonitor = new NodejsMetricsMonitor(nodejsMeter);
|
|
97
162
|
|
|
98
163
|
this.hostMetrics.start();
|
|
99
|
-
this.
|
|
164
|
+
this.nodejsMetricsMonitor.start();
|
|
100
165
|
}
|
|
101
166
|
|
|
102
167
|
public isEnabled() {
|
|
@@ -112,7 +177,7 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
112
177
|
}
|
|
113
178
|
|
|
114
179
|
public async stop() {
|
|
115
|
-
this.
|
|
180
|
+
this.nodejsMetricsMonitor?.stop();
|
|
116
181
|
|
|
117
182
|
const flushAndShutdown = async (provider?: { forceFlush: () => Promise<void>; shutdown: () => Promise<void> }) => {
|
|
118
183
|
if (!provider) {
|
|
@@ -129,18 +194,46 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
129
194
|
]);
|
|
130
195
|
}
|
|
131
196
|
|
|
197
|
+
public getTraceContext(): string | undefined {
|
|
198
|
+
const carrier: Record<string, string> = {};
|
|
199
|
+
propagation.inject(context.active(), carrier);
|
|
200
|
+
return carrier['traceparent'];
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
public extractPropagatedContext(traceContext: string): Context {
|
|
204
|
+
const extractedContext = propagation.extract(ROOT_CONTEXT, {
|
|
205
|
+
traceparent: traceContext,
|
|
206
|
+
});
|
|
207
|
+
return extractedContext;
|
|
208
|
+
}
|
|
209
|
+
|
|
132
210
|
public static createMeterProvider(
|
|
133
211
|
resource: IResource,
|
|
134
|
-
|
|
212
|
+
exporters: Array<PeriodicExportingMetricReaderOptions>,
|
|
135
213
|
): MeterProvider {
|
|
136
214
|
return new MeterProvider({
|
|
137
215
|
resource,
|
|
138
|
-
readers: options
|
|
139
|
-
? [new PeriodicExportingMetricReader(options as PeriodicExportingMetricReaderOptions)]
|
|
140
|
-
: [],
|
|
216
|
+
readers: exporters.map(options => new PeriodicExportingMetricReader(options)),
|
|
141
217
|
|
|
142
218
|
views: [
|
|
143
219
|
// Every histogram matching the selector (type + unit) gets these custom buckets assigned
|
|
220
|
+
new View({
|
|
221
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
222
|
+
instrumentUnit: 'Mmana',
|
|
223
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
224
|
+
[0.1, 0.5, 1, 2, 4, 8, 10, 25, 50, 100, 500, 1000, 5000, 10000],
|
|
225
|
+
true,
|
|
226
|
+
),
|
|
227
|
+
}),
|
|
228
|
+
new View({
|
|
229
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
230
|
+
instrumentUnit: 'tx',
|
|
231
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
232
|
+
// TPS
|
|
233
|
+
[0.1 * 36, 0.2 * 36, 0.5 * 36, 1 * 36, 2 * 36, 5 * 36, 10 * 36, 15 * 36].map(Math.ceil),
|
|
234
|
+
true,
|
|
235
|
+
),
|
|
236
|
+
}),
|
|
144
237
|
new View({
|
|
145
238
|
instrumentType: InstrumentType.HISTOGRAM,
|
|
146
239
|
instrumentUnit: 's',
|
|
@@ -242,6 +335,36 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
242
335
|
true,
|
|
243
336
|
),
|
|
244
337
|
}),
|
|
338
|
+
// L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
|
|
339
|
+
new View({
|
|
340
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
341
|
+
instrumentUnit: 'gwei',
|
|
342
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
343
|
+
[0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
|
|
344
|
+
true,
|
|
345
|
+
),
|
|
346
|
+
}),
|
|
347
|
+
// L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
|
|
348
|
+
new View({
|
|
349
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
350
|
+
instrumentUnit: 'gas',
|
|
351
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
352
|
+
[
|
|
353
|
+
10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
|
|
354
|
+
30_000_000,
|
|
355
|
+
],
|
|
356
|
+
true,
|
|
357
|
+
),
|
|
358
|
+
}),
|
|
359
|
+
// L1 tx total fee in ETH: typically 0.001 - 1 ETH
|
|
360
|
+
new View({
|
|
361
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
362
|
+
instrumentUnit: 'eth',
|
|
363
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
364
|
+
[0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
|
|
365
|
+
true,
|
|
366
|
+
),
|
|
367
|
+
}),
|
|
245
368
|
],
|
|
246
369
|
});
|
|
247
370
|
}
|
|
@@ -251,26 +374,68 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
251
374
|
const tracerProvider = new NodeTracerProvider({
|
|
252
375
|
resource,
|
|
253
376
|
spanProcessors: config.tracesCollectorUrl
|
|
254
|
-
? [
|
|
377
|
+
? [
|
|
378
|
+
new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
|
|
379
|
+
maxQueueSize: config.otelBspMaxQueueSize,
|
|
380
|
+
minTraceDurationMs: config.otelMinTraceDurationMs,
|
|
381
|
+
}),
|
|
382
|
+
]
|
|
255
383
|
: [],
|
|
256
384
|
});
|
|
257
385
|
|
|
258
|
-
tracerProvider.register(
|
|
259
|
-
|
|
260
|
-
const meterProvider = OpenTelemetryClient.createMeterProvider(resource, {
|
|
261
|
-
exporter: config.metricsCollectorUrl
|
|
262
|
-
? new OtelFilterMetricExporter(
|
|
263
|
-
new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
|
|
264
|
-
config.otelExcludeMetrics ?? [],
|
|
265
|
-
)
|
|
266
|
-
: undefined,
|
|
267
|
-
exportTimeoutMillis: config.otelExportTimeoutMs,
|
|
268
|
-
exportIntervalMillis: config.otelCollectIntervalMs,
|
|
386
|
+
tracerProvider.register({
|
|
387
|
+
propagator: new W3CTraceContextPropagator(),
|
|
269
388
|
});
|
|
270
389
|
|
|
390
|
+
const exporters: PeriodicExportingMetricReaderOptions[] = [];
|
|
391
|
+
if (config.metricsCollectorUrl) {
|
|
392
|
+
// Default to a blacklist that is empty (allow all metrics)
|
|
393
|
+
let filter: string[] = [];
|
|
394
|
+
let mode: 'allow' | 'deny' = 'deny';
|
|
395
|
+
if (config.otelExcludeMetrics.length > 0) {
|
|
396
|
+
// Implement a blacklist as specified in config
|
|
397
|
+
log.info(`Excluding metrics from export: ${config.otelExcludeMetrics}`);
|
|
398
|
+
filter = config.otelExcludeMetrics;
|
|
399
|
+
mode = 'deny';
|
|
400
|
+
} else if (config.otelIncludeMetrics.length > 0) {
|
|
401
|
+
// Implement a whitelist as specified in config
|
|
402
|
+
log.info(`Including only specified metrics for export: ${config.otelIncludeMetrics}`);
|
|
403
|
+
filter = config.otelIncludeMetrics;
|
|
404
|
+
mode = 'allow';
|
|
405
|
+
}
|
|
406
|
+
exporters.push({
|
|
407
|
+
exporter: new OtelFilterMetricExporter(
|
|
408
|
+
new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
|
|
409
|
+
filter,
|
|
410
|
+
mode,
|
|
411
|
+
),
|
|
412
|
+
exportTimeoutMillis: config.otelExportTimeoutMs,
|
|
413
|
+
exportIntervalMillis: config.otelCollectIntervalMs,
|
|
414
|
+
});
|
|
415
|
+
}
|
|
416
|
+
|
|
417
|
+
let publicExporter: PublicOtelFilterMetricExporter | undefined;
|
|
418
|
+
if (config.publicMetricsCollectorUrl && !config.publicMetricsOptOut) {
|
|
419
|
+
log.info(`Exporting public metrics: ${config.publicIncludeMetrics}`, {
|
|
420
|
+
publicMetrics: config.publicIncludeMetrics,
|
|
421
|
+
collectorUrl: config.publicMetricsCollectorUrl,
|
|
422
|
+
});
|
|
423
|
+
publicExporter = new PublicOtelFilterMetricExporter(
|
|
424
|
+
config.publicMetricsCollectFrom,
|
|
425
|
+
new OTLPMetricExporter({ url: config.publicMetricsCollectorUrl.href }),
|
|
426
|
+
config.publicIncludeMetrics,
|
|
427
|
+
);
|
|
428
|
+
exporters.push({
|
|
429
|
+
exporter: publicExporter,
|
|
430
|
+
exportTimeoutMillis: config.otelExportTimeoutMs,
|
|
431
|
+
exportIntervalMillis: config.otelCollectIntervalMs,
|
|
432
|
+
});
|
|
433
|
+
}
|
|
434
|
+
|
|
435
|
+
const meterProvider = OpenTelemetryClient.createMeterProvider(resource, exporters);
|
|
271
436
|
const loggerProvider = registerOtelLoggerProvider(resource, config.logsCollectorUrl);
|
|
272
437
|
|
|
273
|
-
return new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
|
|
438
|
+
return new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, publicExporter, log);
|
|
274
439
|
};
|
|
275
440
|
}
|
|
276
441
|
|