@aztec/telemetry-client 0.0.1-commit.03f7ef2 → 0.0.1-commit.04d373f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/attributes.d.ts +14 -2
- package/dest/attributes.d.ts.map +1 -1
- package/dest/attributes.js +7 -1
- package/dest/bench.d.ts +3 -1
- package/dest/bench.d.ts.map +1 -1
- package/dest/bench.js +24 -14
- package/dest/config.d.ts +3 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -9
- package/dest/l1_metrics.d.ts +2 -2
- package/dest/l1_metrics.d.ts.map +1 -1
- package/dest/l1_metrics.js +4 -20
- package/dest/lmdb_metrics.d.ts +3 -3
- package/dest/lmdb_metrics.d.ts.map +1 -1
- package/dest/lmdb_metrics.js +4 -20
- package/dest/metric-utils.d.ts +24 -0
- package/dest/metric-utils.d.ts.map +1 -0
- package/dest/metric-utils.js +59 -0
- package/dest/metrics.d.ts +311 -223
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +1593 -221
- package/dest/monitored_batch_span_processor.d.ts +29 -0
- package/dest/monitored_batch_span_processor.d.ts.map +1 -0
- package/dest/monitored_batch_span_processor.js +70 -0
- package/dest/nodejs_metrics_monitor.d.ts +3 -5
- package/dest/nodejs_metrics_monitor.d.ts.map +1 -1
- package/dest/nodejs_metrics_monitor.js +19 -40
- package/dest/noop.d.ts +6 -3
- package/dest/noop.d.ts.map +1 -1
- package/dest/noop.js +32 -1
- package/dest/otel.d.ts +5 -3
- package/dest/otel.d.ts.map +1 -1
- package/dest/otel.js +113 -7
- package/dest/otel_propagation.d.ts +3 -1
- package/dest/otel_propagation.d.ts.map +1 -1
- package/dest/otel_propagation.js +49 -1
- package/dest/otel_resource.d.ts +1 -1
- package/dest/otel_resource.d.ts.map +1 -1
- package/dest/otel_resource.js +14 -2
- package/dest/prom_otel_adapter.d.ts +4 -4
- package/dest/prom_otel_adapter.d.ts.map +1 -1
- package/dest/prom_otel_adapter.js +19 -10
- package/dest/start.d.ts +3 -2
- package/dest/start.d.ts.map +1 -1
- package/dest/start.js +3 -3
- package/dest/telemetry.d.ts +38 -26
- package/dest/telemetry.d.ts.map +1 -1
- package/dest/telemetry.js +46 -23
- package/dest/wrappers/fetch.d.ts +3 -3
- package/dest/wrappers/fetch.d.ts.map +1 -1
- package/dest/wrappers/fetch.js +3 -2
- package/dest/wrappers/l2_block_stream.d.ts +2 -2
- package/dest/wrappers/l2_block_stream.d.ts.map +1 -1
- package/dest/wrappers/l2_block_stream.js +385 -11
- package/package.json +4 -4
- package/src/attributes.ts +19 -1
- package/src/bench.ts +27 -15
- package/src/config.ts +24 -9
- package/src/l1_metrics.ts +5 -20
- package/src/lmdb_metrics.ts +5 -26
- package/src/metric-utils.ts +75 -0
- package/src/metrics.ts +1655 -249
- package/src/monitored_batch_span_processor.ts +93 -0
- package/src/nodejs_metrics_monitor.ts +18 -51
- package/src/noop.ts +61 -3
- package/src/otel.ts +114 -8
- package/src/otel_propagation.ts +42 -1
- package/src/otel_resource.ts +19 -2
- package/src/prom_otel_adapter.ts +16 -23
- package/src/start.ts +12 -4
- package/src/telemetry.ts +105 -67
- package/src/wrappers/fetch.ts +9 -3
- package/src/wrappers/l2_block_stream.ts +1 -1
|
@@ -0,0 +1,93 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
|
|
3
|
+
import { type Context, SpanStatusCode } from '@opentelemetry/api';
|
|
4
|
+
import { hrTimeToMilliseconds } from '@opentelemetry/core';
|
|
5
|
+
import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
|
|
6
|
+
import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
|
|
7
|
+
|
|
8
|
+
/** Minimum interval between drop warnings to avoid log spam. */
|
|
9
|
+
const DROP_WARNING_INTERVAL_MS = 30_000;
|
|
10
|
+
|
|
11
|
+
const DEFAULT_MIN_TRACE_DURATION_MS = 10;
|
|
12
|
+
|
|
13
|
+
export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
|
|
14
|
+
minTraceDurationMs?: number;
|
|
15
|
+
};
|
|
16
|
+
|
|
17
|
+
/**
|
|
18
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
19
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
20
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
21
|
+
*/
|
|
22
|
+
export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
23
|
+
private readonly maxQueueSize: number;
|
|
24
|
+
private readonly minTraceDurationMs: number;
|
|
25
|
+
private readonly log: Logger;
|
|
26
|
+
|
|
27
|
+
private approxQueueSize = 0;
|
|
28
|
+
private droppedSinceLastWarning = 0;
|
|
29
|
+
private totalDropped = 0;
|
|
30
|
+
private lastWarningTime = 0;
|
|
31
|
+
|
|
32
|
+
constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig) {
|
|
33
|
+
const maxQueueSize = config?.maxQueueSize ?? 2048;
|
|
34
|
+
super(exporter, { ...config, maxQueueSize });
|
|
35
|
+
this.maxQueueSize = maxQueueSize;
|
|
36
|
+
this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
|
|
37
|
+
this.log = log;
|
|
38
|
+
}
|
|
39
|
+
|
|
40
|
+
override onStart(span: Span, parentContext: Context): void {
|
|
41
|
+
super.onStart(span, parentContext);
|
|
42
|
+
}
|
|
43
|
+
|
|
44
|
+
override onEnd(span: ReadableSpan): void {
|
|
45
|
+
if (this.shouldDropShortSpan(span)) {
|
|
46
|
+
return;
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
if (this.approxQueueSize >= this.maxQueueSize) {
|
|
50
|
+
this.droppedSinceLastWarning++;
|
|
51
|
+
this.totalDropped++;
|
|
52
|
+
this.maybeLogDropWarning();
|
|
53
|
+
} else {
|
|
54
|
+
this.approxQueueSize++;
|
|
55
|
+
}
|
|
56
|
+
super.onEnd(span);
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
override async forceFlush(): Promise<void> {
|
|
60
|
+
await super.forceFlush();
|
|
61
|
+
this.approxQueueSize = 0;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
override async shutdown(): Promise<void> {
|
|
65
|
+
if (this.totalDropped > 0) {
|
|
66
|
+
this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
|
|
67
|
+
totalDropped: this.totalDropped,
|
|
68
|
+
});
|
|
69
|
+
}
|
|
70
|
+
await super.shutdown();
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
private shouldDropShortSpan(span: ReadableSpan): boolean {
|
|
74
|
+
return (
|
|
75
|
+
this.minTraceDurationMs > 0 &&
|
|
76
|
+
span.status.code !== SpanStatusCode.ERROR &&
|
|
77
|
+
hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs
|
|
78
|
+
);
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
private maybeLogDropWarning(): void {
|
|
82
|
+
const now = Date.now();
|
|
83
|
+
if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
|
|
84
|
+
this.log.warn(
|
|
85
|
+
`BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` +
|
|
86
|
+
`${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`,
|
|
87
|
+
{ droppedSinceLastWarning: this.droppedSinceLastWarning, totalDropped: this.totalDropped },
|
|
88
|
+
);
|
|
89
|
+
this.droppedSinceLastWarning = 0;
|
|
90
|
+
this.lastWarningTime = now;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
}
|
|
@@ -2,18 +2,11 @@ import type { Observable } from '@opentelemetry/api';
|
|
|
2
2
|
import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
|
3
3
|
|
|
4
4
|
import * as Attributes from './attributes.js';
|
|
5
|
+
import { createUpDownCounterWithDefault } from './metric-utils.js';
|
|
5
6
|
import * as Metrics from './metrics.js';
|
|
6
|
-
import {
|
|
7
|
-
|
|
8
|
-
|
|
9
|
-
type ObservableGauge,
|
|
10
|
-
type UpDownCounter,
|
|
11
|
-
ValueType,
|
|
12
|
-
} from './telemetry.js';
|
|
13
|
-
|
|
14
|
-
/**
|
|
15
|
-
* Detector for custom Aztec attributes
|
|
16
|
-
*/
|
|
7
|
+
import type { BatchObservableResult, Meter, ObservableGauge, UpDownCounter } from './telemetry.js';
|
|
8
|
+
|
|
9
|
+
/** Monitors Node.js runtime metrics */
|
|
17
10
|
export class NodejsMetricsMonitor {
|
|
18
11
|
private eventLoopDelayGauges: {
|
|
19
12
|
min: ObservableGauge;
|
|
@@ -38,53 +31,27 @@ export class NodejsMetricsMonitor {
|
|
|
38
31
|
private eventLoopDelay: IntervalHistogram;
|
|
39
32
|
|
|
40
33
|
constructor(private meter: Meter) {
|
|
41
|
-
const nsObsGauge = (name: (typeof Metrics)[keyof typeof Metrics], description: string) =>
|
|
42
|
-
meter.createObservableGauge(name, {
|
|
43
|
-
unit: 'ns',
|
|
44
|
-
valueType: ValueType.INT,
|
|
45
|
-
description,
|
|
46
|
-
});
|
|
47
|
-
|
|
48
34
|
this.eventLoopDelayGauges = {
|
|
49
|
-
min:
|
|
50
|
-
mean:
|
|
51
|
-
max:
|
|
52
|
-
stddev:
|
|
53
|
-
p50:
|
|
54
|
-
p90:
|
|
55
|
-
p99:
|
|
35
|
+
min: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN),
|
|
36
|
+
mean: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN),
|
|
37
|
+
max: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX),
|
|
38
|
+
stddev: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV),
|
|
39
|
+
p50: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50),
|
|
40
|
+
p90: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90),
|
|
41
|
+
p99: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99),
|
|
56
42
|
};
|
|
57
43
|
|
|
58
|
-
this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION
|
|
59
|
-
|
|
60
|
-
|
|
44
|
+
this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
|
|
45
|
+
this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
|
|
46
|
+
[Attributes.NODEJS_EVENT_LOOP_STATE]: ['idle', 'active'],
|
|
61
47
|
});
|
|
62
|
-
|
|
63
|
-
this.eventLoopTime = meter.createUpDownCounter(Metrics.NODEJS_EVENT_LOOP_TIME, {
|
|
64
|
-
unit: 'ms',
|
|
65
|
-
valueType: ValueType.INT,
|
|
66
|
-
description: 'How much time the event loop has spent in a given state',
|
|
67
|
-
});
|
|
68
|
-
|
|
69
48
|
this.eventLoopDelay = monitorEventLoopDelay();
|
|
70
49
|
|
|
71
50
|
this.memoryGauges = {
|
|
72
|
-
heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE,
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL, {
|
|
77
|
-
unit: 'By',
|
|
78
|
-
description: 'The max size the V8 heap can grow to',
|
|
79
|
-
}),
|
|
80
|
-
arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE, {
|
|
81
|
-
unit: 'By',
|
|
82
|
-
description: 'Memory allocated for buffers (includes native memory used)',
|
|
83
|
-
}),
|
|
84
|
-
external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE, {
|
|
85
|
-
unit: 'By',
|
|
86
|
-
description: 'Memory allocated for native C++ objects',
|
|
87
|
-
}),
|
|
51
|
+
heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE),
|
|
52
|
+
heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL),
|
|
53
|
+
arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE),
|
|
54
|
+
external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE),
|
|
88
55
|
};
|
|
89
56
|
}
|
|
90
57
|
|
package/src/noop.ts
CHANGED
|
@@ -1,13 +1,63 @@
|
|
|
1
|
-
import { type
|
|
1
|
+
import { type Context, type Span, type SpanContext, type Tracer, createNoopMeter } from '@opentelemetry/api';
|
|
2
2
|
|
|
3
|
-
import type {
|
|
3
|
+
import type { MetricDefinition } from './metrics.js';
|
|
4
|
+
import type {
|
|
5
|
+
Gauge,
|
|
6
|
+
Histogram,
|
|
7
|
+
Meter,
|
|
8
|
+
ObservableGauge,
|
|
9
|
+
ObservableUpDownCounter,
|
|
10
|
+
TelemetryClient,
|
|
11
|
+
UpDownCounter,
|
|
12
|
+
} from './telemetry.js';
|
|
13
|
+
|
|
14
|
+
/** A no-op meter that implements our custom Meter interface */
|
|
15
|
+
class NoopMeter implements Meter {
|
|
16
|
+
private otelMeter = createNoopMeter();
|
|
17
|
+
|
|
18
|
+
createGauge(_metric: MetricDefinition): Gauge {
|
|
19
|
+
return this.otelMeter.createGauge('');
|
|
20
|
+
}
|
|
21
|
+
|
|
22
|
+
createObservableGauge(_metric: MetricDefinition): ObservableGauge {
|
|
23
|
+
return this.otelMeter.createObservableGauge('');
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
createHistogram(_metric: MetricDefinition, _extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
|
|
27
|
+
return this.otelMeter.createHistogram('');
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
createUpDownCounter(_metric: MetricDefinition): UpDownCounter {
|
|
31
|
+
return this.otelMeter.createUpDownCounter('');
|
|
32
|
+
}
|
|
33
|
+
|
|
34
|
+
createObservableUpDownCounter(_metric: MetricDefinition): ObservableUpDownCounter {
|
|
35
|
+
return this.otelMeter.createObservableUpDownCounter('');
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
addBatchObservableCallback(
|
|
39
|
+
callback: Parameters<Meter['addBatchObservableCallback']>[0],
|
|
40
|
+
observables: Parameters<Meter['addBatchObservableCallback']>[1],
|
|
41
|
+
): void {
|
|
42
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
43
|
+
}
|
|
44
|
+
|
|
45
|
+
removeBatchObservableCallback(
|
|
46
|
+
callback: Parameters<Meter['removeBatchObservableCallback']>[0],
|
|
47
|
+
observables: Parameters<Meter['removeBatchObservableCallback']>[1],
|
|
48
|
+
): void {
|
|
49
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
50
|
+
}
|
|
51
|
+
}
|
|
4
52
|
|
|
5
53
|
export class NoopTelemetryClient implements TelemetryClient {
|
|
54
|
+
private meter = new NoopMeter();
|
|
55
|
+
|
|
6
56
|
setExportedPublicTelemetry(_prefixes: string[]): void {}
|
|
7
57
|
setPublicTelemetryCollectFrom(_roles: string[]): void {}
|
|
8
58
|
|
|
9
59
|
getMeter(): Meter {
|
|
10
|
-
return
|
|
60
|
+
return this.meter;
|
|
11
61
|
}
|
|
12
62
|
|
|
13
63
|
getTracer(): Tracer {
|
|
@@ -25,6 +75,14 @@ export class NoopTelemetryClient implements TelemetryClient {
|
|
|
25
75
|
isEnabled() {
|
|
26
76
|
return false;
|
|
27
77
|
}
|
|
78
|
+
|
|
79
|
+
getTraceContext(): string | undefined {
|
|
80
|
+
return undefined;
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
extractPropagatedContext(_traceContext: string): Context | undefined {
|
|
84
|
+
return undefined;
|
|
85
|
+
}
|
|
28
86
|
}
|
|
29
87
|
|
|
30
88
|
// @opentelemetry/api internally uses NoopTracer and NoopSpan but they're not exported
|
package/src/otel.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { type LogData, type Logger, addLogDataHandler } from '@aztec/foundation/log';
|
|
2
2
|
|
|
3
3
|
import {
|
|
4
|
+
type Context,
|
|
4
5
|
DiagConsoleLogger,
|
|
5
6
|
DiagLogLevel,
|
|
6
|
-
type Meter,
|
|
7
|
+
type Meter as OtelMeter,
|
|
8
|
+
ROOT_CONTEXT,
|
|
7
9
|
type Tracer,
|
|
8
10
|
type TracerProvider,
|
|
9
11
|
context,
|
|
10
12
|
diag,
|
|
11
13
|
isSpanContextValid,
|
|
14
|
+
propagation,
|
|
12
15
|
trace,
|
|
13
16
|
} from '@opentelemetry/api';
|
|
17
|
+
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
|
14
18
|
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
15
19
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
16
20
|
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
@@ -24,22 +28,72 @@ import {
|
|
|
24
28
|
type PeriodicExportingMetricReaderOptions,
|
|
25
29
|
View,
|
|
26
30
|
} from '@opentelemetry/sdk-metrics';
|
|
27
|
-
import {
|
|
31
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
28
32
|
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
29
33
|
|
|
30
34
|
import type { TelemetryClientConfig } from './config.js';
|
|
35
|
+
import { toMetricOptions } from './metric-utils.js';
|
|
36
|
+
import type { MetricDefinition } from './metrics.js';
|
|
37
|
+
import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
|
|
31
38
|
import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
|
|
32
39
|
import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
33
40
|
import { registerOtelLoggerProvider } from './otel_logger_provider.js';
|
|
34
41
|
import { getOtelResource } from './otel_resource.js';
|
|
35
|
-
import type {
|
|
42
|
+
import type {
|
|
43
|
+
Gauge,
|
|
44
|
+
Histogram,
|
|
45
|
+
Meter,
|
|
46
|
+
ObservableGauge,
|
|
47
|
+
ObservableUpDownCounter,
|
|
48
|
+
TelemetryClient,
|
|
49
|
+
UpDownCounter,
|
|
50
|
+
} from './telemetry.js';
|
|
51
|
+
|
|
52
|
+
/** Wraps an OpenTelemetry Meter to implement our custom Meter interface */
|
|
53
|
+
class WrappedMeter implements Meter {
|
|
54
|
+
constructor(private otelMeter: OtelMeter) {}
|
|
55
|
+
|
|
56
|
+
createGauge(metric: MetricDefinition): Gauge {
|
|
57
|
+
return this.otelMeter.createGauge(metric.name, toMetricOptions(metric));
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
createObservableGauge(metric: MetricDefinition): ObservableGauge {
|
|
61
|
+
return this.otelMeter.createObservableGauge(metric.name, toMetricOptions(metric));
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
createHistogram(metric: MetricDefinition, extraOptions?: Parameters<Meter['createHistogram']>[1]): Histogram {
|
|
65
|
+
return this.otelMeter.createHistogram(metric.name, { ...toMetricOptions(metric), ...extraOptions });
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
createUpDownCounter(metric: MetricDefinition): UpDownCounter {
|
|
69
|
+
return this.otelMeter.createUpDownCounter(metric.name, toMetricOptions(metric));
|
|
70
|
+
}
|
|
71
|
+
|
|
72
|
+
createObservableUpDownCounter(metric: MetricDefinition): ObservableUpDownCounter {
|
|
73
|
+
return this.otelMeter.createObservableUpDownCounter(metric.name, toMetricOptions(metric));
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
addBatchObservableCallback(
|
|
77
|
+
callback: Parameters<Meter['addBatchObservableCallback']>[0],
|
|
78
|
+
observables: Parameters<Meter['addBatchObservableCallback']>[1],
|
|
79
|
+
): void {
|
|
80
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
81
|
+
}
|
|
82
|
+
|
|
83
|
+
removeBatchObservableCallback(
|
|
84
|
+
callback: Parameters<Meter['removeBatchObservableCallback']>[0],
|
|
85
|
+
observables: Parameters<Meter['removeBatchObservableCallback']>[1],
|
|
86
|
+
): void {
|
|
87
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
88
|
+
}
|
|
89
|
+
}
|
|
36
90
|
|
|
37
91
|
export type OpenTelemetryClientFactory = (resource: IResource, log: Logger) => OpenTelemetryClient;
|
|
38
92
|
|
|
39
93
|
export class OpenTelemetryClient implements TelemetryClient {
|
|
40
94
|
hostMetrics: HostMetrics | undefined;
|
|
41
95
|
nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
|
|
42
|
-
private meters: Map<string,
|
|
96
|
+
private meters: Map<string, WrappedMeter> = new Map<string, WrappedMeter>();
|
|
43
97
|
private tracers: Map<string, Tracer> = new Map<string, Tracer>();
|
|
44
98
|
|
|
45
99
|
protected constructor(
|
|
@@ -62,7 +116,8 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
62
116
|
getMeter(name: string): Meter {
|
|
63
117
|
let meter = this.meters.get(name);
|
|
64
118
|
if (!meter) {
|
|
65
|
-
|
|
119
|
+
const otelMeter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION] as string);
|
|
120
|
+
meter = new WrappedMeter(otelMeter);
|
|
66
121
|
this.meters.set(name, meter);
|
|
67
122
|
}
|
|
68
123
|
return meter;
|
|
@@ -100,9 +155,10 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
100
155
|
meterProvider: this.meterProvider,
|
|
101
156
|
});
|
|
102
157
|
|
|
103
|
-
|
|
158
|
+
const nodejsMeter = new WrappedMeter(
|
|
104
159
|
this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME] as string),
|
|
105
160
|
);
|
|
161
|
+
this.nodejsMetricsMonitor = new NodejsMetricsMonitor(nodejsMeter);
|
|
106
162
|
|
|
107
163
|
this.hostMetrics.start();
|
|
108
164
|
this.nodejsMetricsMonitor.start();
|
|
@@ -138,6 +194,19 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
138
194
|
]);
|
|
139
195
|
}
|
|
140
196
|
|
|
197
|
+
public getTraceContext(): string | undefined {
|
|
198
|
+
const carrier: Record<string, string> = {};
|
|
199
|
+
propagation.inject(context.active(), carrier);
|
|
200
|
+
return carrier['traceparent'];
|
|
201
|
+
}
|
|
202
|
+
|
|
203
|
+
public extractPropagatedContext(traceContext: string): Context {
|
|
204
|
+
const extractedContext = propagation.extract(ROOT_CONTEXT, {
|
|
205
|
+
traceparent: traceContext,
|
|
206
|
+
});
|
|
207
|
+
return extractedContext;
|
|
208
|
+
}
|
|
209
|
+
|
|
141
210
|
public static createMeterProvider(
|
|
142
211
|
resource: IResource,
|
|
143
212
|
exporters: Array<PeriodicExportingMetricReaderOptions>,
|
|
@@ -266,6 +335,36 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
266
335
|
true,
|
|
267
336
|
),
|
|
268
337
|
}),
|
|
338
|
+
// L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
|
|
339
|
+
new View({
|
|
340
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
341
|
+
instrumentUnit: 'gwei',
|
|
342
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
343
|
+
[0.1, 0.5, 1, 2, 5, 10, 20, 50, 100, 200, 500, 1_000],
|
|
344
|
+
true,
|
|
345
|
+
),
|
|
346
|
+
}),
|
|
347
|
+
// L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
|
|
348
|
+
new View({
|
|
349
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
350
|
+
instrumentUnit: 'gas',
|
|
351
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
352
|
+
[
|
|
353
|
+
10_000, 50_000, 100_000, 250_000, 500_000, 1_000_000, 2_000_000, 5_000_000, 10_000_000, 15_000_000,
|
|
354
|
+
30_000_000,
|
|
355
|
+
],
|
|
356
|
+
true,
|
|
357
|
+
),
|
|
358
|
+
}),
|
|
359
|
+
// L1 tx total fee in ETH: typically 0.001 - 1 ETH
|
|
360
|
+
new View({
|
|
361
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
362
|
+
instrumentUnit: 'eth',
|
|
363
|
+
aggregation: new ExplicitBucketHistogramAggregation(
|
|
364
|
+
[0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05, 0.1, 0.5, 1, 5, 10],
|
|
365
|
+
true,
|
|
366
|
+
),
|
|
367
|
+
}),
|
|
269
368
|
],
|
|
270
369
|
});
|
|
271
370
|
}
|
|
@@ -275,11 +374,18 @@ export class OpenTelemetryClient implements TelemetryClient {
|
|
|
275
374
|
const tracerProvider = new NodeTracerProvider({
|
|
276
375
|
resource,
|
|
277
376
|
spanProcessors: config.tracesCollectorUrl
|
|
278
|
-
? [
|
|
377
|
+
? [
|
|
378
|
+
new MonitoredBatchSpanProcessor(new OTLPTraceExporter({ url: config.tracesCollectorUrl.href }), log, {
|
|
379
|
+
maxQueueSize: config.otelBspMaxQueueSize,
|
|
380
|
+
minTraceDurationMs: config.otelMinTraceDurationMs,
|
|
381
|
+
}),
|
|
382
|
+
]
|
|
279
383
|
: [],
|
|
280
384
|
});
|
|
281
385
|
|
|
282
|
-
tracerProvider.register(
|
|
386
|
+
tracerProvider.register({
|
|
387
|
+
propagator: new W3CTraceContextPropagator(),
|
|
388
|
+
});
|
|
283
389
|
|
|
284
390
|
const exporters: PeriodicExportingMetricReaderOptions[] = [];
|
|
285
391
|
if (config.metricsCollectorUrl) {
|
package/src/otel_propagation.ts
CHANGED
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
|
|
2
|
+
|
|
1
3
|
import { ROOT_CONTEXT, type Span, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
|
|
2
4
|
import type Koa from 'koa';
|
|
3
5
|
|
|
@@ -17,7 +19,7 @@ export function getOtelJsonRpcPropagationMiddleware(
|
|
|
17
19
|
const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
|
|
18
20
|
const method = (ctx.request.body as any)?.method;
|
|
19
21
|
return tracer.startActiveSpan(
|
|
20
|
-
`JsonRpcServer.${method ?? '
|
|
22
|
+
`JsonRpcServer.${method ?? 'batch'}`,
|
|
21
23
|
{ kind: SpanKind.SERVER },
|
|
22
24
|
context,
|
|
23
25
|
async (span: Span): Promise<void> => {
|
|
@@ -48,3 +50,42 @@ export function getOtelJsonRpcPropagationMiddleware(
|
|
|
48
50
|
);
|
|
49
51
|
};
|
|
50
52
|
}
|
|
53
|
+
|
|
54
|
+
export function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware {
|
|
55
|
+
return function otelJsonRpcDiagnostics(ctx, next) {
|
|
56
|
+
const [namespace, method] = splitNamespace(ctx.method);
|
|
57
|
+
const scope = namespace ?? 'UnknownHandler';
|
|
58
|
+
const tracer = getTelemetryClient().getTracer(scope);
|
|
59
|
+
return tracer.startActiveSpan(
|
|
60
|
+
`${scope}.${method}`,
|
|
61
|
+
{ kind: SpanKind.INTERNAL, attributes: { [ATTR_JSONRPC_METHOD]: ctx.method } },
|
|
62
|
+
async span => {
|
|
63
|
+
if (ctx.id !== null) {
|
|
64
|
+
span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
|
|
65
|
+
}
|
|
66
|
+
|
|
67
|
+
try {
|
|
68
|
+
await next();
|
|
69
|
+
span.setStatus({ code: SpanStatusCode.OK });
|
|
70
|
+
} catch (err) {
|
|
71
|
+
span.setStatus({ code: SpanStatusCode.ERROR, message: err instanceof Error ? err.message : String(err) });
|
|
72
|
+
if (typeof err === 'string' || err instanceof Error) {
|
|
73
|
+
span.recordException(err);
|
|
74
|
+
}
|
|
75
|
+
throw err;
|
|
76
|
+
} finally {
|
|
77
|
+
span.end();
|
|
78
|
+
}
|
|
79
|
+
},
|
|
80
|
+
);
|
|
81
|
+
};
|
|
82
|
+
}
|
|
83
|
+
|
|
84
|
+
function splitNamespace(fullMethod: string): [namespace: string | undefined, method: string] {
|
|
85
|
+
const idx = fullMethod.indexOf('_');
|
|
86
|
+
if (idx > -1) {
|
|
87
|
+
return [fullMethod.slice(0, idx), fullMethod.slice(idx + 1)];
|
|
88
|
+
} else {
|
|
89
|
+
return [undefined, fullMethod];
|
|
90
|
+
}
|
|
91
|
+
}
|
package/src/otel_resource.ts
CHANGED
|
@@ -7,10 +7,26 @@ import {
|
|
|
7
7
|
osDetectorSync,
|
|
8
8
|
serviceInstanceIdDetectorSync,
|
|
9
9
|
} from '@opentelemetry/resources';
|
|
10
|
-
import {
|
|
10
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
11
|
+
import { readFileSync } from 'fs';
|
|
12
|
+
import { dirname, resolve } from 'path';
|
|
13
|
+
import { fileURLToPath } from 'url';
|
|
11
14
|
|
|
12
15
|
import { AZTEC_NODE_ROLE, AZTEC_REGISTRY_ADDRESS, AZTEC_ROLLUP_ADDRESS, AZTEC_ROLLUP_VERSION } from './attributes.js';
|
|
13
16
|
|
|
17
|
+
/** Reads the Aztec client version from the release manifest. */
|
|
18
|
+
function getAztecVersion(): string | undefined {
|
|
19
|
+
try {
|
|
20
|
+
const releasePleasePath = resolve(
|
|
21
|
+
dirname(fileURLToPath(import.meta.url)),
|
|
22
|
+
'../../../.release-please-manifest.json',
|
|
23
|
+
);
|
|
24
|
+
return JSON.parse(readFileSync(releasePleasePath, 'utf-8'))['.'];
|
|
25
|
+
} catch {
|
|
26
|
+
return undefined;
|
|
27
|
+
}
|
|
28
|
+
}
|
|
29
|
+
|
|
14
30
|
export function getOtelResource(): IResource {
|
|
15
31
|
const resource = detectResourcesSync({
|
|
16
32
|
detectors: [
|
|
@@ -42,7 +58,8 @@ const aztecNetworkDetectorSync: DetectorSync = {
|
|
|
42
58
|
}
|
|
43
59
|
const aztecAttributes = {
|
|
44
60
|
// this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)
|
|
45
|
-
[
|
|
61
|
+
[ATTR_SERVICE_NAME]: role ? `aztec-${role}` : undefined,
|
|
62
|
+
[ATTR_SERVICE_VERSION]: getAztecVersion(),
|
|
46
63
|
[AZTEC_NODE_ROLE]: role,
|
|
47
64
|
[AZTEC_ROLLUP_VERSION]: process.env.ROLLUP_VERSION ?? 'canonical',
|
|
48
65
|
[AZTEC_ROLLUP_ADDRESS]: process.env.ROLLUP_CONTRACT_ADDRESS,
|
package/src/prom_otel_adapter.ts
CHANGED
|
@@ -1,9 +1,10 @@
|
|
|
1
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
1
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
2
2
|
import { Timer } from '@aztec/foundation/timer';
|
|
3
3
|
|
|
4
4
|
import { Registry } from 'prom-client';
|
|
5
5
|
|
|
6
|
-
import type {
|
|
6
|
+
import type { MetricDefinition } from './metrics.js';
|
|
7
|
+
import type { Histogram, Meter, ObservableGauge, TelemetryClient } from './telemetry.js';
|
|
7
8
|
|
|
8
9
|
/**
|
|
9
10
|
* Types matching the gossipsub and libp2p services
|
|
@@ -94,9 +95,8 @@ export class OtelGauge<Labels extends LabelsGeneric = NoLabels> implements IGaug
|
|
|
94
95
|
help: string,
|
|
95
96
|
private labelNames: Array<keyof Labels> = [],
|
|
96
97
|
) {
|
|
97
|
-
|
|
98
|
-
|
|
99
|
-
});
|
|
98
|
+
const metricDef: MetricDefinition = { name, description: help };
|
|
99
|
+
this.gauge = meter.createObservableGauge(metricDef);
|
|
100
100
|
|
|
101
101
|
// Only observe in the callback when collect() is called
|
|
102
102
|
this.gauge.addCallback(this.handleObservation.bind(this));
|
|
@@ -214,8 +214,8 @@ export class OtelHistogram<Labels extends LabelsGeneric = NoLabels> implements I
|
|
|
214
214
|
buckets: number[] = [],
|
|
215
215
|
private labelNames: Array<keyof Labels> = [],
|
|
216
216
|
) {
|
|
217
|
-
|
|
218
|
-
|
|
217
|
+
const metricDef: MetricDefinition = { name, description: help };
|
|
218
|
+
this.histogram = meter.createHistogram(metricDef, {
|
|
219
219
|
advice: buckets.length ? { explicitBucketBoundaries: buckets } : undefined,
|
|
220
220
|
});
|
|
221
221
|
}
|
|
@@ -289,15 +289,9 @@ export class OtelAvgMinMax<Labels extends LabelsGeneric = NoLabels> implements I
|
|
|
289
289
|
) {
|
|
290
290
|
// Create three separate gauges for avg, min, and max
|
|
291
291
|
this.gauges = {
|
|
292
|
-
avg: meter.createObservableGauge(`${name}_avg`
|
|
293
|
-
|
|
294
|
-
}),
|
|
295
|
-
min: meter.createObservableGauge(`${name}_min` as MetricsType, {
|
|
296
|
-
description: `${help} (minimum)`,
|
|
297
|
-
}),
|
|
298
|
-
max: meter.createObservableGauge(`${name}_max` as MetricsType, {
|
|
299
|
-
description: `${help} (maximum)`,
|
|
300
|
-
}),
|
|
292
|
+
avg: meter.createObservableGauge({ name: `${name}_avg`, description: `${help} (average)` }),
|
|
293
|
+
min: meter.createObservableGauge({ name: `${name}_min`, description: `${help} (minimum)` }),
|
|
294
|
+
max: meter.createObservableGauge({ name: `${name}_max`, description: `${help} (maximum)` }),
|
|
301
295
|
};
|
|
302
296
|
|
|
303
297
|
// Register callbacks for each gauge
|
|
@@ -416,12 +410,11 @@ function parseLabelsSafely<Labels extends LabelsGeneric>(labelStr: string, logge
|
|
|
416
410
|
*/
|
|
417
411
|
export class OtelMetricsAdapter extends Registry implements MetricsRegister {
|
|
418
412
|
private readonly meter: Meter;
|
|
413
|
+
private logger: Logger;
|
|
419
414
|
|
|
420
|
-
constructor(
|
|
421
|
-
telemetryClient: TelemetryClient,
|
|
422
|
-
private logger: Logger = createLogger('telemetry:otel-metrics-adapter'),
|
|
423
|
-
) {
|
|
415
|
+
constructor(telemetryClient: TelemetryClient, bindings?: LoggerBindings) {
|
|
424
416
|
super();
|
|
417
|
+
this.logger = createLogger('telemetry:otel-metrics-adapter', bindings);
|
|
425
418
|
this.meter = telemetryClient.getMeter('metrics-adapter');
|
|
426
419
|
}
|
|
427
420
|
|
|
@@ -429,7 +422,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister {
|
|
|
429
422
|
return new OtelGauge<Labels>(
|
|
430
423
|
this.logger,
|
|
431
424
|
this.meter,
|
|
432
|
-
configuration.name
|
|
425
|
+
configuration.name,
|
|
433
426
|
configuration.help,
|
|
434
427
|
configuration.labelNames,
|
|
435
428
|
);
|
|
@@ -439,7 +432,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister {
|
|
|
439
432
|
return new OtelHistogram<Labels>(
|
|
440
433
|
this.logger,
|
|
441
434
|
this.meter,
|
|
442
|
-
configuration.name
|
|
435
|
+
configuration.name,
|
|
443
436
|
configuration.help,
|
|
444
437
|
configuration.buckets,
|
|
445
438
|
configuration.labelNames,
|
|
@@ -450,7 +443,7 @@ export class OtelMetricsAdapter extends Registry implements MetricsRegister {
|
|
|
450
443
|
return new OtelAvgMinMax<Labels>(
|
|
451
444
|
this.logger,
|
|
452
445
|
this.meter,
|
|
453
|
-
configuration.name
|
|
446
|
+
configuration.name,
|
|
454
447
|
configuration.help,
|
|
455
448
|
configuration.labelNames,
|
|
456
449
|
);
|