@aztec/telemetry-client 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/attributes.d.ts +24 -2
- package/dest/attributes.d.ts.map +1 -1
- package/dest/attributes.js +12 -1
- package/dest/bench.d.ts +3 -1
- package/dest/bench.d.ts.map +1 -1
- package/dest/bench.js +24 -14
- package/dest/config.d.ts +4 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +24 -10
- package/dest/index.d.ts +1 -1
- package/dest/l1_metrics.d.ts +2 -2
- package/dest/l1_metrics.d.ts.map +1 -1
- package/dest/l1_metrics.js +4 -20
- package/dest/lmdb_metrics.d.ts +3 -3
- package/dest/lmdb_metrics.d.ts.map +1 -1
- package/dest/lmdb_metrics.js +4 -20
- package/dest/metric-utils.d.ts +24 -0
- package/dest/metric-utils.d.ts.map +1 -0
- package/dest/metric-utils.js +59 -0
- package/dest/metrics.d.ts +312 -196
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +1599 -194
- package/dest/monitored_batch_span_processor.d.ts +29 -0
- package/dest/monitored_batch_span_processor.d.ts.map +1 -0
- package/dest/monitored_batch_span_processor.js +75 -0
- package/dest/nodejs_metrics_monitor.d.ts +3 -5
- package/dest/nodejs_metrics_monitor.d.ts.map +1 -1
- package/dest/nodejs_metrics_monitor.js +19 -40
- package/dest/noop.d.ts +6 -3
- package/dest/noop.d.ts.map +1 -1
- package/dest/noop.js +32 -1
- package/dest/otel.d.ts +10 -3
- package/dest/otel.d.ts.map +1 -1
- package/dest/otel.js +139 -9
- package/dest/otel_filter_metric_exporter.d.ts +1 -1
- package/dest/otel_filter_metric_exporter.d.ts.map +1 -1
- package/dest/otel_logger_provider.d.ts +1 -1
- package/dest/otel_propagation.d.ts +3 -1
- package/dest/otel_propagation.d.ts.map +1 -1
- package/dest/otel_propagation.js +49 -1
- package/dest/otel_resource.d.ts +1 -1
- package/dest/otel_resource.d.ts.map +1 -1
- package/dest/otel_resource.js +14 -2
- package/dest/prom_otel_adapter.d.ts +4 -4
- package/dest/prom_otel_adapter.d.ts.map +1 -1
- package/dest/prom_otel_adapter.js +19 -10
- package/dest/start.d.ts +3 -2
- package/dest/start.d.ts.map +1 -1
- package/dest/start.js +5 -4
- package/dest/telemetry.d.ts +38 -26
- package/dest/telemetry.d.ts.map +1 -1
- package/dest/telemetry.js +46 -23
- package/dest/vendor/attributes.d.ts +1 -1
- package/dest/vendor/otel-pino-stream.d.ts +1 -1
- package/dest/vendor/otel-pino-stream.d.ts.map +1 -1
- package/dest/with_tracer.d.ts +1 -1
- package/dest/with_tracer.d.ts.map +1 -1
- package/dest/wrappers/fetch.d.ts +3 -3
- package/dest/wrappers/fetch.d.ts.map +1 -1
- package/dest/wrappers/fetch.js +3 -2
- package/dest/wrappers/index.d.ts +1 -1
- package/dest/wrappers/json_rpc_server.d.ts +1 -1
- package/dest/wrappers/l2_block_stream.d.ts +4 -3
- package/dest/wrappers/l2_block_stream.d.ts.map +1 -1
- package/dest/wrappers/l2_block_stream.js +385 -11
- package/package.json +9 -6
- package/src/attributes.ts +32 -1
- package/src/bench.ts +27 -15
- package/src/config.ts +38 -10
- package/src/l1_metrics.ts +5 -20
- package/src/lmdb_metrics.ts +5 -26
- package/src/metric-utils.ts +75 -0
- package/src/metrics.ts +1682 -237
- package/src/monitored_batch_span_processor.ts +100 -0
- package/src/nodejs_metrics_monitor.ts +18 -51
- package/src/noop.ts +61 -3
- package/src/otel.ts +144 -11
- package/src/otel_propagation.ts +42 -1
- package/src/otel_resource.ts +19 -2
- package/src/prom_otel_adapter.ts +16 -23
- package/src/start.ts +14 -5
- package/src/telemetry.ts +105 -67
- package/src/wrappers/fetch.ts +9 -3
- package/src/wrappers/l2_block_stream.ts +3 -2
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { type Context } from '@opentelemetry/api';
|
|
3
|
+
import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
|
|
4
|
+
import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
|
|
5
|
+
export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
|
|
6
|
+
minTraceDurationMs?: number;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
10
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
11
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
12
|
+
*/
|
|
13
|
+
export declare class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
14
|
+
private readonly maxQueueSize;
|
|
15
|
+
private readonly minTraceDurationMs;
|
|
16
|
+
private readonly log;
|
|
17
|
+
private approxQueueSize;
|
|
18
|
+
private droppedSinceLastWarning;
|
|
19
|
+
private totalDropped;
|
|
20
|
+
private lastWarningTime;
|
|
21
|
+
constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig);
|
|
22
|
+
onStart(span: Span, parentContext: Context): void;
|
|
23
|
+
onEnd(span: ReadableSpan): void;
|
|
24
|
+
forceFlush(): Promise<void>;
|
|
25
|
+
shutdown(): Promise<void>;
|
|
26
|
+
private shouldDropShortSpan;
|
|
27
|
+
private maybeLogDropWarning;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXBELE9BQU8sRUFBRSxLQUFLLE9BQU8sRUFBa0IsTUFBTSxvQkFBb0IsQ0FBQztBQUVsRSxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUNsRSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxJQUFJLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQWFwSCxNQUFNLE1BQU0saUNBQWlDLEdBQUcsWUFBWSxHQUFHO0lBQzdELGtCQUFrQixDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQzdCLENBQUM7QUFFRjs7OztHQUlHO0FBQ0gscUJBQWEsMkJBQTRCLFNBQVEsa0JBQWtCO0lBQ2pFLE9BQU8sQ0FBQyxRQUFRLENBQUMsWUFBWSxDQUFTO0lBQ3RDLE9BQU8sQ0FBQyxRQUFRLENBQUMsa0JBQWtCLENBQVM7SUFDNUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFFN0IsT0FBTyxDQUFDLGVBQWUsQ0FBSztJQUM1QixPQUFPLENBQUMsdUJBQXVCLENBQUs7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBSztJQUN6QixPQUFPLENBQUMsZUFBZSxDQUFLO0lBRTVCLFlBQVksUUFBUSxFQUFFLFlBQVksRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxFQUFFLGlDQUFpQyxFQU8xRjtJQUVRLE9BQU8sQ0FBQyxJQUFJLEVBQUUsSUFBSSxFQUFFLGFBQWEsRUFBRSxPQUFPLEdBQUcsSUFBSSxDQUV6RDtJQUVRLEtBQUssQ0FBQyxJQUFJLEVBQUUsWUFBWSxHQUFHLElBQUksQ0FhdkM7SUFFYyxVQUFVLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUd6QztJQUVjLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBT3ZDO0lBRUQsT0FBTyxDQUFDLG1CQUFtQjtJQVEzQixPQUFPLENBQUMsbUJBQW1CO0NBWTVCIn0=
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"monitored_batch_span_processor.d.ts","sourceRoot":"","sources":["../src/monitored_batch_span_processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,EAAE,KAAK,OAAO,EAAkB,MAAM,oBAAoB,CAAC;AAElE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,KAAK,IAAI,EAAE,MAAM,+BAA+B,CAAC;AAapH,MAAM,MAAM,iCAAiC,GAAG,YAAY,GAAG;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,kBAAkB;IACjE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAK;IAE5B,YAAY,QAAQ,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,iCAAiC,EAO1F;IAEQ,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,GAAG,IAAI,CAEzD;IAEQ,KAAK,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI,CAavC;IAEc,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzC;IAEc,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAOvC;IAED,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;CAY5B"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { SpanStatusCode } from '@opentelemetry/api';
|
|
2
|
+
import { hrTimeToMilliseconds } from '@opentelemetry/core';
|
|
3
|
+
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
|
|
4
|
+
/** Minimum interval between drop warnings to avoid log spam. */ const DROP_WARNING_INTERVAL_MS = 30_000;
|
|
5
|
+
const DEFAULT_MIN_TRACE_DURATION_MS = 10;
|
|
6
|
+
const DEFAULT_MAX_QUEUE_SIZE = 16384;
|
|
7
|
+
/** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
|
|
8
|
+
* at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */ const DEFAULT_MAX_EXPORT_BATCH_SIZE = 2048;
|
|
9
|
+
/**
|
|
10
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
11
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
12
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
13
|
+
*/ export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
14
|
+
maxQueueSize;
|
|
15
|
+
minTraceDurationMs;
|
|
16
|
+
log;
|
|
17
|
+
approxQueueSize = 0;
|
|
18
|
+
droppedSinceLastWarning = 0;
|
|
19
|
+
totalDropped = 0;
|
|
20
|
+
lastWarningTime = 0;
|
|
21
|
+
constructor(exporter, log, config){
|
|
22
|
+
const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
|
|
23
|
+
const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
|
|
24
|
+
super(exporter, {
|
|
25
|
+
...config,
|
|
26
|
+
maxQueueSize,
|
|
27
|
+
maxExportBatchSize
|
|
28
|
+
});
|
|
29
|
+
this.maxQueueSize = maxQueueSize;
|
|
30
|
+
this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
|
|
31
|
+
this.log = log;
|
|
32
|
+
}
|
|
33
|
+
onStart(span, parentContext) {
|
|
34
|
+
super.onStart(span, parentContext);
|
|
35
|
+
}
|
|
36
|
+
onEnd(span) {
|
|
37
|
+
if (this.shouldDropShortSpan(span)) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (this.approxQueueSize >= this.maxQueueSize) {
|
|
41
|
+
this.droppedSinceLastWarning++;
|
|
42
|
+
this.totalDropped++;
|
|
43
|
+
this.maybeLogDropWarning();
|
|
44
|
+
} else {
|
|
45
|
+
this.approxQueueSize++;
|
|
46
|
+
}
|
|
47
|
+
super.onEnd(span);
|
|
48
|
+
}
|
|
49
|
+
async forceFlush() {
|
|
50
|
+
await super.forceFlush();
|
|
51
|
+
this.approxQueueSize = 0;
|
|
52
|
+
}
|
|
53
|
+
async shutdown() {
|
|
54
|
+
if (this.totalDropped > 0) {
|
|
55
|
+
this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
|
|
56
|
+
totalDropped: this.totalDropped
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
await super.shutdown();
|
|
60
|
+
}
|
|
61
|
+
shouldDropShortSpan(span) {
|
|
62
|
+
return this.minTraceDurationMs > 0 && span.status.code !== SpanStatusCode.ERROR && hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs;
|
|
63
|
+
}
|
|
64
|
+
maybeLogDropWarning() {
|
|
65
|
+
const now = Date.now();
|
|
66
|
+
if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
|
|
67
|
+
this.log.warn(`BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` + `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`, {
|
|
68
|
+
droppedSinceLastWarning: this.droppedSinceLastWarning,
|
|
69
|
+
totalDropped: this.totalDropped
|
|
70
|
+
});
|
|
71
|
+
this.droppedSinceLastWarning = 0;
|
|
72
|
+
this.lastWarningTime = now;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
|
@@ -1,7 +1,5 @@
|
|
|
1
|
-
import {
|
|
2
|
-
/**
|
|
3
|
-
* Detector for custom Aztec attributes
|
|
4
|
-
*/
|
|
1
|
+
import type { Meter } from './telemetry.js';
|
|
2
|
+
/** Monitors Node.js runtime metrics */
|
|
5
3
|
export declare class NodejsMetricsMonitor {
|
|
6
4
|
private meter;
|
|
7
5
|
private eventLoopDelayGauges;
|
|
@@ -18,4 +16,4 @@ export declare class NodejsMetricsMonitor {
|
|
|
18
16
|
private measureMemoryUsage;
|
|
19
17
|
private measureEventLoopDelay;
|
|
20
18
|
}
|
|
21
|
-
//# sourceMappingURL=
|
|
19
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9kZWpzX21ldHJpY3NfbW9uaXRvci5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL25vZGVqc19tZXRyaWNzX21vbml0b3IudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBTUEsT0FBTyxLQUFLLEVBQXlCLEtBQUssRUFBa0MsTUFBTSxnQkFBZ0IsQ0FBQztBQUVuRyx1Q0FBdUM7QUFDdkMscUJBQWEsb0JBQW9CO0lBdUJuQixPQUFPLENBQUMsS0FBSztJQXRCekIsT0FBTyxDQUFDLG9CQUFvQixDQVExQjtJQUlGLE9BQU8sQ0FBQyxZQUFZLENBQW9FO0lBRXhGLE9BQU8sQ0FBQyxtQkFBbUIsQ0FBa0I7SUFDN0MsT0FBTyxDQUFDLGFBQWEsQ0FBZ0I7SUFFckMsT0FBTyxDQUFDLE9BQU8sQ0FBUztJQUV4QixPQUFPLENBQUMsT0FBTyxDQUFtQztJQUNsRCxPQUFPLENBQUMsY0FBYyxDQUFvQjtJQUUxQyxZQUFvQixLQUFLLEVBQUUsS0FBSyxFQXVCL0I7SUFFRCxLQUFLLElBQUksSUFBSSxDQVlaO0lBRUQsSUFBSSxJQUFJLElBQUksQ0FZWDtJQUVELE9BQU8sQ0FBQyxPQUFPLENBR2I7SUFFRixPQUFPLENBQUMsa0JBQWtCLENBT3hCO0lBRUYsT0FBTyxDQUFDLHFCQUFxQixDQXlCM0I7Q0FDSCJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"nodejs_metrics_monitor.d.ts","sourceRoot":"","sources":["../src/nodejs_metrics_monitor.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"nodejs_metrics_monitor.d.ts","sourceRoot":"","sources":["../src/nodejs_metrics_monitor.ts"],"names":[],"mappings":"AAMA,OAAO,KAAK,EAAyB,KAAK,EAAkC,MAAM,gBAAgB,CAAC;AAEnG,uCAAuC;AACvC,qBAAa,oBAAoB;IAuBnB,OAAO,CAAC,KAAK;IAtBzB,OAAO,CAAC,oBAAoB,CAQ1B;IAIF,OAAO,CAAC,YAAY,CAAoE;IAExF,OAAO,CAAC,mBAAmB,CAAkB;IAC7C,OAAO,CAAC,aAAa,CAAgB;IAErC,OAAO,CAAC,OAAO,CAAS;IAExB,OAAO,CAAC,OAAO,CAAmC;IAClD,OAAO,CAAC,cAAc,CAAoB;IAE1C,YAAoB,KAAK,EAAE,KAAK,EAuB/B;IAED,KAAK,IAAI,IAAI,CAYZ;IAED,IAAI,IAAI,IAAI,CAYX;IAED,OAAO,CAAC,OAAO,CAGb;IAEF,OAAO,CAAC,kBAAkB,CAOxB;IAEF,OAAO,CAAC,qBAAqB,CAyB3B;CACH"}
|
|
@@ -1,10 +1,8 @@
|
|
|
1
1
|
import { monitorEventLoopDelay, performance } from 'node:perf_hooks';
|
|
2
2
|
import * as Attributes from './attributes.js';
|
|
3
|
+
import { createUpDownCounterWithDefault } from './metric-utils.js';
|
|
3
4
|
import * as Metrics from './metrics.js';
|
|
4
|
-
|
|
5
|
-
/**
|
|
6
|
-
* Detector for custom Aztec attributes
|
|
7
|
-
*/ export class NodejsMetricsMonitor {
|
|
5
|
+
/** Monitors Node.js runtime metrics */ export class NodejsMetricsMonitor {
|
|
8
6
|
meter;
|
|
9
7
|
eventLoopDelayGauges;
|
|
10
8
|
// skip `rss` because that's already tracked by @opentelemetry/host-metrics
|
|
@@ -55,47 +53,28 @@ import { ValueType } from './telemetry.js';
|
|
|
55
53
|
safeObserveInt(obs, this.eventLoopDelayGauges.p99, this.eventLoopDelay.percentile(99));
|
|
56
54
|
this.eventLoopDelay.reset();
|
|
57
55
|
};
|
|
58
|
-
const nsObsGauge = (name, description)=>meter.createObservableGauge(name, {
|
|
59
|
-
unit: 'ns',
|
|
60
|
-
valueType: ValueType.INT,
|
|
61
|
-
description
|
|
62
|
-
});
|
|
63
56
|
this.eventLoopDelayGauges = {
|
|
64
|
-
min:
|
|
65
|
-
mean:
|
|
66
|
-
max:
|
|
67
|
-
stddev:
|
|
68
|
-
p50:
|
|
69
|
-
p90:
|
|
70
|
-
p99:
|
|
57
|
+
min: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MIN),
|
|
58
|
+
mean: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MEAN),
|
|
59
|
+
max: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_MAX),
|
|
60
|
+
stddev: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_STDDEV),
|
|
61
|
+
p50: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P50),
|
|
62
|
+
p90: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P90),
|
|
63
|
+
p99: meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_DELAY_P99)
|
|
71
64
|
};
|
|
72
|
-
this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION
|
|
73
|
-
|
|
74
|
-
|
|
75
|
-
|
|
76
|
-
|
|
77
|
-
|
|
78
|
-
valueType: ValueType.INT,
|
|
79
|
-
description: 'How much time the event loop has spent in a given state'
|
|
65
|
+
this.eventLoopUilization = meter.createObservableGauge(Metrics.NODEJS_EVENT_LOOP_UTILIZATION);
|
|
66
|
+
this.eventLoopTime = createUpDownCounterWithDefault(meter, Metrics.NODEJS_EVENT_LOOP_TIME, {
|
|
67
|
+
[Attributes.NODEJS_EVENT_LOOP_STATE]: [
|
|
68
|
+
'idle',
|
|
69
|
+
'active'
|
|
70
|
+
]
|
|
80
71
|
});
|
|
81
72
|
this.eventLoopDelay = monitorEventLoopDelay();
|
|
82
73
|
this.memoryGauges = {
|
|
83
|
-
heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE,
|
|
84
|
-
|
|
85
|
-
|
|
86
|
-
|
|
87
|
-
heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL, {
|
|
88
|
-
unit: 'By',
|
|
89
|
-
description: 'The max size the V8 heap can grow to'
|
|
90
|
-
}),
|
|
91
|
-
arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE, {
|
|
92
|
-
unit: 'By',
|
|
93
|
-
description: 'Memory allocated for buffers (includes native memory used)'
|
|
94
|
-
}),
|
|
95
|
-
external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE, {
|
|
96
|
-
unit: 'By',
|
|
97
|
-
description: 'Memory allocated for native C++ objects'
|
|
98
|
-
})
|
|
74
|
+
heapUsed: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_USAGE),
|
|
75
|
+
heapTotal: meter.createObservableGauge(Metrics.NODEJS_MEMORY_HEAP_TOTAL),
|
|
76
|
+
arrayBuffers: meter.createObservableGauge(Metrics.NODEJS_MEMORY_BUFFER_USAGE),
|
|
77
|
+
external: meter.createObservableGauge(Metrics.NODEJS_MEMORY_NATIVE_USAGE)
|
|
99
78
|
};
|
|
100
79
|
}
|
|
101
80
|
start() {
|
package/dest/noop.d.ts
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
|
-
import { type
|
|
2
|
-
import type { TelemetryClient } from './telemetry.js';
|
|
1
|
+
import { type Context, type Span, type Tracer } from '@opentelemetry/api';
|
|
2
|
+
import type { Meter, TelemetryClient } from './telemetry.js';
|
|
3
3
|
export declare class NoopTelemetryClient implements TelemetryClient {
|
|
4
|
+
private meter;
|
|
4
5
|
setExportedPublicTelemetry(_prefixes: string[]): void;
|
|
5
6
|
setPublicTelemetryCollectFrom(_roles: string[]): void;
|
|
6
7
|
getMeter(): Meter;
|
|
@@ -8,9 +9,11 @@ export declare class NoopTelemetryClient implements TelemetryClient {
|
|
|
8
9
|
stop(): Promise<void>;
|
|
9
10
|
flush(): Promise<void>;
|
|
10
11
|
isEnabled(): boolean;
|
|
12
|
+
getTraceContext(): string | undefined;
|
|
13
|
+
extractPropagatedContext(_traceContext: string): Context | undefined;
|
|
11
14
|
}
|
|
12
15
|
export declare class NoopTracer implements Tracer {
|
|
13
16
|
startSpan(): Span;
|
|
14
17
|
startActiveSpan<F extends (...args: any[]) => any>(_name: string, ...args: unknown[]): ReturnType<F>;
|
|
15
18
|
}
|
|
16
|
-
//# sourceMappingURL=
|
|
19
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibm9vcC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL25vb3AudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFFLEtBQUssT0FBTyxFQUFFLEtBQUssSUFBSSxFQUFvQixLQUFLLE1BQU0sRUFBbUIsTUFBTSxvQkFBb0IsQ0FBQztBQUc3RyxPQUFPLEtBQUssRUFHVixLQUFLLEVBR0wsZUFBZSxFQUVoQixNQUFNLGdCQUFnQixDQUFDO0FBeUN4QixxQkFBYSxtQkFBb0IsWUFBVyxlQUFlO0lBQ3pELE9BQU8sQ0FBQyxLQUFLLENBQW1CO0lBRWhDLDBCQUEwQixDQUFDLFNBQVMsRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBQUc7SUFDeEQsNkJBQTZCLENBQUMsTUFBTSxFQUFFLE1BQU0sRUFBRSxHQUFHLElBQUksQ0FBRztJQUV4RCxRQUFRLElBQUksS0FBSyxDQUVoQjtJQUVELFNBQVMsSUFBSSxNQUFNLENBRWxCO0lBRUQsSUFBSSxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FFcEI7SUFFRCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUVyQjtJQUVELFNBQVMsWUFFUjtJQUVELGVBQWUsSUFBSSxNQUFNLEdBQUcsU0FBUyxDQUVwQztJQUVELHdCQUF3QixDQUFDLGFBQWEsRUFBRSxNQUFNLEdBQUcsT0FBTyxHQUFHLFNBQVMsQ0FFbkU7Q0FDRjtBQUtELHFCQUFhLFVBQVcsWUFBVyxNQUFNO0lBQ3ZDLFNBQVMsSUFBSSxJQUFJLENBRWhCO0lBRUQsZUFBZSxDQUFDLENBQUMsU0FBUyxDQUFDLEdBQUcsSUFBSSxFQUFFLEdBQUcsRUFBRSxLQUFLLEdBQUcsRUFBRSxLQUFLLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxFQUFFLE9BQU8sRUFBRSxHQUFHLFVBQVUsQ0FBQyxDQUFDLENBQUMsQ0FJbkc7Q0FDRiJ9
|
package/dest/noop.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"noop.d.ts","sourceRoot":"","sources":["../src/noop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,
|
|
1
|
+
{"version":3,"file":"noop.d.ts","sourceRoot":"","sources":["../src/noop.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,OAAO,EAAE,KAAK,IAAI,EAAoB,KAAK,MAAM,EAAmB,MAAM,oBAAoB,CAAC;AAG7G,OAAO,KAAK,EAGV,KAAK,EAGL,eAAe,EAEhB,MAAM,gBAAgB,CAAC;AAyCxB,qBAAa,mBAAoB,YAAW,eAAe;IACzD,OAAO,CAAC,KAAK,CAAmB;IAEhC,0BAA0B,CAAC,SAAS,EAAE,MAAM,EAAE,GAAG,IAAI,CAAG;IACxD,6BAA6B,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,IAAI,CAAG;IAExD,QAAQ,IAAI,KAAK,CAEhB;IAED,SAAS,IAAI,MAAM,CAElB;IAED,IAAI,IAAI,OAAO,CAAC,IAAI,CAAC,CAEpB;IAED,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAErB;IAED,SAAS,YAER;IAED,eAAe,IAAI,MAAM,GAAG,SAAS,CAEpC;IAED,wBAAwB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,GAAG,SAAS,CAEnE;CACF;AAKD,qBAAa,UAAW,YAAW,MAAM;IACvC,SAAS,IAAI,IAAI,CAEhB;IAED,eAAe,CAAC,CAAC,SAAS,CAAC,GAAG,IAAI,EAAE,GAAG,EAAE,KAAK,GAAG,EAAE,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,EAAE,OAAO,EAAE,GAAG,UAAU,CAAC,CAAC,CAAC,CAInG;CACF"}
|
package/dest/noop.js
CHANGED
|
@@ -1,9 +1,34 @@
|
|
|
1
1
|
import { createNoopMeter } from '@opentelemetry/api';
|
|
2
|
+
/** A no-op meter that implements our custom Meter interface */ class NoopMeter {
|
|
3
|
+
otelMeter = createNoopMeter();
|
|
4
|
+
createGauge(_metric) {
|
|
5
|
+
return this.otelMeter.createGauge('');
|
|
6
|
+
}
|
|
7
|
+
createObservableGauge(_metric) {
|
|
8
|
+
return this.otelMeter.createObservableGauge('');
|
|
9
|
+
}
|
|
10
|
+
createHistogram(_metric, _extraOptions) {
|
|
11
|
+
return this.otelMeter.createHistogram('');
|
|
12
|
+
}
|
|
13
|
+
createUpDownCounter(_metric) {
|
|
14
|
+
return this.otelMeter.createUpDownCounter('');
|
|
15
|
+
}
|
|
16
|
+
createObservableUpDownCounter(_metric) {
|
|
17
|
+
return this.otelMeter.createObservableUpDownCounter('');
|
|
18
|
+
}
|
|
19
|
+
addBatchObservableCallback(callback, observables) {
|
|
20
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
21
|
+
}
|
|
22
|
+
removeBatchObservableCallback(callback, observables) {
|
|
23
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
24
|
+
}
|
|
25
|
+
}
|
|
2
26
|
export class NoopTelemetryClient {
|
|
27
|
+
meter = new NoopMeter();
|
|
3
28
|
setExportedPublicTelemetry(_prefixes) {}
|
|
4
29
|
setPublicTelemetryCollectFrom(_roles) {}
|
|
5
30
|
getMeter() {
|
|
6
|
-
return
|
|
31
|
+
return this.meter;
|
|
7
32
|
}
|
|
8
33
|
getTracer() {
|
|
9
34
|
return new NoopTracer();
|
|
@@ -17,6 +42,12 @@ export class NoopTelemetryClient {
|
|
|
17
42
|
isEnabled() {
|
|
18
43
|
return false;
|
|
19
44
|
}
|
|
45
|
+
getTraceContext() {
|
|
46
|
+
return undefined;
|
|
47
|
+
}
|
|
48
|
+
extractPropagatedContext(_traceContext) {
|
|
49
|
+
return undefined;
|
|
50
|
+
}
|
|
20
51
|
}
|
|
21
52
|
// @opentelemetry/api internally uses NoopTracer and NoopSpan but they're not exported
|
|
22
53
|
// make our own versions
|
package/dest/otel.d.ts
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { type Logger } from '@aztec/foundation/log';
|
|
2
|
-
import { type
|
|
2
|
+
import { type Context, type Tracer, type TracerProvider } from '@opentelemetry/api';
|
|
3
3
|
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
4
4
|
import type { IResource } from '@opentelemetry/resources';
|
|
5
5
|
import type { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
@@ -7,7 +7,7 @@ import { MeterProvider, type PeriodicExportingMetricReaderOptions } from '@opent
|
|
|
7
7
|
import type { TelemetryClientConfig } from './config.js';
|
|
8
8
|
import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
|
|
9
9
|
import { PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
10
|
-
import type { TelemetryClient } from './telemetry.js';
|
|
10
|
+
import type { Meter, TelemetryClient } from './telemetry.js';
|
|
11
11
|
export type OpenTelemetryClientFactory = (resource: IResource, log: Logger) => OpenTelemetryClient;
|
|
12
12
|
export declare class OpenTelemetryClient implements TelemetryClient {
|
|
13
13
|
private resource;
|
|
@@ -20,6 +20,10 @@ export declare class OpenTelemetryClient implements TelemetryClient {
|
|
|
20
20
|
nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
|
|
21
21
|
private meters;
|
|
22
22
|
private tracers;
|
|
23
|
+
/** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
|
|
24
|
+
* so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
|
|
25
|
+
* "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
|
|
26
|
+
private stopPromise;
|
|
23
27
|
protected constructor(resource: IResource, meterProvider: MeterProvider, traceProvider: TracerProvider, loggerProvider: LoggerProvider | undefined, publicMetricExporter: PublicOtelFilterMetricExporter | undefined, log: Logger);
|
|
24
28
|
setExportedPublicTelemetry(metrics: string[]): void;
|
|
25
29
|
setPublicTelemetryCollectFrom(roles: string[]): void;
|
|
@@ -29,8 +33,11 @@ export declare class OpenTelemetryClient implements TelemetryClient {
|
|
|
29
33
|
isEnabled(): boolean;
|
|
30
34
|
flush(): Promise<void>;
|
|
31
35
|
stop(): Promise<void>;
|
|
36
|
+
private doStop;
|
|
37
|
+
getTraceContext(): string | undefined;
|
|
38
|
+
extractPropagatedContext(traceContext: string): Context;
|
|
32
39
|
static createMeterProvider(resource: IResource, exporters: Array<PeriodicExportingMetricReaderOptions>): MeterProvider;
|
|
33
40
|
private static getCustomClientFactory;
|
|
34
41
|
static createAndStart(config: TelemetryClientConfig, log: Logger): OpenTelemetryClient;
|
|
35
42
|
}
|
|
36
|
-
//# sourceMappingURL=
|
|
43
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFnQixLQUFLLE1BQU0sRUFBcUIsTUFBTSx1QkFBdUIsQ0FBQztBQUVyRixPQUFPLEVBQ0wsS0FBSyxPQUFPLEVBS1osS0FBSyxNQUFNLEVBQ1gsS0FBSyxjQUFjLEVBTXBCLE1BQU0sb0JBQW9CLENBQUM7QUFJNUIsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzlELE9BQU8sRUFHTCxhQUFhLEVBRWIsS0FBSyxvQ0FBb0MsRUFFMUMsTUFBTSw0QkFBNEIsQ0FBQztBQUlwQyxPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUl6RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNuRSxPQUFPLEVBQTRCLDhCQUE4QixFQUFFLE1BQU0sa0NBQWtDLENBQUM7QUFHNUcsT0FBTyxLQUFLLEVBR1YsS0FBSyxFQUdMLGVBQWUsRUFFaEIsTUFBTSxnQkFBZ0IsQ0FBQztBQXlDeEIsTUFBTSxNQUFNLDBCQUEwQixHQUFHLENBQUMsUUFBUSxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsTUFBTSxLQUFLLG1CQUFtQixDQUFDO0FBRW5HLHFCQUFhLG1CQUFvQixZQUFXLGVBQWU7SUFZdkQsT0FBTyxDQUFDLFFBQVE7SUFDaEIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGNBQWM7SUFDdEIsT0FBTyxDQUFDLG9CQUFvQjtJQUM1QixPQUFPLENBQUMsR0FBRztJQWhCYixXQUFXLEVBQUUsV0FBVyxHQUFHLFNBQVMsQ0FBQztJQUNyQyxvQkFBb0IsRUFBRSxvQkFBb0IsR0FBRyxTQUFTLENBQUM7SUFDdkQsT0FBTyxDQUFDLE1BQU0sQ0FBOEQ7SUFDNUUsT0FBTyxDQUFDLE9BQU8sQ0FBa0Q7SUFFakU7O3lIQUVxSDtJQUNySCxPQUFPLENBQUMsV0FBVyxDQUE0QjtJQUUvQyxTQUFTLGFBQ0MsUUFBUSxFQUFFLFNBQVMsRUFDbkIsYUFBYSxFQUFFLGFBQWEsRUFDNUIsYUFBYSxFQUFFLGNBQWMsRUFDN0IsY0FBYyxFQUFFLGNBQWMsR0FBRyxTQUFTLEVBQzFDLG9CQUFvQixFQUFFLDhCQUE4QixHQUFHLFNBQVMsRUFDaEUsR0FBRyxFQUFFLE1BQU0sRUFDakI7SUFFSiwwQkFBMEIsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUVsRDtJQUVELDZCQUE2QixDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBRW5EO0lBRUQsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEdBQUcsS0FBSyxDQVE1QjtJQUVELFNBQVMsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FPOUI7SUFFTSxLQUFLLFNBOEJYO0lBRU0sU0FBUyxZQUVmO0lBRVksS0FBSyxrQkFVakI7SUFFTSxJQUFJLGtCQUVWO1lBRWEsTUFBTTtJQWtCYixlQUFlLElBQUksTUFBTSxHQUFHLFNBQVMsQ0FJM0M7SUFFTSx3QkFBd0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FLN0Q7SUFFRCxPQUFjLG1CQUFtQixDQUMvQixRQUFRLEVBQUUsU0FBUyxFQUNuQixTQUFTLEVBQUUsS0FBSyxDQUFDLG9DQUFvQyxDQUFDLEdBQ3JELGFBQWEsQ0E2SmY7SUFFRCxPQUFPLENBQUMsTUFBTSxDQUFDLHNCQUFzQjtJQXNFckMsT0FBYyxjQUFjLENBQUMsTUFBTSxFQUFFLHFCQUFxQixFQUFFLEdBQUcsRUFBRSxNQUFNLEdBQUcsbUJBQW1CLENBUTVGO0NBQ0YifQ==
|
package/dest/otel.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,
|
|
1
|
+
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,EACL,KAAK,OAAO,EAKZ,KAAK,MAAM,EACX,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAGL,aAAa,EAEb,KAAK,oCAAoC,EAE1C,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAIzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAA4B,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAG5G,OAAO,KAAK,EAGV,KAAK,EAGL,eAAe,EAEhB,MAAM,gBAAgB,CAAC;AAyCxB,MAAM,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAEnG,qBAAa,mBAAoB,YAAW,eAAe;IAYvD,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,GAAG;IAhBb,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACvD,OAAO,CAAC,MAAM,CAA8D;IAC5E,OAAO,CAAC,OAAO,CAAkD;IAEjE;;yHAEqH;IACrH,OAAO,CAAC,WAAW,CAA4B;IAE/C,SAAS,aACC,QAAQ,EAAE,SAAS,EACnB,aAAa,EAAE,aAAa,EAC5B,aAAa,EAAE,cAAc,EAC7B,cAAc,EAAE,cAAc,GAAG,SAAS,EAC1C,oBAAoB,EAAE,8BAA8B,GAAG,SAAS,EAChE,GAAG,EAAE,MAAM,EACjB;IAEJ,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAElD;IAED,6BAA6B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAEnD;IAED,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAQ5B;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO9B;IAEM,KAAK,SA8BX;IAEM,SAAS,YAEf;IAEY,KAAK,kBAUjB;IAEM,IAAI,kBAEV;YAEa,MAAM;IAkBb,eAAe,IAAI,MAAM,GAAG,SAAS,CAI3C;IAEM,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAK7D;IAED,OAAc,mBAAmB,CAC/B,QAAQ,EAAE,SAAS,EACnB,SAAS,EAAE,KAAK,CAAC,oCAAoC,CAAC,GACrD,aAAa,CA6Jf;IAED,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAsErC,OAAc,cAAc,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAQ5F;CACF"}
|
package/dest/otel.js
CHANGED
|
@@ -1,15 +1,48 @@
|
|
|
1
1
|
import { addLogDataHandler } from '@aztec/foundation/log';
|
|
2
|
-
import { DiagConsoleLogger, DiagLogLevel, context, diag, isSpanContextValid, trace } from '@opentelemetry/api';
|
|
2
|
+
import { DiagConsoleLogger, DiagLogLevel, ROOT_CONTEXT, context, diag, isSpanContextValid, propagation, trace } from '@opentelemetry/api';
|
|
3
|
+
import { W3CTraceContextPropagator } from '@opentelemetry/core';
|
|
3
4
|
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
4
5
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
5
6
|
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
6
7
|
import { ExplicitBucketHistogramAggregation, InstrumentType, MeterProvider, PeriodicExportingMetricReader, View } from '@opentelemetry/sdk-metrics';
|
|
7
|
-
import {
|
|
8
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
8
9
|
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
10
|
+
import { toMetricOptions } from './metric-utils.js';
|
|
11
|
+
import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
|
|
9
12
|
import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
|
|
10
13
|
import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
11
14
|
import { registerOtelLoggerProvider } from './otel_logger_provider.js';
|
|
12
15
|
import { getOtelResource } from './otel_resource.js';
|
|
16
|
+
/** Wraps an OpenTelemetry Meter to implement our custom Meter interface */ class WrappedMeter {
|
|
17
|
+
otelMeter;
|
|
18
|
+
constructor(otelMeter){
|
|
19
|
+
this.otelMeter = otelMeter;
|
|
20
|
+
}
|
|
21
|
+
createGauge(metric) {
|
|
22
|
+
return this.otelMeter.createGauge(metric.name, toMetricOptions(metric));
|
|
23
|
+
}
|
|
24
|
+
createObservableGauge(metric) {
|
|
25
|
+
return this.otelMeter.createObservableGauge(metric.name, toMetricOptions(metric));
|
|
26
|
+
}
|
|
27
|
+
createHistogram(metric, extraOptions) {
|
|
28
|
+
return this.otelMeter.createHistogram(metric.name, {
|
|
29
|
+
...toMetricOptions(metric),
|
|
30
|
+
...extraOptions
|
|
31
|
+
});
|
|
32
|
+
}
|
|
33
|
+
createUpDownCounter(metric) {
|
|
34
|
+
return this.otelMeter.createUpDownCounter(metric.name, toMetricOptions(metric));
|
|
35
|
+
}
|
|
36
|
+
createObservableUpDownCounter(metric) {
|
|
37
|
+
return this.otelMeter.createObservableUpDownCounter(metric.name, toMetricOptions(metric));
|
|
38
|
+
}
|
|
39
|
+
addBatchObservableCallback(callback, observables) {
|
|
40
|
+
this.otelMeter.addBatchObservableCallback(callback, observables);
|
|
41
|
+
}
|
|
42
|
+
removeBatchObservableCallback(callback, observables) {
|
|
43
|
+
this.otelMeter.removeBatchObservableCallback(callback, observables);
|
|
44
|
+
}
|
|
45
|
+
}
|
|
13
46
|
export class OpenTelemetryClient {
|
|
14
47
|
resource;
|
|
15
48
|
meterProvider;
|
|
@@ -21,6 +54,9 @@ export class OpenTelemetryClient {
|
|
|
21
54
|
nodejsMetricsMonitor;
|
|
22
55
|
meters;
|
|
23
56
|
tracers;
|
|
57
|
+
/** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
|
|
58
|
+
* so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
|
|
59
|
+
* "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */ stopPromise;
|
|
24
60
|
constructor(resource, meterProvider, traceProvider, loggerProvider, publicMetricExporter, log){
|
|
25
61
|
this.resource = resource;
|
|
26
62
|
this.meterProvider = meterProvider;
|
|
@@ -40,7 +76,8 @@ export class OpenTelemetryClient {
|
|
|
40
76
|
getMeter(name) {
|
|
41
77
|
let meter = this.meters.get(name);
|
|
42
78
|
if (!meter) {
|
|
43
|
-
|
|
79
|
+
const otelMeter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION]);
|
|
80
|
+
meter = new WrappedMeter(otelMeter);
|
|
44
81
|
this.meters.set(name, meter);
|
|
45
82
|
}
|
|
46
83
|
return meter;
|
|
@@ -71,7 +108,8 @@ export class OpenTelemetryClient {
|
|
|
71
108
|
name: this.resource.attributes[ATTR_SERVICE_NAME],
|
|
72
109
|
meterProvider: this.meterProvider
|
|
73
110
|
});
|
|
74
|
-
|
|
111
|
+
const nodejsMeter = new WrappedMeter(this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME]));
|
|
112
|
+
this.nodejsMetricsMonitor = new NodejsMetricsMonitor(nodejsMeter);
|
|
75
113
|
this.hostMetrics.start();
|
|
76
114
|
this.nodejsMetricsMonitor.start();
|
|
77
115
|
}
|
|
@@ -79,13 +117,20 @@ export class OpenTelemetryClient {
|
|
|
79
117
|
return true;
|
|
80
118
|
}
|
|
81
119
|
async flush() {
|
|
120
|
+
// Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
|
|
121
|
+
if (this.stopPromise) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
82
124
|
await Promise.all([
|
|
83
125
|
this.meterProvider.forceFlush(),
|
|
84
126
|
this.loggerProvider?.forceFlush(),
|
|
85
127
|
this.traceProvider instanceof NodeTracerProvider ? this.traceProvider.forceFlush() : Promise.resolve()
|
|
86
128
|
]);
|
|
87
129
|
}
|
|
88
|
-
|
|
130
|
+
stop() {
|
|
131
|
+
return this.stopPromise ??= this.doStop();
|
|
132
|
+
}
|
|
133
|
+
async doStop() {
|
|
89
134
|
this.nodejsMetricsMonitor?.stop();
|
|
90
135
|
const flushAndShutdown = async (provider)=>{
|
|
91
136
|
if (!provider) {
|
|
@@ -100,6 +145,17 @@ export class OpenTelemetryClient {
|
|
|
100
145
|
this.traceProvider instanceof NodeTracerProvider ? flushAndShutdown(this.traceProvider) : Promise.resolve()
|
|
101
146
|
]);
|
|
102
147
|
}
|
|
148
|
+
getTraceContext() {
|
|
149
|
+
const carrier = {};
|
|
150
|
+
propagation.inject(context.active(), carrier);
|
|
151
|
+
return carrier['traceparent'];
|
|
152
|
+
}
|
|
153
|
+
extractPropagatedContext(traceContext) {
|
|
154
|
+
const extractedContext = propagation.extract(ROOT_CONTEXT, {
|
|
155
|
+
traceparent: traceContext
|
|
156
|
+
});
|
|
157
|
+
return extractedContext;
|
|
158
|
+
}
|
|
103
159
|
static createMeterProvider(resource, exporters) {
|
|
104
160
|
return new MeterProvider({
|
|
105
161
|
resource,
|
|
@@ -324,6 +380,61 @@ export class OpenTelemetryClient {
|
|
|
324
380
|
15_000_000,
|
|
325
381
|
30_000_000
|
|
326
382
|
], true)
|
|
383
|
+
}),
|
|
384
|
+
// L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
|
|
385
|
+
new View({
|
|
386
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
387
|
+
instrumentUnit: 'gwei',
|
|
388
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
389
|
+
0.1,
|
|
390
|
+
0.5,
|
|
391
|
+
1,
|
|
392
|
+
2,
|
|
393
|
+
5,
|
|
394
|
+
10,
|
|
395
|
+
20,
|
|
396
|
+
50,
|
|
397
|
+
100,
|
|
398
|
+
200,
|
|
399
|
+
500,
|
|
400
|
+
1_000
|
|
401
|
+
], true)
|
|
402
|
+
}),
|
|
403
|
+
// L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
|
|
404
|
+
new View({
|
|
405
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
406
|
+
instrumentUnit: 'gas',
|
|
407
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
408
|
+
10_000,
|
|
409
|
+
50_000,
|
|
410
|
+
100_000,
|
|
411
|
+
250_000,
|
|
412
|
+
500_000,
|
|
413
|
+
1_000_000,
|
|
414
|
+
2_000_000,
|
|
415
|
+
5_000_000,
|
|
416
|
+
10_000_000,
|
|
417
|
+
15_000_000,
|
|
418
|
+
30_000_000
|
|
419
|
+
], true)
|
|
420
|
+
}),
|
|
421
|
+
// L1 tx total fee in ETH: typically 0.001 - 1 ETH
|
|
422
|
+
new View({
|
|
423
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
424
|
+
instrumentUnit: 'eth',
|
|
425
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
426
|
+
0.0001,
|
|
427
|
+
0.0005,
|
|
428
|
+
0.001,
|
|
429
|
+
0.005,
|
|
430
|
+
0.01,
|
|
431
|
+
0.05,
|
|
432
|
+
0.1,
|
|
433
|
+
0.5,
|
|
434
|
+
1,
|
|
435
|
+
5,
|
|
436
|
+
10
|
|
437
|
+
], true)
|
|
327
438
|
})
|
|
328
439
|
]
|
|
329
440
|
});
|
|
@@ -333,18 +444,37 @@ export class OpenTelemetryClient {
|
|
|
333
444
|
const tracerProvider = new NodeTracerProvider({
|
|
334
445
|
resource,
|
|
335
446
|
spanProcessors: config.tracesCollectorUrl ? [
|
|
336
|
-
new
|
|
447
|
+
new MonitoredBatchSpanProcessor(new OTLPTraceExporter({
|
|
337
448
|
url: config.tracesCollectorUrl.href
|
|
338
|
-
})
|
|
449
|
+
}), log, {
|
|
450
|
+
maxQueueSize: config.otelBspMaxQueueSize,
|
|
451
|
+
minTraceDurationMs: config.otelMinTraceDurationMs
|
|
452
|
+
})
|
|
339
453
|
] : []
|
|
340
454
|
});
|
|
341
|
-
tracerProvider.register(
|
|
455
|
+
tracerProvider.register({
|
|
456
|
+
propagator: new W3CTraceContextPropagator()
|
|
457
|
+
});
|
|
342
458
|
const exporters = [];
|
|
343
459
|
if (config.metricsCollectorUrl) {
|
|
460
|
+
// Default to a blacklist that is empty (allow all metrics)
|
|
461
|
+
let filter = [];
|
|
462
|
+
let mode = 'deny';
|
|
463
|
+
if (config.otelExcludeMetrics.length > 0) {
|
|
464
|
+
// Implement a blacklist as specified in config
|
|
465
|
+
log.info(`Excluding metrics from export: ${config.otelExcludeMetrics}`);
|
|
466
|
+
filter = config.otelExcludeMetrics;
|
|
467
|
+
mode = 'deny';
|
|
468
|
+
} else if (config.otelIncludeMetrics.length > 0) {
|
|
469
|
+
// Implement a whitelist as specified in config
|
|
470
|
+
log.info(`Including only specified metrics for export: ${config.otelIncludeMetrics}`);
|
|
471
|
+
filter = config.otelIncludeMetrics;
|
|
472
|
+
mode = 'allow';
|
|
473
|
+
}
|
|
344
474
|
exporters.push({
|
|
345
475
|
exporter: new OtelFilterMetricExporter(new OTLPMetricExporter({
|
|
346
476
|
url: config.metricsCollectorUrl.href
|
|
347
|
-
}),
|
|
477
|
+
}), filter, mode),
|
|
348
478
|
exportTimeoutMillis: config.otelExportTimeoutMs,
|
|
349
479
|
exportIntervalMillis: config.otelCollectIntervalMs
|
|
350
480
|
});
|
|
@@ -17,4 +17,4 @@ export declare class PublicOtelFilterMetricExporter extends OtelFilterMetricExpo
|
|
|
17
17
|
export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void;
|
|
18
18
|
setAllowedRoles(roles: string[]): void;
|
|
19
19
|
}
|
|
20
|
-
//# sourceMappingURL=
|
|
20
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9maWx0ZXJfbWV0cmljX2V4cG9ydGVyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvb3RlbF9maWx0ZXJfbWV0cmljX2V4cG9ydGVyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFBRSxLQUFLLFlBQVksRUFBb0IsTUFBTSxxQkFBcUIsQ0FBQztBQUMxRSxPQUFPLEtBQUssRUFBYyxrQkFBa0IsRUFBRSxlQUFlLEVBQUUsTUFBTSw0QkFBNEIsQ0FBQztBQUlsRyxxQkFBYSx3QkFBeUIsWUFBVyxrQkFBa0I7SUFFL0QsT0FBTyxDQUFDLFFBQVEsQ0FBQyxRQUFRO0lBQ3pCLE9BQU8sQ0FBQyxZQUFZO0lBQ3BCLE9BQU8sQ0FBQyxRQUFRLENBQUMsTUFBTTtJQUh6QixZQUNtQixRQUFRLEVBQUUsa0JBQWtCLEVBQ3JDLFlBQVksRUFBRSxNQUFNLEVBQUUsRUFDYixNQUFNLEdBQUUsT0FBTyxHQUFHLE1BQWUsRUFRbkQ7SUFFTSxNQUFNLENBQUMsT0FBTyxFQUFFLGVBQWUsRUFBRSxjQUFjLEVBQUUsQ0FBQyxNQUFNLEVBQUUsWUFBWSxLQUFLLElBQUksR0FBRyxJQUFJLENBUzVGO0lBRUQsT0FBTyxDQUFDLGFBQWE7SUFjZCxVQUFVLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUVqQztJQUVNLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBRS9CO0lBRU0saUJBQWlCLENBQUMsT0FBTyxFQUFFLE1BQU0sRUFBRSxRQUV6QztDQUNGO0FBRUQscUJBQWEsOEJBQStCLFNBQVEsd0JBQXdCO0lBRXhFLE9BQU8sQ0FBQyxZQUFZO0lBRHRCLFlBQ1UsWUFBWSxFQUFFLE1BQU0sRUFBRSxFQUM5QixRQUFRLEVBQUUsa0JBQWtCLEVBQzVCLFlBQVksRUFBRSxNQUFNLEVBQUUsRUFHdkI7SUFFZSxNQUFNLENBQUMsT0FBTyxFQUFFLGVBQWUsRUFBRSxjQUFjLEVBQUUsQ0FBQyxNQUFNLEVBQUUsWUFBWSxLQUFLLElBQUksR0FBRyxJQUFJLENBUXJHO0lBRU0sZUFBZSxDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsUUFFckM7Q0FDRiJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"otel_filter_metric_exporter.d.ts","sourceRoot":"","sources":["../src/otel_filter_metric_exporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAc,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAIlG,qBAAa,wBAAyB,YAAW,kBAAkB;IAE/D,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM;
|
|
1
|
+
{"version":3,"file":"otel_filter_metric_exporter.d.ts","sourceRoot":"","sources":["../src/otel_filter_metric_exporter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,YAAY,EAAoB,MAAM,qBAAqB,CAAC;AAC1E,OAAO,KAAK,EAAc,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAIlG,qBAAa,wBAAyB,YAAW,kBAAkB;IAE/D,OAAO,CAAC,QAAQ,CAAC,QAAQ;IACzB,OAAO,CAAC,YAAY;IACpB,OAAO,CAAC,QAAQ,CAAC,MAAM;IAHzB,YACmB,QAAQ,EAAE,kBAAkB,EACrC,YAAY,EAAE,MAAM,EAAE,EACb,MAAM,GAAE,OAAO,GAAG,MAAe,EAQnD;IAEM,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI,CAS5F;IAED,OAAO,CAAC,aAAa;IAcd,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAEjC;IAEM,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAE/B;IAEM,iBAAiB,CAAC,OAAO,EAAE,MAAM,EAAE,QAEzC;CACF;AAED,qBAAa,8BAA+B,SAAQ,wBAAwB;IAExE,OAAO,CAAC,YAAY;IADtB,YACU,YAAY,EAAE,MAAM,EAAE,EAC9B,QAAQ,EAAE,kBAAkB,EAC5B,YAAY,EAAE,MAAM,EAAE,EAGvB;IAEe,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI,CAQrG;IAEM,eAAe,CAAC,KAAK,EAAE,MAAM,EAAE,QAErC;CACF"}
|
|
@@ -1,4 +1,4 @@
|
|
|
1
1
|
import type { IResource } from '@opentelemetry/resources';
|
|
2
2
|
import { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
3
3
|
export declare function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: URL): LoggerProvider;
|
|
4
|
-
//# sourceMappingURL=
|
|
4
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9sb2dnZXJfcHJvdmlkZXIuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9vdGVsX2xvZ2dlcl9wcm92aWRlci50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFHQSxPQUFPLEtBQUssRUFBRSxTQUFTLEVBQUUsTUFBTSwwQkFBMEIsQ0FBQztBQUMxRCxPQUFPLEVBQTJCLGNBQWMsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBRWxGLHdCQUFnQiwwQkFBMEIsQ0FBQyxRQUFRLEVBQUUsU0FBUyxFQUFFLFdBQVcsQ0FBQyxFQUFFLEdBQUcsa0JBd0JoRiJ9
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
|
|
1
2
|
import type Koa from 'koa';
|
|
2
3
|
export declare function getOtelJsonRpcPropagationMiddleware(scope?: string): (ctx: Koa.Context, next: () => Promise<void>) => Promise<void>;
|
|
3
|
-
|
|
4
|
+
export declare function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware;
|
|
5
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9wcm9wYWdhdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWxfcHJvcGFnYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxtQ0FBbUMsQ0FBQztBQUcvRSxPQUFPLEtBQUssR0FBRyxNQUFNLEtBQUssQ0FBQztBQVUzQix3QkFBZ0IsbUNBQW1DLENBQ2pELEtBQUssU0FBa0IsR0FDdEIsQ0FBQyxHQUFHLEVBQUUsR0FBRyxDQUFDLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLEtBQUssT0FBTyxDQUFDLElBQUksQ0FBQyxDQW9DaEU7QUFFRCx3QkFBZ0IsbUNBQW1DLElBQUkscUJBQXFCLENBNEIzRSJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG/E,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAU3B,wBAAgB,mCAAmC,CACjD,KAAK,SAAkB,GACtB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAoChE;AAED,wBAAgB,mCAAmC,IAAI,qBAAqB,CA4B3E"}
|