@aztec/telemetry-client 0.0.0-test.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/attributes.d.ts +99 -0
- package/dest/attributes.d.ts.map +1 -0
- package/dest/attributes.js +67 -0
- package/dest/bench.d.ts +29 -0
- package/dest/bench.d.ts.map +1 -0
- package/dest/bench.js +98 -0
- package/dest/config.d.ts +12 -0
- package/dest/config.d.ts.map +1 -0
- package/dest/config.js +39 -0
- package/dest/event_loop_monitor.d.ts +18 -0
- package/dest/event_loop_monitor.d.ts.map +1 -0
- package/dest/event_loop_monitor.js +93 -0
- package/dest/index.d.ts +10 -0
- package/dest/index.d.ts.map +1 -0
- package/dest/index.js +9 -0
- package/dest/lmdb_metrics.d.ts +16 -0
- package/dest/lmdb_metrics.d.ts.map +1 -0
- package/dest/lmdb_metrics.js +42 -0
- package/dest/metrics.d.ts +129 -0
- package/dest/metrics.d.ts.map +1 -0
- package/dest/metrics.js +126 -0
- package/dest/noop.d.ts +14 -0
- package/dest/noop.d.ts.map +1 -0
- package/dest/noop.js +71 -0
- package/dest/otel.d.ts +32 -0
- package/dest/otel.d.ts.map +1 -0
- package/dest/otel.js +319 -0
- package/dest/otel_filter_metric_exporter.d.ts +12 -0
- package/dest/otel_filter_metric_exporter.d.ts.map +1 -0
- package/dest/otel_filter_metric_exporter.js +33 -0
- package/dest/otel_logger_provider.d.ts +4 -0
- package/dest/otel_logger_provider.d.ts.map +1 -0
- package/dest/otel_logger_provider.js +25 -0
- package/dest/otel_propagation.d.ts +3 -0
- package/dest/otel_propagation.d.ts.map +1 -0
- package/dest/otel_propagation.js +44 -0
- package/dest/otel_resource.d.ts +3 -0
- package/dest/otel_resource.d.ts.map +1 -0
- package/dest/otel_resource.js +12 -0
- package/dest/prom_otel_adapter.d.ts +127 -0
- package/dest/prom_otel_adapter.d.ts.map +1 -0
- package/dest/prom_otel_adapter.js +183 -0
- package/dest/start.d.ts +6 -0
- package/dest/start.d.ts.map +1 -0
- package/dest/start.js +24 -0
- package/dest/telemetry.d.ts +118 -0
- package/dest/telemetry.d.ts.map +1 -0
- package/dest/telemetry.js +116 -0
- package/dest/vendor/attributes.d.ts +5 -0
- package/dest/vendor/attributes.d.ts.map +1 -0
- package/dest/vendor/attributes.js +5 -0
- package/dest/vendor/otel-pino-stream.d.ts +41 -0
- package/dest/vendor/otel-pino-stream.d.ts.map +1 -0
- package/dest/vendor/otel-pino-stream.js +229 -0
- package/dest/with_tracer.d.ts +33 -0
- package/dest/with_tracer.d.ts.map +1 -0
- package/dest/with_tracer.js +32 -0
- package/dest/wrappers/fetch.d.ts +16 -0
- package/dest/wrappers/fetch.d.ts.map +1 -0
- package/dest/wrappers/fetch.js +39 -0
- package/dest/wrappers/index.d.ts +4 -0
- package/dest/wrappers/index.d.ts.map +1 -0
- package/dest/wrappers/index.js +3 -0
- package/dest/wrappers/json_rpc_server.d.ts +4 -0
- package/dest/wrappers/json_rpc_server.d.ts.map +1 -0
- package/dest/wrappers/json_rpc_server.js +11 -0
- package/dest/wrappers/l2_block_stream.d.ts +15 -0
- package/dest/wrappers/l2_block_stream.d.ts.map +1 -0
- package/dest/wrappers/l2_block_stream.js +26 -0
- package/package.json +89 -0
- package/src/attributes.ts +115 -0
- package/src/bench.ts +147 -0
- package/src/config.ts +56 -0
- package/src/event_loop_monitor.ts +119 -0
- package/src/index.ts +9 -0
- package/src/lmdb_metrics.ts +45 -0
- package/src/metrics.ts +153 -0
- package/src/noop.ts +91 -0
- package/src/otel.ts +286 -0
- package/src/otel_filter_metric_exporter.ts +38 -0
- package/src/otel_logger_provider.ts +31 -0
- package/src/otel_propagation.ts +50 -0
- package/src/otel_resource.ts +16 -0
- package/src/prom_otel_adapter.ts +326 -0
- package/src/start.ts +33 -0
- package/src/telemetry.ts +267 -0
- package/src/vendor/attributes.ts +5 -0
- package/src/vendor/otel-pino-stream.ts +282 -0
- package/src/with_tracer.ts +35 -0
- package/src/wrappers/fetch.ts +52 -0
- package/src/wrappers/index.ts +3 -0
- package/src/wrappers/json_rpc_server.ts +15 -0
- package/src/wrappers/l2_block_stream.ts +37 -0
package/dest/otel.js
ADDED
|
@@ -0,0 +1,319 @@
|
|
|
1
|
+
import { addLogDataHandler } from '@aztec/foundation/log';
|
|
2
|
+
import { DiagConsoleLogger, DiagLogLevel, context, diag, isSpanContextValid, trace } from '@opentelemetry/api';
|
|
3
|
+
import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
4
|
+
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
5
|
+
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
6
|
+
import { ExplicitBucketHistogramAggregation, InstrumentType, MeterProvider, PeriodicExportingMetricReader, View } from '@opentelemetry/sdk-metrics';
|
|
7
|
+
import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
8
|
+
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
9
|
+
import { EventLoopMonitor } from './event_loop_monitor.js';
|
|
10
|
+
import { OtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
11
|
+
import { registerOtelLoggerProvider } from './otel_logger_provider.js';
|
|
12
|
+
import { getOtelResource } from './otel_resource.js';
|
|
13
|
+
export class OpenTelemetryClient {
|
|
14
|
+
resource;
|
|
15
|
+
meterProvider;
|
|
16
|
+
traceProvider;
|
|
17
|
+
loggerProvider;
|
|
18
|
+
log;
|
|
19
|
+
hostMetrics;
|
|
20
|
+
eventLoopMonitor;
|
|
21
|
+
meters;
|
|
22
|
+
tracers;
|
|
23
|
+
constructor(resource, meterProvider, traceProvider, loggerProvider, log){
|
|
24
|
+
this.resource = resource;
|
|
25
|
+
this.meterProvider = meterProvider;
|
|
26
|
+
this.traceProvider = traceProvider;
|
|
27
|
+
this.loggerProvider = loggerProvider;
|
|
28
|
+
this.log = log;
|
|
29
|
+
this.meters = new Map();
|
|
30
|
+
this.tracers = new Map();
|
|
31
|
+
}
|
|
32
|
+
getMeter(name) {
|
|
33
|
+
let meter = this.meters.get(name);
|
|
34
|
+
if (!meter) {
|
|
35
|
+
meter = this.meterProvider.getMeter(name, this.resource.attributes[ATTR_SERVICE_VERSION]);
|
|
36
|
+
this.meters.set(name, meter);
|
|
37
|
+
}
|
|
38
|
+
return meter;
|
|
39
|
+
}
|
|
40
|
+
getTracer(name) {
|
|
41
|
+
let tracer = this.tracers.get(name);
|
|
42
|
+
if (!tracer) {
|
|
43
|
+
tracer = this.traceProvider.getTracer(name, this.resource.attributes[ATTR_SERVICE_VERSION]);
|
|
44
|
+
this.tracers.set(name, tracer);
|
|
45
|
+
}
|
|
46
|
+
return tracer;
|
|
47
|
+
}
|
|
48
|
+
start() {
|
|
49
|
+
this.log.info('Starting OpenTelemetry client');
|
|
50
|
+
diag.setLogger(new DiagConsoleLogger(), DiagLogLevel.INFO);
|
|
51
|
+
// Add a callback to the logger to set context data from current trace
|
|
52
|
+
// Adapted from open-telemetry/opentelemetry-js-contrib PinoInstrumentation._getMixinFunction
|
|
53
|
+
addLogDataHandler((data)=>{
|
|
54
|
+
const spanContext = trace.getSpan(context.active())?.spanContext();
|
|
55
|
+
return spanContext && isSpanContextValid(spanContext) ? {
|
|
56
|
+
...data,
|
|
57
|
+
['trace_id']: spanContext.traceId,
|
|
58
|
+
['span_id']: spanContext.spanId,
|
|
59
|
+
['trace_flags']: `0${spanContext.traceFlags.toString(16)}`
|
|
60
|
+
} : data;
|
|
61
|
+
});
|
|
62
|
+
this.hostMetrics = new HostMetrics({
|
|
63
|
+
name: this.resource.attributes[ATTR_SERVICE_NAME],
|
|
64
|
+
meterProvider: this.meterProvider
|
|
65
|
+
});
|
|
66
|
+
this.eventLoopMonitor = new EventLoopMonitor(this.meterProvider.getMeter(this.resource.attributes[ATTR_SERVICE_NAME]));
|
|
67
|
+
this.hostMetrics.start();
|
|
68
|
+
this.eventLoopMonitor.start();
|
|
69
|
+
}
|
|
70
|
+
isEnabled() {
|
|
71
|
+
return true;
|
|
72
|
+
}
|
|
73
|
+
async flush() {
|
|
74
|
+
await Promise.all([
|
|
75
|
+
this.meterProvider.forceFlush(),
|
|
76
|
+
this.loggerProvider?.forceFlush(),
|
|
77
|
+
this.traceProvider instanceof NodeTracerProvider ? this.traceProvider.forceFlush() : Promise.resolve()
|
|
78
|
+
]);
|
|
79
|
+
}
|
|
80
|
+
async stop() {
|
|
81
|
+
this.eventLoopMonitor?.stop();
|
|
82
|
+
const flushAndShutdown = async (provider)=>{
|
|
83
|
+
if (!provider) {
|
|
84
|
+
return;
|
|
85
|
+
}
|
|
86
|
+
await provider.forceFlush();
|
|
87
|
+
await provider.shutdown();
|
|
88
|
+
};
|
|
89
|
+
await Promise.all([
|
|
90
|
+
flushAndShutdown(this.meterProvider),
|
|
91
|
+
flushAndShutdown(this.loggerProvider),
|
|
92
|
+
this.traceProvider instanceof NodeTracerProvider ? flushAndShutdown(this.traceProvider) : Promise.resolve()
|
|
93
|
+
]);
|
|
94
|
+
}
|
|
95
|
+
static createMeterProvider(resource, options) {
|
|
96
|
+
return new MeterProvider({
|
|
97
|
+
resource,
|
|
98
|
+
readers: options.exporter ? [
|
|
99
|
+
new PeriodicExportingMetricReader(options)
|
|
100
|
+
] : [],
|
|
101
|
+
views: [
|
|
102
|
+
// Every histogram matching the selector (type + unit) gets these custom buckets assigned
|
|
103
|
+
new View({
|
|
104
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
105
|
+
instrumentUnit: 's',
|
|
106
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
107
|
+
1,
|
|
108
|
+
2,
|
|
109
|
+
4,
|
|
110
|
+
6,
|
|
111
|
+
10,
|
|
112
|
+
15,
|
|
113
|
+
30,
|
|
114
|
+
60,
|
|
115
|
+
90,
|
|
116
|
+
120,
|
|
117
|
+
180,
|
|
118
|
+
240,
|
|
119
|
+
300,
|
|
120
|
+
480,
|
|
121
|
+
600,
|
|
122
|
+
900,
|
|
123
|
+
1200
|
|
124
|
+
], true)
|
|
125
|
+
}),
|
|
126
|
+
new View({
|
|
127
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
128
|
+
instrumentUnit: 'ms',
|
|
129
|
+
aggregation: new ExplicitBucketHistogramAggregation(// 10ms to 1 minute
|
|
130
|
+
[
|
|
131
|
+
10,
|
|
132
|
+
20,
|
|
133
|
+
35,
|
|
134
|
+
50,
|
|
135
|
+
75,
|
|
136
|
+
100,
|
|
137
|
+
250,
|
|
138
|
+
500,
|
|
139
|
+
750,
|
|
140
|
+
1_000,
|
|
141
|
+
2_500,
|
|
142
|
+
5_000,
|
|
143
|
+
7_500,
|
|
144
|
+
10_000,
|
|
145
|
+
15_000,
|
|
146
|
+
30_000,
|
|
147
|
+
60_000
|
|
148
|
+
], true)
|
|
149
|
+
}),
|
|
150
|
+
new View({
|
|
151
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
152
|
+
instrumentUnit: 'us',
|
|
153
|
+
aggregation: new ExplicitBucketHistogramAggregation(// 1us to 1s
|
|
154
|
+
[
|
|
155
|
+
5,
|
|
156
|
+
10,
|
|
157
|
+
25,
|
|
158
|
+
50,
|
|
159
|
+
75,
|
|
160
|
+
100,
|
|
161
|
+
250,
|
|
162
|
+
500,
|
|
163
|
+
750,
|
|
164
|
+
1_000,
|
|
165
|
+
2_500,
|
|
166
|
+
5_000,
|
|
167
|
+
7_500,
|
|
168
|
+
10_000,
|
|
169
|
+
25_000,
|
|
170
|
+
50_000,
|
|
171
|
+
100_000,
|
|
172
|
+
250_000,
|
|
173
|
+
500_000,
|
|
174
|
+
1_000_000
|
|
175
|
+
], true)
|
|
176
|
+
}),
|
|
177
|
+
new View({
|
|
178
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
179
|
+
instrumentUnit: 'By',
|
|
180
|
+
aggregation: new ExplicitBucketHistogramAggregation(// from 32 bytes to 2MB
|
|
181
|
+
[
|
|
182
|
+
32,
|
|
183
|
+
64,
|
|
184
|
+
128,
|
|
185
|
+
256,
|
|
186
|
+
512,
|
|
187
|
+
1024,
|
|
188
|
+
2048,
|
|
189
|
+
4096,
|
|
190
|
+
8192,
|
|
191
|
+
16384,
|
|
192
|
+
32768,
|
|
193
|
+
65536,
|
|
194
|
+
131072,
|
|
195
|
+
262144,
|
|
196
|
+
524288,
|
|
197
|
+
1048576,
|
|
198
|
+
1572864,
|
|
199
|
+
2097152
|
|
200
|
+
], true)
|
|
201
|
+
}),
|
|
202
|
+
new View({
|
|
203
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
204
|
+
instrumentUnit: 'gas/s',
|
|
205
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
206
|
+
1_000,
|
|
207
|
+
5_000,
|
|
208
|
+
10_000,
|
|
209
|
+
25_000,
|
|
210
|
+
50_000,
|
|
211
|
+
100_000,
|
|
212
|
+
250_000,
|
|
213
|
+
500_000,
|
|
214
|
+
750_000,
|
|
215
|
+
1_000_000,
|
|
216
|
+
2_000_000,
|
|
217
|
+
4_000_000,
|
|
218
|
+
8_000_000,
|
|
219
|
+
10_000_000,
|
|
220
|
+
15_000_000,
|
|
221
|
+
30_000_000
|
|
222
|
+
], true)
|
|
223
|
+
}),
|
|
224
|
+
new View({
|
|
225
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
226
|
+
instrumentUnit: 'mana/s',
|
|
227
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
228
|
+
1_000,
|
|
229
|
+
5_000,
|
|
230
|
+
10_000,
|
|
231
|
+
25_000,
|
|
232
|
+
50_000,
|
|
233
|
+
100_000,
|
|
234
|
+
250_000,
|
|
235
|
+
500_000,
|
|
236
|
+
750_000,
|
|
237
|
+
1_000_000,
|
|
238
|
+
2_000_000,
|
|
239
|
+
4_000_000,
|
|
240
|
+
8_000_000,
|
|
241
|
+
10_000_000,
|
|
242
|
+
15_000_000,
|
|
243
|
+
30_000_000
|
|
244
|
+
], true)
|
|
245
|
+
}),
|
|
246
|
+
new View({
|
|
247
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
248
|
+
instrumentUnit: 'gas/block',
|
|
249
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
250
|
+
1_000,
|
|
251
|
+
5_000,
|
|
252
|
+
10_000,
|
|
253
|
+
25_000,
|
|
254
|
+
50_000,
|
|
255
|
+
100_000,
|
|
256
|
+
250_000,
|
|
257
|
+
500_000,
|
|
258
|
+
750_000,
|
|
259
|
+
1_000_000,
|
|
260
|
+
2_000_000,
|
|
261
|
+
4_000_000,
|
|
262
|
+
8_000_000,
|
|
263
|
+
10_000_000,
|
|
264
|
+
15_000_000,
|
|
265
|
+
30_000_000
|
|
266
|
+
], true)
|
|
267
|
+
}),
|
|
268
|
+
new View({
|
|
269
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
270
|
+
instrumentUnit: 'gas/tx',
|
|
271
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
272
|
+
25_000,
|
|
273
|
+
50_000,
|
|
274
|
+
100_000,
|
|
275
|
+
250_000,
|
|
276
|
+
500_000,
|
|
277
|
+
750_000,
|
|
278
|
+
1_000_000,
|
|
279
|
+
2_000_000,
|
|
280
|
+
4_000_000,
|
|
281
|
+
8_000_000,
|
|
282
|
+
10_000_000,
|
|
283
|
+
15_000_000,
|
|
284
|
+
30_000_000
|
|
285
|
+
], true)
|
|
286
|
+
})
|
|
287
|
+
]
|
|
288
|
+
});
|
|
289
|
+
}
|
|
290
|
+
static getCustomClientFactory(config) {
|
|
291
|
+
return (resource, log)=>{
|
|
292
|
+
const tracerProvider = new NodeTracerProvider({
|
|
293
|
+
resource,
|
|
294
|
+
spanProcessors: config.tracesCollectorUrl ? [
|
|
295
|
+
new BatchSpanProcessor(new OTLPTraceExporter({
|
|
296
|
+
url: config.tracesCollectorUrl.href
|
|
297
|
+
}))
|
|
298
|
+
] : []
|
|
299
|
+
});
|
|
300
|
+
tracerProvider.register();
|
|
301
|
+
const meterProvider = OpenTelemetryClient.createMeterProvider(resource, {
|
|
302
|
+
exporter: config.metricsCollectorUrl ? new OtelFilterMetricExporter(new OTLPMetricExporter({
|
|
303
|
+
url: config.metricsCollectorUrl.href
|
|
304
|
+
}), config.otelExcludeMetrics ?? []) : undefined,
|
|
305
|
+
exportTimeoutMillis: config.otelExportTimeoutMs,
|
|
306
|
+
exportIntervalMillis: config.otelCollectIntervalMs
|
|
307
|
+
});
|
|
308
|
+
const loggerProvider = registerOtelLoggerProvider(resource, config.logsCollectorUrl);
|
|
309
|
+
return new OpenTelemetryClient(resource, meterProvider, tracerProvider, loggerProvider, log);
|
|
310
|
+
};
|
|
311
|
+
}
|
|
312
|
+
static createAndStart(config, log) {
|
|
313
|
+
const resource = getOtelResource();
|
|
314
|
+
const factory = OpenTelemetryClient.getCustomClientFactory(config);
|
|
315
|
+
const service = factory(resource, log);
|
|
316
|
+
service.start();
|
|
317
|
+
return service;
|
|
318
|
+
}
|
|
319
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import type { ExportResult } from '@opentelemetry/core';
|
|
2
|
+
import type { PushMetricExporter, ResourceMetrics } from '@opentelemetry/sdk-metrics';
|
|
3
|
+
export declare class OtelFilterMetricExporter implements PushMetricExporter {
|
|
4
|
+
private readonly exporter;
|
|
5
|
+
private readonly excludeMetricPrefixes;
|
|
6
|
+
constructor(exporter: PushMetricExporter, excludeMetricPrefixes: string[]);
|
|
7
|
+
export(metrics: ResourceMetrics, resultCallback: (result: ExportResult) => void): void;
|
|
8
|
+
private filterMetrics;
|
|
9
|
+
forceFlush(): Promise<void>;
|
|
10
|
+
shutdown(): Promise<void>;
|
|
11
|
+
}
|
|
12
|
+
//# sourceMappingURL=otel_filter_metric_exporter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel_filter_metric_exporter.d.ts","sourceRoot":"","sources":["../src/otel_filter_metric_exporter.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AACxD,OAAO,KAAK,EAAc,kBAAkB,EAAE,eAAe,EAAE,MAAM,4BAA4B,CAAC;AAElG,qBAAa,wBAAyB,YAAW,kBAAkB;IACrD,OAAO,CAAC,QAAQ,CAAC,QAAQ;IAAsB,OAAO,CAAC,QAAQ,CAAC,qBAAqB;gBAApE,QAAQ,EAAE,kBAAkB,EAAmB,qBAAqB,EAAE,MAAM,EAAE;IASpG,MAAM,CAAC,OAAO,EAAE,eAAe,EAAE,cAAc,EAAE,CAAC,MAAM,EAAE,YAAY,KAAK,IAAI,GAAG,IAAI;IAW7F,OAAO,CAAC,aAAa;IAMd,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;IAI3B,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC;CAGjC"}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
export class OtelFilterMetricExporter {
|
|
2
|
+
exporter;
|
|
3
|
+
excludeMetricPrefixes;
|
|
4
|
+
constructor(exporter, excludeMetricPrefixes){
|
|
5
|
+
this.exporter = exporter;
|
|
6
|
+
this.excludeMetricPrefixes = excludeMetricPrefixes;
|
|
7
|
+
if (exporter.selectAggregation) {
|
|
8
|
+
this.selectAggregation = exporter.selectAggregation.bind(exporter);
|
|
9
|
+
}
|
|
10
|
+
if (exporter.selectAggregationTemporality) {
|
|
11
|
+
this.selectAggregationTemporality = exporter.selectAggregationTemporality.bind(exporter);
|
|
12
|
+
}
|
|
13
|
+
}
|
|
14
|
+
export(metrics, resultCallback) {
|
|
15
|
+
const filteredMetrics = {
|
|
16
|
+
resource: metrics.resource,
|
|
17
|
+
scopeMetrics: metrics.scopeMetrics.map(({ scope, metrics })=>({
|
|
18
|
+
scope,
|
|
19
|
+
metrics: this.filterMetrics(metrics)
|
|
20
|
+
})).filter(({ metrics })=>metrics.length > 0)
|
|
21
|
+
};
|
|
22
|
+
this.exporter.export(filteredMetrics, resultCallback);
|
|
23
|
+
}
|
|
24
|
+
filterMetrics(metrics) {
|
|
25
|
+
return metrics.filter((metric)=>!this.excludeMetricPrefixes.some((prefix)=>metric.descriptor.name.startsWith(prefix)));
|
|
26
|
+
}
|
|
27
|
+
forceFlush() {
|
|
28
|
+
return this.exporter.forceFlush();
|
|
29
|
+
}
|
|
30
|
+
shutdown() {
|
|
31
|
+
return this.exporter.shutdown();
|
|
32
|
+
}
|
|
33
|
+
}
|
|
@@ -0,0 +1,4 @@
|
|
|
1
|
+
import type { IResource } from '@opentelemetry/resources';
|
|
2
|
+
import { LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
3
|
+
export declare function registerOtelLoggerProvider(resource: IResource, otelLogsUrl?: URL): LoggerProvider;
|
|
4
|
+
//# sourceMappingURL=otel_logger_provider.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel_logger_provider.d.ts","sourceRoot":"","sources":["../src/otel_logger_provider.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAA2B,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAElF,wBAAgB,0BAA0B,CAAC,QAAQ,EAAE,SAAS,EAAE,WAAW,CAAC,EAAE,GAAG,kBAwBhF"}
|
|
@@ -0,0 +1,25 @@
|
|
|
1
|
+
import { logs as otelLogs } from '@opentelemetry/api-logs';
|
|
2
|
+
import { OTLPLogExporter } from '@opentelemetry/exporter-logs-otlp-http';
|
|
3
|
+
import { CompressionAlgorithm } from '@opentelemetry/otlp-exporter-base';
|
|
4
|
+
import { BatchLogRecordProcessor, LoggerProvider } from '@opentelemetry/sdk-logs';
|
|
5
|
+
export function registerOtelLoggerProvider(resource, otelLogsUrl) {
|
|
6
|
+
const loggerProvider = new LoggerProvider({
|
|
7
|
+
resource
|
|
8
|
+
});
|
|
9
|
+
if (!otelLogsUrl) {
|
|
10
|
+
// If no URL provided, return it disconnected.
|
|
11
|
+
return loggerProvider;
|
|
12
|
+
}
|
|
13
|
+
const logExporter = new OTLPLogExporter({
|
|
14
|
+
compression: CompressionAlgorithm.GZIP,
|
|
15
|
+
url: otelLogsUrl.href
|
|
16
|
+
});
|
|
17
|
+
// Add a processor to the logger provider
|
|
18
|
+
loggerProvider.addLogRecordProcessor(new BatchLogRecordProcessor(logExporter, {
|
|
19
|
+
/** The maximum batch size of every export. It must be smaller or equal to
|
|
20
|
+
* maxQueueSize. */ maxExportBatchSize: 1024,
|
|
21
|
+
/** The maximum queue size. After the size is reached log records are dropped. */ maxQueueSize: 4096
|
|
22
|
+
}));
|
|
23
|
+
otelLogs.setGlobalLoggerProvider(loggerProvider);
|
|
24
|
+
return loggerProvider;
|
|
25
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAU3B,wBAAgB,mCAAmC,CACjD,KAAK,SAAkB,GACtB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAoChE"}
|
|
@@ -0,0 +1,44 @@
|
|
|
1
|
+
import { ROOT_CONTEXT, SpanKind, SpanStatusCode, propagation } from '@opentelemetry/api';
|
|
2
|
+
import { getTelemetryClient } from './start.js';
|
|
3
|
+
import { ATTR_JSONRPC_ERROR_CODE, ATTR_JSONRPC_ERROR_MSG, ATTR_JSONRPC_METHOD, ATTR_JSONRPC_REQUEST_ID } from './vendor/attributes.js';
|
|
4
|
+
export function getOtelJsonRpcPropagationMiddleware(scope = 'JsonRpcServer') {
|
|
5
|
+
return function otelJsonRpcPropagation(ctx, next) {
|
|
6
|
+
const tracer = getTelemetryClient().getTracer(scope);
|
|
7
|
+
const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
|
|
8
|
+
const method = ctx.request.body?.method;
|
|
9
|
+
return tracer.startActiveSpan(`JsonRpcServer.${method ?? 'unknown'}`, {
|
|
10
|
+
kind: SpanKind.SERVER
|
|
11
|
+
}, context, async (span)=>{
|
|
12
|
+
if (ctx.id) {
|
|
13
|
+
span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
|
|
14
|
+
}
|
|
15
|
+
if (method) {
|
|
16
|
+
span.setAttribute(ATTR_JSONRPC_METHOD, method);
|
|
17
|
+
}
|
|
18
|
+
try {
|
|
19
|
+
await next();
|
|
20
|
+
const err = ctx.body.error?.message;
|
|
21
|
+
const code = ctx.body.error?.code;
|
|
22
|
+
if (err) {
|
|
23
|
+
span.setStatus({
|
|
24
|
+
code: SpanStatusCode.ERROR,
|
|
25
|
+
message: err
|
|
26
|
+
});
|
|
27
|
+
span.setAttribute(ATTR_JSONRPC_ERROR_CODE, code);
|
|
28
|
+
span.setAttribute(ATTR_JSONRPC_ERROR_MSG, err);
|
|
29
|
+
} else {
|
|
30
|
+
span.setStatus({
|
|
31
|
+
code: SpanStatusCode.OK
|
|
32
|
+
});
|
|
33
|
+
}
|
|
34
|
+
} catch (err) {
|
|
35
|
+
span.setStatus({
|
|
36
|
+
code: SpanStatusCode.ERROR,
|
|
37
|
+
message: String(err)
|
|
38
|
+
});
|
|
39
|
+
} finally{
|
|
40
|
+
span.end();
|
|
41
|
+
}
|
|
42
|
+
});
|
|
43
|
+
};
|
|
44
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"otel_resource.d.ts","sourceRoot":"","sources":["../src/otel_resource.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,SAAS,EAMf,MAAM,0BAA0B,CAAC;AAElC,wBAAgB,eAAe,IAAI,SAAS,CAM3C"}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { detectResourcesSync, envDetectorSync, osDetectorSync, processDetectorSync, serviceInstanceIdDetectorSync } from '@opentelemetry/resources';
|
|
2
|
+
export function getOtelResource() {
|
|
3
|
+
const resource = detectResourcesSync({
|
|
4
|
+
detectors: [
|
|
5
|
+
osDetectorSync,
|
|
6
|
+
envDetectorSync,
|
|
7
|
+
processDetectorSync,
|
|
8
|
+
serviceInstanceIdDetectorSync
|
|
9
|
+
]
|
|
10
|
+
});
|
|
11
|
+
return resource;
|
|
12
|
+
}
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
import { type Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { Registry } from 'prom-client';
|
|
3
|
+
import type { Meter, TelemetryClient } from './telemetry.js';
|
|
4
|
+
/**
|
|
5
|
+
* Types matching the gossipsub and libp2p services
|
|
6
|
+
*/
|
|
7
|
+
type TopicStr = string;
|
|
8
|
+
export type TopicLabel = string;
|
|
9
|
+
export type TopicStrToLabel = Map<TopicStr, TopicLabel>;
|
|
10
|
+
export declare enum MessageSource {
|
|
11
|
+
forward = "forward",
|
|
12
|
+
publish = "publish"
|
|
13
|
+
}
|
|
14
|
+
type NoLabels = Record<string, never>;
|
|
15
|
+
type LabelsGeneric = Record<string, string | number>;
|
|
16
|
+
type LabelKeys<Labels extends LabelsGeneric> = Extract<keyof Labels, string>;
|
|
17
|
+
interface CollectFn<Labels extends LabelsGeneric> {
|
|
18
|
+
(metric: IGauge<Labels>): void;
|
|
19
|
+
}
|
|
20
|
+
interface IGauge<Labels extends LabelsGeneric = NoLabels> {
|
|
21
|
+
inc: NoLabels extends Labels ? (value?: number) => void : (labels: Labels, value?: number) => void;
|
|
22
|
+
set: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void;
|
|
23
|
+
collect?(): void;
|
|
24
|
+
addCollect(collectFn: CollectFn<Labels>): void;
|
|
25
|
+
}
|
|
26
|
+
interface IHistogram<Labels extends LabelsGeneric = NoLabels> {
|
|
27
|
+
startTimer(): () => void;
|
|
28
|
+
observe: NoLabels extends Labels ? (value: number) => void : (labels: Labels, value: number) => void;
|
|
29
|
+
reset(): void;
|
|
30
|
+
}
|
|
31
|
+
interface IAvgMinMax<Labels extends LabelsGeneric = NoLabels> {
|
|
32
|
+
set: NoLabels extends Labels ? (values: number[]) => void : (labels: Labels, values: number[]) => void;
|
|
33
|
+
}
|
|
34
|
+
export type GaugeConfig<Labels extends LabelsGeneric> = {
|
|
35
|
+
name: string;
|
|
36
|
+
help: string;
|
|
37
|
+
} & (NoLabels extends Labels ? {
|
|
38
|
+
labelNames?: never;
|
|
39
|
+
} : {
|
|
40
|
+
labelNames: [LabelKeys<Labels>, ...Array<LabelKeys<Labels>>];
|
|
41
|
+
});
|
|
42
|
+
export type HistogramConfig<Labels extends LabelsGeneric> = GaugeConfig<Labels> & {
|
|
43
|
+
buckets?: number[];
|
|
44
|
+
};
|
|
45
|
+
export type AvgMinMaxConfig<Labels extends LabelsGeneric> = GaugeConfig<Labels>;
|
|
46
|
+
export interface MetricsRegister {
|
|
47
|
+
gauge<Labels extends LabelsGeneric = NoLabels>(config: GaugeConfig<Labels>): IGauge<Labels>;
|
|
48
|
+
histogram<Labels extends LabelsGeneric = NoLabels>(config: HistogramConfig<Labels>): IHistogram<Labels>;
|
|
49
|
+
avgMinMax<Labels extends LabelsGeneric = NoLabels>(config: AvgMinMaxConfig<Labels>): IAvgMinMax<Labels>;
|
|
50
|
+
}
|
|
51
|
+
/**Otel MetricsType Adapters
|
|
52
|
+
*
|
|
53
|
+
* Some dependencies we use export metrics directly in a Prometheus format
|
|
54
|
+
* This adapter is used to convert those metrics to a format that we can use with OpenTelemetry
|
|
55
|
+
*
|
|
56
|
+
* Affected services include:
|
|
57
|
+
* - chainsafe/gossipsub
|
|
58
|
+
* - libp2p
|
|
59
|
+
*/
|
|
60
|
+
export declare class OtelGauge<Labels extends LabelsGeneric = NoLabels> implements IGauge<Labels> {
|
|
61
|
+
private logger;
|
|
62
|
+
private labelNames;
|
|
63
|
+
private gauge;
|
|
64
|
+
private currentValue;
|
|
65
|
+
private labeledValues;
|
|
66
|
+
private collectFns;
|
|
67
|
+
private _collect;
|
|
68
|
+
get collect(): () => void;
|
|
69
|
+
set collect(fn: () => void);
|
|
70
|
+
constructor(logger: Logger, meter: Meter, name: string, help: string, labelNames?: Array<keyof Labels>);
|
|
71
|
+
/**
|
|
72
|
+
* Add a collect callback
|
|
73
|
+
* @param collectFn - Callback function
|
|
74
|
+
*/
|
|
75
|
+
addCollect(collectFn: CollectFn<Labels>): void;
|
|
76
|
+
handleObservation(result: any): void;
|
|
77
|
+
/**
|
|
78
|
+
* Increments the gauge value
|
|
79
|
+
* @param labelsOrValue - Labels object or numeric value
|
|
80
|
+
* @param value - Value to increment by (defaults to 1)
|
|
81
|
+
*/
|
|
82
|
+
inc(value?: number): void;
|
|
83
|
+
inc(labels: Labels, value?: number): void;
|
|
84
|
+
/**
|
|
85
|
+
* Sets the gauge value
|
|
86
|
+
* @param labelsOrValue - Labels object or numeric value
|
|
87
|
+
* @param value - Value to set
|
|
88
|
+
*/
|
|
89
|
+
set(value: number): void;
|
|
90
|
+
set(labels: Labels, value: number): void;
|
|
91
|
+
/**
|
|
92
|
+
* Decrements the gauge value
|
|
93
|
+
* @param labels - Optional labels object
|
|
94
|
+
*/
|
|
95
|
+
dec(labels?: Labels): void;
|
|
96
|
+
/**
|
|
97
|
+
* Resets the gauge to initial state
|
|
98
|
+
*/
|
|
99
|
+
reset(): void;
|
|
100
|
+
/**
|
|
101
|
+
* Validates that provided labels match the expected schema
|
|
102
|
+
* @param labels - Labels object to validate
|
|
103
|
+
* @throws Error if invalid labels are provided
|
|
104
|
+
*/
|
|
105
|
+
private validateLabels;
|
|
106
|
+
/**
|
|
107
|
+
* Safely parses label string back to object
|
|
108
|
+
* @param labelStr - Stringified labels object
|
|
109
|
+
* @returns Labels object or null if parsing fails
|
|
110
|
+
*/
|
|
111
|
+
private parseLabelsSafely;
|
|
112
|
+
}
|
|
113
|
+
/**
|
|
114
|
+
* Otel metrics Adapter
|
|
115
|
+
*
|
|
116
|
+
* Maps the PromClient based MetricsRegister from gossipsub and discv5 services to the Otel MetricsRegister
|
|
117
|
+
*/
|
|
118
|
+
export declare class OtelMetricsAdapter extends Registry implements MetricsRegister {
|
|
119
|
+
private logger;
|
|
120
|
+
private readonly meter;
|
|
121
|
+
constructor(telemetryClient: TelemetryClient, logger?: Logger);
|
|
122
|
+
gauge<Labels extends LabelsGeneric = NoLabels>(configuration: GaugeConfig<Labels>): IGauge<Labels>;
|
|
123
|
+
histogram<Labels extends LabelsGeneric = NoLabels>(configuration: HistogramConfig<Labels>): IHistogram<Labels>;
|
|
124
|
+
avgMinMax<Labels extends LabelsGeneric = NoLabels>(configuration: AvgMinMaxConfig<Labels>): IAvgMinMax<Labels>;
|
|
125
|
+
}
|
|
126
|
+
export {};
|
|
127
|
+
//# sourceMappingURL=prom_otel_adapter.d.ts.map
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"prom_otel_adapter.d.ts","sourceRoot":"","sources":["../src/prom_otel_adapter.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,MAAM,EAAgB,MAAM,uBAAuB,CAAC;AAElE,OAAO,EAAE,QAAQ,EAAE,MAAM,aAAa,CAAC;AAEvC,OAAO,KAAK,EAAE,KAAK,EAAgC,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAE3F;;GAEG;AACH,KAAK,QAAQ,GAAG,MAAM,CAAC;AACvB,MAAM,MAAM,UAAU,GAAG,MAAM,CAAC;AAChC,MAAM,MAAM,eAAe,GAAG,GAAG,CAAC,QAAQ,EAAE,UAAU,CAAC,CAAC;AAExD,oBAAY,aAAa;IACvB,OAAO,YAAY;IACnB,OAAO,YAAY;CACpB;AAED,KAAK,QAAQ,GAAG,MAAM,CAAC,MAAM,EAAE,KAAK,CAAC,CAAC;AACtC,KAAK,aAAa,GAAG,MAAM,CAAC,MAAM,EAAE,MAAM,GAAG,MAAM,CAAC,CAAC;AACrD,KAAK,SAAS,CAAC,MAAM,SAAS,aAAa,IAAI,OAAO,CAAC,MAAM,MAAM,EAAE,MAAM,CAAC,CAAC;AAC7E,UAAU,SAAS,CAAC,MAAM,SAAS,aAAa;IAC9C,CAAC,MAAM,EAAE,MAAM,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;CAChC;AAED,UAAU,MAAM,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ;IACtD,GAAG,EAAE,QAAQ,SAAS,MAAM,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,KAAK,IAAI,CAAC;IACnG,GAAG,EAAE,QAAQ,SAAS,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAEjG,OAAO,CAAC,IAAI,IAAI,CAAC;IACjB,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI,CAAC;CAChD;AAED,UAAU,UAAU,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ;IAC1D,UAAU,IAAI,MAAM,IAAI,CAAC;IAEzB,OAAO,EAAE,QAAQ,SAAS,MAAM,GAAG,CAAC,KAAK,EAAE,MAAM,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,KAAK,IAAI,CAAC;IAErG,KAAK,IAAI,IAAI,CAAC;CACf;AAED,UAAU,UAAU,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ;IAC1D,GAAG,EAAE,QAAQ,SAAS,MAAM,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,MAAM,EAAE,KAAK,IAAI,CAAC;CACxG;AAED,MAAM,MAAM,WAAW,CAAC,MAAM,SAAS,aAAa,IAAI;IACtD,IAAI,EAAE,MAAM,CAAC;IACb,IAAI,EAAE,MAAM,CAAC;CACd,GAAG,CAAC,QAAQ,SAAS,MAAM,GACxB;IAAE,UAAU,CAAC,EAAE,KAAK,CAAA;CAAE,GACtB;IAAE,UAAU,EAAE,CAAC,SAAS,CAAC,MAAM,CAAC,EAAE,GAAG,KAAK,CAAC,SAAS,CAAC,MAAM,CAAC,CAAC,CAAC,CAAA;CAAE,CAAC,CAAC;AAEtE,MAAM,MAAM,eAAe,CAAC,MAAM,SAAS,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,GAAG;IAChF,OAAO,CAAC,EAAE,MAAM,EAAE,CAAC;CACpB,CAAC;AAEF,MAAM,MAAM,eAAe,CAAC,MAAM,SAAS,aAAa,IAAI,WAAW,CAAC,MAAM,CAAC,CAAC;AAEhF,MAAM,WAAW,eAAe;IAC9B,KAAK,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,MAAM,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC,CAAC;IAC5F,SAAS,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;IACxG,SAAS,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,MAAM,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC,CAAC;CACzG;AAED;;;;;;;;GAQG;AAEH,qBAAa,SAAS,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,CAAE,YAAW,MAAM,CAAC,MAAM,CAAC;IAerF,OAAO,CAAC,MAAM;IAId,OAAO,CAAC,UAAU;IAlBpB,OAAO,CAAC,KAAK,CAAkB;IAC/B,OAAO,CAAC,YAAY,CAAa;IACjC,OAAO,CAAC,aAAa,CAAkC;IACvD,OAAO,CAAC,UAAU,CAA2B;IAE7C,OAAO,CAAC,QAAQ,CAAwB;IACxC,IAAI,OAAO,IAAI,MAAM,IAAI,CAExB;IACD,IAAI,OAAO,CAAC,EAAE,EAAE,MAAM,IAAI,EAEzB;gBAGS,MAAM,EAAE,MAAM,EACtB,KAAK,EAAE,KAAK,EACZ,IAAI,EAAE,MAAM,EACZ,IAAI,EAAE,MAAM,EACJ,UAAU,GAAE,KAAK,CAAC,MAAM,MAAM,CAAM;IAU9C;;;OAGG;IACH,UAAU,CAAC,SAAS,EAAE,SAAS,CAAC,MAAM,CAAC,GAAG,IAAI;IAI9C,iBAAiB,CAAC,MAAM,EAAE,GAAG,GAAG,IAAI;IAuBpC;;;;OAIG;IACH,GAAG,CAAC,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IACzB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,CAAC,EAAE,MAAM,GAAG,IAAI;IAkBzC;;;;OAIG;IACH,GAAG,CAAC,KAAK,EAAE,MAAM,GAAG,IAAI;IACxB,GAAG,CAAC,MAAM,EAAE,MAAM,EAAE,KAAK,EAAE,MAAM,GAAG,IAAI;IAYxC;;;OAGG;IACH,GAAG,CAAC,MAAM,CAAC,EAAE,MAAM,GAAG,IAAI;IAY1B;;OAEG;IACH,KAAK,IAAI,IAAI;IAKb;;;;OAIG;IACH,OAAO,CAAC,cAAc;IAYtB;;;;OAIG;IACH,OAAO,CAAC,iBAAiB;CAQ1B;AAiDD;;;;GAIG;AACH,qBAAa,kBAAmB,SAAQ,QAAS,YAAW,eAAe;IAKvE,OAAO,CAAC,MAAM;IAJhB,OAAO,CAAC,QAAQ,CAAC,KAAK,CAAQ;gBAG5B,eAAe,EAAE,eAAe,EACxB,MAAM,GAAE,MAAuD;IAMzE,KAAK,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,aAAa,EAAE,WAAW,CAAC,MAAM,CAAC,GAAG,MAAM,CAAC,MAAM,CAAC;IAUlG,SAAS,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,aAAa,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;IAW9G,SAAS,CAAC,MAAM,SAAS,aAAa,GAAG,QAAQ,EAAE,aAAa,EAAE,eAAe,CAAC,MAAM,CAAC,GAAG,UAAU,CAAC,MAAM,CAAC;CAS/G"}
|