@aztec/telemetry-client 0.0.1-commit.c2eed6949 → 0.0.1-commit.c52d6e7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/attributes.d.ts +7 -2
- package/dest/attributes.d.ts.map +1 -1
- package/dest/attributes.js +3 -1
- package/dest/config.d.ts +3 -1
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +17 -9
- package/dest/lmdb_metrics.d.ts +2 -2
- package/dest/lmdb_metrics.d.ts.map +1 -1
- package/dest/metrics.d.ts +28 -4
- package/dest/metrics.d.ts.map +1 -1
- package/dest/metrics.js +148 -17
- package/dest/monitored_batch_span_processor.d.ts +29 -0
- package/dest/monitored_batch_span_processor.d.ts.map +1 -0
- package/dest/monitored_batch_span_processor.js +75 -0
- package/dest/otel.d.ts +6 -1
- package/dest/otel.d.ts.map +1 -1
- package/dest/otel.js +99 -4
- package/dest/otel_propagation.d.ts +3 -1
- package/dest/otel_propagation.d.ts.map +1 -1
- package/dest/otel_propagation.js +49 -1
- package/dest/start.d.ts +1 -1
- package/dest/start.d.ts.map +1 -1
- package/dest/start.js +1 -1
- package/dest/telemetry.d.ts +2 -2
- package/dest/telemetry.d.ts.map +1 -1
- package/dest/wrappers/fetch.d.ts +3 -3
- package/dest/wrappers/fetch.d.ts.map +1 -1
- package/dest/wrappers/fetch.js +3 -2
- package/dest/wrappers/index.d.ts +1 -2
- package/dest/wrappers/index.d.ts.map +1 -1
- package/dest/wrappers/index.js +0 -1
- package/package.json +3 -3
- package/src/attributes.ts +6 -1
- package/src/config.ts +24 -9
- package/src/metrics.ts +163 -17
- package/src/monitored_batch_span_processor.ts +100 -0
- package/src/otel.ts +69 -3
- package/src/otel_propagation.ts +42 -1
- package/src/start.ts +6 -1
- package/src/telemetry.ts +0 -1
- package/src/wrappers/fetch.ts +9 -3
- package/src/wrappers/index.ts +0 -1
- package/dest/wrappers/l2_block_stream.d.ts +0 -16
- package/dest/wrappers/l2_block_stream.d.ts.map +0 -1
- package/dest/wrappers/l2_block_stream.js +0 -400
- package/src/wrappers/l2_block_stream.ts +0 -41
package/dest/metrics.js
CHANGED
|
@@ -128,7 +128,7 @@ export const MEMPOOL_TX_ADDED_COUNT = {
|
|
|
128
128
|
};
|
|
129
129
|
export const MEMPOOL_TX_MINED_DELAY = {
|
|
130
130
|
name: 'aztec.mempool.tx_mined_delay',
|
|
131
|
-
description: 'Delay
|
|
131
|
+
description: 'Delay (ms) from a transaction being received into the pool to being mined',
|
|
132
132
|
unit: 'ms',
|
|
133
133
|
valueType: ValueType.INT
|
|
134
134
|
};
|
|
@@ -267,6 +267,12 @@ export const ARCHIVER_SYNC_PER_BLOCK = {
|
|
|
267
267
|
unit: 'ms',
|
|
268
268
|
valueType: ValueType.INT
|
|
269
269
|
};
|
|
270
|
+
export const ARCHIVER_SYNC_PER_CHECKPOINT = {
|
|
271
|
+
name: 'aztec.archiver.checkpoint.sync_per_item_duration',
|
|
272
|
+
description: 'Duration to sync a checkpoint',
|
|
273
|
+
unit: 'ms',
|
|
274
|
+
valueType: ValueType.INT
|
|
275
|
+
};
|
|
270
276
|
export const ARCHIVER_SYNC_BLOCK_COUNT = {
|
|
271
277
|
name: 'aztec.archiver.block.sync_count',
|
|
272
278
|
description: 'Number of blocks synced from L1',
|
|
@@ -291,7 +297,7 @@ export const ARCHIVER_PRUNE_DURATION = {
|
|
|
291
297
|
};
|
|
292
298
|
export const ARCHIVER_PRUNE_COUNT = {
|
|
293
299
|
name: 'aztec.archiver.prune_count',
|
|
294
|
-
description: 'Number of prunes detected',
|
|
300
|
+
description: 'Number of prunes detected, dimensioned by prune_type: unproven (epoch prune of checkpoints that will not be proven), uncheckpointed (proposed blocks whose slot ended without a checkpoint), l1_conflict (proposed blocks conflicting with an L1 checkpoint), orphan (proposed blocks whose matching proposed checkpoint never arrived before the deadline), and l1_mismatch (the local checkpointed tip diverged from L1 — an L1 reorg or a pruned/missed-proof checkpoint — rewinding already-checkpointed blocks).',
|
|
295
301
|
valueType: ValueType.INT
|
|
296
302
|
};
|
|
297
303
|
export const ARCHIVER_TOTAL_TXS = {
|
|
@@ -310,6 +316,11 @@ export const ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY = {
|
|
|
310
316
|
unit: 's',
|
|
311
317
|
valueType: ValueType.INT
|
|
312
318
|
};
|
|
319
|
+
export const ARCHIVER_CHECKPOINT_PROMOTED_COUNT = {
|
|
320
|
+
name: 'aztec.archiver.checkpoint_promoted_count',
|
|
321
|
+
description: 'Number of checkpoints promoted from proposed (blob fetch skipped)',
|
|
322
|
+
valueType: ValueType.INT
|
|
323
|
+
};
|
|
313
324
|
export const NODE_RECEIVE_TX_DURATION = {
|
|
314
325
|
name: 'aztec.node.receive_tx.duration',
|
|
315
326
|
description: 'The duration of the receiveTx method',
|
|
@@ -338,6 +349,12 @@ export const SEQUENCER_STATE_TRANSITION_BUFFER_DURATION = {
|
|
|
338
349
|
unit: 'ms',
|
|
339
350
|
valueType: ValueType.INT
|
|
340
351
|
};
|
|
352
|
+
export const SEQUENCER_STATE_DURATION = {
|
|
353
|
+
name: 'aztec.sequencer.state_duration',
|
|
354
|
+
description: 'Wall-clock time spent in each sequencer state, labelled by the state being left',
|
|
355
|
+
unit: 'ms',
|
|
356
|
+
valueType: ValueType.INT
|
|
357
|
+
};
|
|
341
358
|
export const SEQUENCER_BLOCK_BUILD_DURATION = {
|
|
342
359
|
name: 'aztec.sequencer.block.build_duration',
|
|
343
360
|
description: 'Duration to build a block',
|
|
@@ -355,6 +372,12 @@ export const SEQUENCER_BLOCK_COUNT = {
|
|
|
355
372
|
description: 'Number of blocks built by this sequencer',
|
|
356
373
|
valueType: ValueType.INT
|
|
357
374
|
};
|
|
375
|
+
export const SEQUENCER_BLOCK_INTER_BLOCK_TIME = {
|
|
376
|
+
name: 'aztec.sequencer.block.inter_block_time',
|
|
377
|
+
description: 'Wall-clock time elapsed between consecutive blocks being built by this sequencer',
|
|
378
|
+
unit: 'ms',
|
|
379
|
+
valueType: ValueType.INT
|
|
380
|
+
};
|
|
358
381
|
export const SEQUENCER_CURRENT_SLOT_REWARDS = {
|
|
359
382
|
name: 'aztec.sequencer.current_slot_rewards',
|
|
360
383
|
description: 'The rewards earned per filled slot',
|
|
@@ -424,6 +447,24 @@ export const SEQUENCER_CHECKPOINT_BUILD_DURATION = {
|
|
|
424
447
|
unit: 'ms',
|
|
425
448
|
valueType: ValueType.INT
|
|
426
449
|
};
|
|
450
|
+
export const SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION = {
|
|
451
|
+
name: 'aztec.sequencer.checkpoint.start_to_first_block_duration',
|
|
452
|
+
description: 'Time from starting checkpoint work to the first block finishing build',
|
|
453
|
+
unit: 'ms',
|
|
454
|
+
valueType: ValueType.INT
|
|
455
|
+
};
|
|
456
|
+
export const SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION = {
|
|
457
|
+
name: 'aztec.sequencer.checkpoint.last_block_to_broadcast_duration',
|
|
458
|
+
description: 'Time from the final block finishing build to the checkpoint proposal being broadcast',
|
|
459
|
+
unit: 'ms',
|
|
460
|
+
valueType: ValueType.INT
|
|
461
|
+
};
|
|
462
|
+
export const SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY = {
|
|
463
|
+
name: 'aztec.sequencer.pipelined_checkpoint.build_start_offset_from_slot_boundary',
|
|
464
|
+
description: 'Absolute offset from the wall-clock slot boundary when a pipelined checkpoint build starts. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
|
|
465
|
+
unit: 'ms',
|
|
466
|
+
valueType: ValueType.INT
|
|
467
|
+
};
|
|
427
468
|
export const SEQUENCER_CHECKPOINT_BLOCK_COUNT = {
|
|
428
469
|
name: 'aztec.sequencer.checkpoint.block_count',
|
|
429
470
|
description: 'Number of blocks built in a checkpoint',
|
|
@@ -446,6 +487,37 @@ export const SEQUENCER_SLASHING_ATTEMPTS_COUNT = {
|
|
|
446
487
|
description: 'The number of slashing action attempts',
|
|
447
488
|
valueType: ValueType.INT
|
|
448
489
|
};
|
|
490
|
+
export const SLASHER_ROUND_EXECUTED_COUNT = {
|
|
491
|
+
name: 'aztec.slasher.round.executed_count',
|
|
492
|
+
description: 'The number of slashing rounds executed',
|
|
493
|
+
valueType: ValueType.INT
|
|
494
|
+
};
|
|
495
|
+
export const SLASHER_OWN_VALIDATOR_TARGETED_COUNT = {
|
|
496
|
+
name: 'aztec.slasher.own_validator.targeted_count',
|
|
497
|
+
description: "The number of times one of the node's own validators was named as a slash target by an onchain vote, counted " + 'once per vote per validator. Votes cast while the node is offline or starting up are not counted, and L1 ' + 'reorgs can cause small drift within a round.',
|
|
498
|
+
valueType: ValueType.INT
|
|
499
|
+
};
|
|
500
|
+
export const SLASHER_OWN_VALIDATOR_CURRENT_ROUND_VOTES_MAX = {
|
|
501
|
+
name: 'aztec.slasher.own_validator.current_round_votes_max',
|
|
502
|
+
description: "Highest number of votes cast against any committee position held by the node's own validators in the current " + 'slashing round, to compare against aztec.slasher.quorum_size (the contract tallies quorum per position). ' + 'Resets to zero when a new round starts. Votes cast while the node is offline or starting up are not counted, ' + 'and L1 reorgs can cause small drift within a round.',
|
|
503
|
+
valueType: ValueType.INT
|
|
504
|
+
};
|
|
505
|
+
export const SLASHER_QUORUM_SIZE = {
|
|
506
|
+
name: 'aztec.slasher.quorum_size',
|
|
507
|
+
description: 'The number of votes a validator must receive in a round to be slashed',
|
|
508
|
+
valueType: ValueType.INT
|
|
509
|
+
};
|
|
510
|
+
export const SLASHER_OWN_VALIDATOR_SLASHED_COUNT = {
|
|
511
|
+
name: 'aztec.slasher.own_validator.slashed_count',
|
|
512
|
+
description: "The number of times one of the node's own validators was slashed, from executed Slashed events",
|
|
513
|
+
valueType: ValueType.INT
|
|
514
|
+
};
|
|
515
|
+
export const SLASHER_OWN_VALIDATOR_SLASHED_AMOUNT = {
|
|
516
|
+
name: 'aztec.slasher.own_validator.slashed_amount',
|
|
517
|
+
description: "Cumulative amount slashed from the node's own validators in whole staking-asset tokens, from executed " + 'Slashed events',
|
|
518
|
+
unit: 'tokens',
|
|
519
|
+
valueType: ValueType.DOUBLE
|
|
520
|
+
};
|
|
449
521
|
export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT = {
|
|
450
522
|
name: 'aztec.sequencer.checkpoint.success_count',
|
|
451
523
|
description: 'The number of times checkpoint publishing succeeded',
|
|
@@ -461,6 +533,11 @@ export const SEQUENCER_PIPELINE_DISCARDS_COUNT = {
|
|
|
461
533
|
description: 'The number of times a pipeline was discarded',
|
|
462
534
|
valueType: ValueType.INT
|
|
463
535
|
};
|
|
536
|
+
export const SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT = {
|
|
537
|
+
name: 'aztec.sequencer.pipeline.parent_checkpoint_mismatch_count',
|
|
538
|
+
description: 'The number of times a pipelined checkpoint was discarded because the parent did not match expectations',
|
|
539
|
+
valueType: ValueType.INT
|
|
540
|
+
};
|
|
464
541
|
// Fisherman fee analysis metrics
|
|
465
542
|
export const FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED = {
|
|
466
543
|
name: 'aztec.fisherman.fee_analysis.would_be_included',
|
|
@@ -625,6 +702,24 @@ export const L1_PUBLISHER_TX_TOTAL_FEE = {
|
|
|
625
702
|
unit: 'eth',
|
|
626
703
|
valueType: ValueType.DOUBLE
|
|
627
704
|
};
|
|
705
|
+
export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS = {
|
|
706
|
+
name: 'aztec.prover_node.estimated_submission.gas',
|
|
707
|
+
description: 'Estimated gas for a proof submission tx (proof publishing disabled, not actually sent)',
|
|
708
|
+
unit: 'gas',
|
|
709
|
+
valueType: ValueType.INT
|
|
710
|
+
};
|
|
711
|
+
export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE = {
|
|
712
|
+
name: 'aztec.prover_node.estimated_submission.gas_price',
|
|
713
|
+
description: 'Estimated effective gas price for a proof submission tx (proof publishing disabled, not actually sent)',
|
|
714
|
+
unit: 'gwei',
|
|
715
|
+
valueType: ValueType.DOUBLE
|
|
716
|
+
};
|
|
717
|
+
export const PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE = {
|
|
718
|
+
name: 'aztec.prover_node.estimated_submission.total_fee',
|
|
719
|
+
description: 'Estimated total L1 fee for a proof submission tx (proof publishing disabled, not actually sent)',
|
|
720
|
+
unit: 'eth',
|
|
721
|
+
valueType: ValueType.DOUBLE
|
|
722
|
+
};
|
|
628
723
|
export const L1_BLOCK_HEIGHT = {
|
|
629
724
|
name: 'aztec.l1.block_height',
|
|
630
725
|
description: 'The current L1 block height',
|
|
@@ -845,6 +940,11 @@ export const P2P_GOSSIP_AGG_MESSAGE_VALIDATION_DURATION_AVG = {
|
|
|
845
940
|
unit: 'ms',
|
|
846
941
|
valueType: ValueType.INT
|
|
847
942
|
};
|
|
943
|
+
export const P2P_GOSSIP_SLOW_VALIDATION_COUNT = {
|
|
944
|
+
name: 'aztec.p2p.gossip.slow_validation_count',
|
|
945
|
+
description: 'Number of gossip validations that exceeded 75% of the mcache eviction window',
|
|
946
|
+
valueType: ValueType.INT
|
|
947
|
+
};
|
|
848
948
|
export const PUBLIC_PROCESSOR_TX_DURATION = {
|
|
849
949
|
name: 'aztec.public_processor.tx_duration',
|
|
850
950
|
description: 'Duration to process a public transaction',
|
|
@@ -908,6 +1008,17 @@ export const PUBLIC_PROCESSOR_TREE_INSERTION = {
|
|
|
908
1008
|
unit: 'ms',
|
|
909
1009
|
valueType: ValueType.INT
|
|
910
1010
|
};
|
|
1011
|
+
export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_COUNT = {
|
|
1012
|
+
name: 'aztec.public_processor.silently_skipped_count',
|
|
1013
|
+
description: 'Public txs fully processed then skipped (e.g. blob-field limit); not counted as processed or failed',
|
|
1014
|
+
valueType: ValueType.INT
|
|
1015
|
+
};
|
|
1016
|
+
export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_DURATION = {
|
|
1017
|
+
name: 'aztec.public_processor.silently_skipped_duration',
|
|
1018
|
+
description: 'Wall-clock time spent processing txs that were then silently skipped',
|
|
1019
|
+
unit: 'ms',
|
|
1020
|
+
valueType: ValueType.INT
|
|
1021
|
+
};
|
|
911
1022
|
export const PUBLIC_EXECUTOR_PREFIX = 'aztec.public_executor.';
|
|
912
1023
|
export const PUBLIC_EXECUTOR_SIMULATION_COUNT = {
|
|
913
1024
|
name: 'aztec.public_executor.simulation_count',
|
|
@@ -1047,12 +1158,6 @@ export const PROVING_AGENT_IDLE = {
|
|
|
1047
1158
|
unit: 's',
|
|
1048
1159
|
valueType: ValueType.DOUBLE
|
|
1049
1160
|
};
|
|
1050
|
-
export const PROVER_NODE_EXECUTION_DURATION = {
|
|
1051
|
-
name: 'aztec.prover_node.execution.duration',
|
|
1052
|
-
description: 'Duration of execution of an epoch by the prover',
|
|
1053
|
-
unit: 'ms',
|
|
1054
|
-
valueType: ValueType.INT
|
|
1055
|
-
};
|
|
1056
1161
|
export const PROVER_NODE_JOB_DURATION = {
|
|
1057
1162
|
name: 'aztec.prover_node.job_duration',
|
|
1058
1163
|
description: 'Duration of proving job',
|
|
@@ -1080,12 +1185,6 @@ export const PROVER_NODE_BLOB_PROCESSING_LAST_DURATION = {
|
|
|
1080
1185
|
unit: 'ms',
|
|
1081
1186
|
valueType: ValueType.INT
|
|
1082
1187
|
};
|
|
1083
|
-
export const PROVER_NODE_CHONK_VERIFIER_LAST_DURATION = {
|
|
1084
|
-
name: 'aztec.prover_node.chonk_verifier.last_duration',
|
|
1085
|
-
description: 'Duration of chonk verifier enqueuing in epoch proving job',
|
|
1086
|
-
unit: 'ms',
|
|
1087
|
-
valueType: ValueType.INT
|
|
1088
|
-
};
|
|
1089
1188
|
export const PROVER_NODE_BLOCK_PROCESSING_DURATION = {
|
|
1090
1189
|
name: 'aztec.prover_node.block_processing.duration',
|
|
1091
1190
|
description: 'Duration of processing a single block in epoch proving job',
|
|
@@ -1098,12 +1197,32 @@ export const PROVER_NODE_CHECKPOINT_PROCESSING_DURATION = {
|
|
|
1098
1197
|
unit: 'ms',
|
|
1099
1198
|
valueType: ValueType.INT
|
|
1100
1199
|
};
|
|
1101
|
-
export const
|
|
1102
|
-
name: 'aztec.prover_node.
|
|
1103
|
-
description: '
|
|
1200
|
+
export const PROVER_NODE_CHECKPOINT_BLOCKS = {
|
|
1201
|
+
name: 'aztec.prover_node.checkpoint_blocks',
|
|
1202
|
+
description: 'Number of blocks in a proven checkpoint',
|
|
1203
|
+
valueType: ValueType.INT
|
|
1204
|
+
};
|
|
1205
|
+
export const PROVER_NODE_CHECKPOINT_TRANSACTIONS = {
|
|
1206
|
+
name: 'aztec.prover_node.checkpoint_transactions',
|
|
1207
|
+
description: 'Number of transactions in a proven checkpoint',
|
|
1208
|
+
valueType: ValueType.INT
|
|
1209
|
+
};
|
|
1210
|
+
export const PROVER_NODE_CHECKPOINT_PROVING_DURATION = {
|
|
1211
|
+
name: 'aztec.prover_node.checkpoint_proving.duration',
|
|
1212
|
+
description: 'Duration from the start of checkpoint processing to its block proofs being ready (excludes tx gathering)',
|
|
1104
1213
|
unit: 'ms',
|
|
1105
1214
|
valueType: ValueType.INT
|
|
1106
1215
|
};
|
|
1216
|
+
export const PROVER_NODE_ACTIVE_CHECKPOINTS = {
|
|
1217
|
+
name: 'aztec.prover_node.active_checkpoints',
|
|
1218
|
+
description: 'Current number of canonical CheckpointProvers in the store (i.e. checkpoints currently being proven)',
|
|
1219
|
+
valueType: ValueType.INT
|
|
1220
|
+
};
|
|
1221
|
+
export const PROVER_NODE_ACTIVE_EPOCH_SESSIONS = {
|
|
1222
|
+
name: 'aztec.prover_node.active_epoch_sessions',
|
|
1223
|
+
description: 'Current number of live EpochSessions, broken down by kind (full|partial)',
|
|
1224
|
+
valueType: ValueType.INT
|
|
1225
|
+
};
|
|
1107
1226
|
export const PROVER_NODE_REWARDS_TOTAL = {
|
|
1108
1227
|
name: 'aztec.prover_node.rewards_total',
|
|
1109
1228
|
description: 'The rewards earned (total)',
|
|
@@ -1214,6 +1333,18 @@ export const VALIDATOR_RE_EXECUTION_TX_COUNT = {
|
|
|
1214
1333
|
unit: 'tx',
|
|
1215
1334
|
valueType: ValueType.INT
|
|
1216
1335
|
};
|
|
1336
|
+
export const VALIDATOR_CHECKPOINT_PROPOSAL_TO_PIPELINED_STATE_DURATION = {
|
|
1337
|
+
name: 'aztec.validator.checkpoint_proposal_to_pipelined_state_duration',
|
|
1338
|
+
description: 'Time from receiving a checkpoint proposal to setting proposed checkpoint state for pipelining',
|
|
1339
|
+
unit: 'ms',
|
|
1340
|
+
valueType: ValueType.INT
|
|
1341
|
+
};
|
|
1342
|
+
export const VALIDATOR_CHECKPOINT_PROPOSAL_RECEIVE_OFFSET_FROM_NEXT_SLOT_BOUNDARY = {
|
|
1343
|
+
name: 'aztec.validator.checkpoint.proposal_receive_offset_from_next_slot_boundary',
|
|
1344
|
+
description: 'Absolute offset from the next slot boundary when a foreign checkpoint proposal is received. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
|
|
1345
|
+
unit: 'ms',
|
|
1346
|
+
valueType: ValueType.INT
|
|
1347
|
+
};
|
|
1217
1348
|
export const VALIDATOR_FAILED_REEXECUTION_COUNT = {
|
|
1218
1349
|
name: 'aztec.validator.failed_reexecution_count',
|
|
1219
1350
|
description: 'The number of failed re-executions',
|
|
@@ -0,0 +1,29 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { type Context } from '@opentelemetry/api';
|
|
3
|
+
import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
|
|
4
|
+
import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
|
|
5
|
+
export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
|
|
6
|
+
minTraceDurationMs?: number;
|
|
7
|
+
};
|
|
8
|
+
/**
|
|
9
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
10
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
11
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
12
|
+
*/
|
|
13
|
+
export declare class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
14
|
+
private readonly maxQueueSize;
|
|
15
|
+
private readonly minTraceDurationMs;
|
|
16
|
+
private readonly log;
|
|
17
|
+
private approxQueueSize;
|
|
18
|
+
private droppedSinceLastWarning;
|
|
19
|
+
private totalDropped;
|
|
20
|
+
private lastWarningTime;
|
|
21
|
+
constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig);
|
|
22
|
+
onStart(span: Span, parentContext: Context): void;
|
|
23
|
+
onEnd(span: ReadableSpan): void;
|
|
24
|
+
forceFlush(): Promise<void>;
|
|
25
|
+
shutdown(): Promise<void>;
|
|
26
|
+
private shouldDropShortSpan;
|
|
27
|
+
private maybeLogDropWarning;
|
|
28
|
+
}
|
|
29
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXBELE9BQU8sRUFBRSxLQUFLLE9BQU8sRUFBa0IsTUFBTSxvQkFBb0IsQ0FBQztBQUVsRSxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUNsRSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxJQUFJLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQWFwSCxNQUFNLE1BQU0saUNBQWlDLEdBQUcsWUFBWSxHQUFHO0lBQzdELGtCQUFrQixDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQzdCLENBQUM7QUFFRjs7OztHQUlHO0FBQ0gscUJBQWEsMkJBQTRCLFNBQVEsa0JBQWtCO0lBQ2pFLE9BQU8sQ0FBQyxRQUFRLENBQUMsWUFBWSxDQUFTO0lBQ3RDLE9BQU8sQ0FBQyxRQUFRLENBQUMsa0JBQWtCLENBQVM7SUFDNUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFFN0IsT0FBTyxDQUFDLGVBQWUsQ0FBSztJQUM1QixPQUFPLENBQUMsdUJBQXVCLENBQUs7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBSztJQUN6QixPQUFPLENBQUMsZUFBZSxDQUFLO0lBRTVCLFlBQVksUUFBUSxFQUFFLFlBQVksRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxFQUFFLGlDQUFpQyxFQU8xRjtJQUVRLE9BQU8sQ0FBQyxJQUFJLEVBQUUsSUFBSSxFQUFFLGFBQWEsRUFBRSxPQUFPLEdBQUcsSUFBSSxDQUV6RDtJQUVRLEtBQUssQ0FBQyxJQUFJLEVBQUUsWUFBWSxHQUFHLElBQUksQ0FhdkM7SUFFYyxVQUFVLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUd6QztJQUVjLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBT3ZDO0lBRUQsT0FBTyxDQUFDLG1CQUFtQjtJQVEzQixPQUFPLENBQUMsbUJBQW1CO0NBWTVCIn0=
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
{"version":3,"file":"monitored_batch_span_processor.d.ts","sourceRoot":"","sources":["../src/monitored_batch_span_processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,EAAE,KAAK,OAAO,EAAkB,MAAM,oBAAoB,CAAC;AAElE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,KAAK,IAAI,EAAE,MAAM,+BAA+B,CAAC;AAapH,MAAM,MAAM,iCAAiC,GAAG,YAAY,GAAG;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,kBAAkB;IACjE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAK;IAE5B,YAAY,QAAQ,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,iCAAiC,EAO1F;IAEQ,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,GAAG,IAAI,CAEzD;IAEQ,KAAK,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI,CAavC;IAEc,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzC;IAEc,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAOvC;IAED,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;CAY5B"}
|
|
@@ -0,0 +1,75 @@
|
|
|
1
|
+
import { SpanStatusCode } from '@opentelemetry/api';
|
|
2
|
+
import { hrTimeToMilliseconds } from '@opentelemetry/core';
|
|
3
|
+
import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
|
|
4
|
+
/** Minimum interval between drop warnings to avoid log spam. */ const DROP_WARNING_INTERVAL_MS = 30_000;
|
|
5
|
+
const DEFAULT_MIN_TRACE_DURATION_MS = 10;
|
|
6
|
+
const DEFAULT_MAX_QUEUE_SIZE = 16384;
|
|
7
|
+
/** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
|
|
8
|
+
* at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */ const DEFAULT_MAX_EXPORT_BATCH_SIZE = 2048;
|
|
9
|
+
/**
|
|
10
|
+
* Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
|
|
11
|
+
* The standard BatchSpanProcessor silently discards spans when its internal queue reaches
|
|
12
|
+
* maxQueueSize, making telemetry data loss invisible to operators.
|
|
13
|
+
*/ export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
|
|
14
|
+
maxQueueSize;
|
|
15
|
+
minTraceDurationMs;
|
|
16
|
+
log;
|
|
17
|
+
approxQueueSize = 0;
|
|
18
|
+
droppedSinceLastWarning = 0;
|
|
19
|
+
totalDropped = 0;
|
|
20
|
+
lastWarningTime = 0;
|
|
21
|
+
constructor(exporter, log, config){
|
|
22
|
+
const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
|
|
23
|
+
const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
|
|
24
|
+
super(exporter, {
|
|
25
|
+
...config,
|
|
26
|
+
maxQueueSize,
|
|
27
|
+
maxExportBatchSize
|
|
28
|
+
});
|
|
29
|
+
this.maxQueueSize = maxQueueSize;
|
|
30
|
+
this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
|
|
31
|
+
this.log = log;
|
|
32
|
+
}
|
|
33
|
+
onStart(span, parentContext) {
|
|
34
|
+
super.onStart(span, parentContext);
|
|
35
|
+
}
|
|
36
|
+
onEnd(span) {
|
|
37
|
+
if (this.shouldDropShortSpan(span)) {
|
|
38
|
+
return;
|
|
39
|
+
}
|
|
40
|
+
if (this.approxQueueSize >= this.maxQueueSize) {
|
|
41
|
+
this.droppedSinceLastWarning++;
|
|
42
|
+
this.totalDropped++;
|
|
43
|
+
this.maybeLogDropWarning();
|
|
44
|
+
} else {
|
|
45
|
+
this.approxQueueSize++;
|
|
46
|
+
}
|
|
47
|
+
super.onEnd(span);
|
|
48
|
+
}
|
|
49
|
+
async forceFlush() {
|
|
50
|
+
await super.forceFlush();
|
|
51
|
+
this.approxQueueSize = 0;
|
|
52
|
+
}
|
|
53
|
+
async shutdown() {
|
|
54
|
+
if (this.totalDropped > 0) {
|
|
55
|
+
this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
|
|
56
|
+
totalDropped: this.totalDropped
|
|
57
|
+
});
|
|
58
|
+
}
|
|
59
|
+
await super.shutdown();
|
|
60
|
+
}
|
|
61
|
+
shouldDropShortSpan(span) {
|
|
62
|
+
return this.minTraceDurationMs > 0 && span.status.code !== SpanStatusCode.ERROR && hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs;
|
|
63
|
+
}
|
|
64
|
+
maybeLogDropWarning() {
|
|
65
|
+
const now = Date.now();
|
|
66
|
+
if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
|
|
67
|
+
this.log.warn(`BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` + `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`, {
|
|
68
|
+
droppedSinceLastWarning: this.droppedSinceLastWarning,
|
|
69
|
+
totalDropped: this.totalDropped
|
|
70
|
+
});
|
|
71
|
+
this.droppedSinceLastWarning = 0;
|
|
72
|
+
this.lastWarningTime = now;
|
|
73
|
+
}
|
|
74
|
+
}
|
|
75
|
+
}
|
package/dest/otel.d.ts
CHANGED
|
@@ -20,6 +20,10 @@ export declare class OpenTelemetryClient implements TelemetryClient {
|
|
|
20
20
|
nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
|
|
21
21
|
private meters;
|
|
22
22
|
private tracers;
|
|
23
|
+
/** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
|
|
24
|
+
* so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
|
|
25
|
+
* "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
|
|
26
|
+
private stopPromise;
|
|
23
27
|
protected constructor(resource: IResource, meterProvider: MeterProvider, traceProvider: TracerProvider, loggerProvider: LoggerProvider | undefined, publicMetricExporter: PublicOtelFilterMetricExporter | undefined, log: Logger);
|
|
24
28
|
setExportedPublicTelemetry(metrics: string[]): void;
|
|
25
29
|
setPublicTelemetryCollectFrom(roles: string[]): void;
|
|
@@ -29,10 +33,11 @@ export declare class OpenTelemetryClient implements TelemetryClient {
|
|
|
29
33
|
isEnabled(): boolean;
|
|
30
34
|
flush(): Promise<void>;
|
|
31
35
|
stop(): Promise<void>;
|
|
36
|
+
private doStop;
|
|
32
37
|
getTraceContext(): string | undefined;
|
|
33
38
|
extractPropagatedContext(traceContext: string): Context;
|
|
34
39
|
static createMeterProvider(resource: IResource, exporters: Array<PeriodicExportingMetricReaderOptions>): MeterProvider;
|
|
35
40
|
private static getCustomClientFactory;
|
|
36
41
|
static createAndStart(config: TelemetryClientConfig, log: Logger): OpenTelemetryClient;
|
|
37
42
|
}
|
|
38
|
-
//# sourceMappingURL=data:application/json;base64,
|
|
43
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFnQixLQUFLLE1BQU0sRUFBcUIsTUFBTSx1QkFBdUIsQ0FBQztBQUVyRixPQUFPLEVBQ0wsS0FBSyxPQUFPLEVBS1osS0FBSyxNQUFNLEVBQ1gsS0FBSyxjQUFjLEVBTXBCLE1BQU0sb0JBQW9CLENBQUM7QUFJNUIsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzlELE9BQU8sRUFHTCxhQUFhLEVBRWIsS0FBSyxvQ0FBb0MsRUFFMUMsTUFBTSw0QkFBNEIsQ0FBQztBQUlwQyxPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUl6RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNuRSxPQUFPLEVBQTRCLDhCQUE4QixFQUFFLE1BQU0sa0NBQWtDLENBQUM7QUFHNUcsT0FBTyxLQUFLLEVBR1YsS0FBSyxFQUdMLGVBQWUsRUFFaEIsTUFBTSxnQkFBZ0IsQ0FBQztBQXlDeEIsTUFBTSxNQUFNLDBCQUEwQixHQUFHLENBQUMsUUFBUSxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsTUFBTSxLQUFLLG1CQUFtQixDQUFDO0FBRW5HLHFCQUFhLG1CQUFvQixZQUFXLGVBQWU7SUFZdkQsT0FBTyxDQUFDLFFBQVE7SUFDaEIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGNBQWM7SUFDdEIsT0FBTyxDQUFDLG9CQUFvQjtJQUM1QixPQUFPLENBQUMsR0FBRztJQWhCYixXQUFXLEVBQUUsV0FBVyxHQUFHLFNBQVMsQ0FBQztJQUNyQyxvQkFBb0IsRUFBRSxvQkFBb0IsR0FBRyxTQUFTLENBQUM7SUFDdkQsT0FBTyxDQUFDLE1BQU0sQ0FBOEQ7SUFDNUUsT0FBTyxDQUFDLE9BQU8sQ0FBa0Q7SUFFakU7O3lIQUVxSDtJQUNySCxPQUFPLENBQUMsV0FBVyxDQUE0QjtJQUUvQyxTQUFTLGFBQ0MsUUFBUSxFQUFFLFNBQVMsRUFDbkIsYUFBYSxFQUFFLGFBQWEsRUFDNUIsYUFBYSxFQUFFLGNBQWMsRUFDN0IsY0FBYyxFQUFFLGNBQWMsR0FBRyxTQUFTLEVBQzFDLG9CQUFvQixFQUFFLDhCQUE4QixHQUFHLFNBQVMsRUFDaEUsR0FBRyxFQUFFLE1BQU0sRUFDakI7SUFFSiwwQkFBMEIsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUVsRDtJQUVELDZCQUE2QixDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBRW5EO0lBRUQsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEdBQUcsS0FBSyxDQVE1QjtJQUVELFNBQVMsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FPOUI7SUFFTSxLQUFLLFNBOEJYO0lBRU0sU0FBUyxZQUVmO0lBRVksS0FBSyxrQkFVakI7SUFFTSxJQUFJLGtCQUVWO1lBRWEsTUFBTTtJQWtCYixlQUFlLElBQUksTUFBTSxHQUFHLFNBQVMsQ0FJM0M7SUFFTSx3QkFBd0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FLN0Q7SUFFRCxPQUFjLG1CQUFtQixDQUMvQixRQUFRLEVBQUUsU0FBUyxFQUNuQixTQUFTLEVBQUUsS0FBSyxDQUFDLG9DQUFvQyxDQUFDLEdBQ3JELGFBQWEsQ0E4S2Y7SUFFRCxPQUFPLENBQUMsTUFBTSxDQUFDLHNCQUFzQjtJQXNFckMsT0FBYyxjQUFjLENBQUMsTUFBTSxFQUFFLHFCQUFxQixFQUFFLEdBQUcsRUFBRSxNQUFNLEdBQUcsbUJBQW1CLENBUTVGO0NBQ0YifQ==
|
package/dest/otel.d.ts.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,EACL,KAAK,OAAO,EAKZ,KAAK,MAAM,EACX,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAGL,aAAa,EAEb,KAAK,oCAAoC,EAE1C,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;
|
|
1
|
+
{"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,EACL,KAAK,OAAO,EAKZ,KAAK,MAAM,EACX,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAGL,aAAa,EAEb,KAAK,oCAAoC,EAE1C,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAIzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAA4B,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAG5G,OAAO,KAAK,EAGV,KAAK,EAGL,eAAe,EAEhB,MAAM,gBAAgB,CAAC;AAyCxB,MAAM,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAEnG,qBAAa,mBAAoB,YAAW,eAAe;IAYvD,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,GAAG;IAhBb,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACvD,OAAO,CAAC,MAAM,CAA8D;IAC5E,OAAO,CAAC,OAAO,CAAkD;IAEjE;;yHAEqH;IACrH,OAAO,CAAC,WAAW,CAA4B;IAE/C,SAAS,aACC,QAAQ,EAAE,SAAS,EACnB,aAAa,EAAE,aAAa,EAC5B,aAAa,EAAE,cAAc,EAC7B,cAAc,EAAE,cAAc,GAAG,SAAS,EAC1C,oBAAoB,EAAE,8BAA8B,GAAG,SAAS,EAChE,GAAG,EAAE,MAAM,EACjB;IAEJ,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAElD;IAED,6BAA6B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAEnD;IAED,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAQ5B;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO9B;IAEM,KAAK,SA8BX;IAEM,SAAS,YAEf;IAEY,KAAK,kBAUjB;IAEM,IAAI,kBAEV;YAEa,MAAM;IAkBb,eAAe,IAAI,MAAM,GAAG,SAAS,CAI3C;IAEM,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAK7D;IAED,OAAc,mBAAmB,CAC/B,QAAQ,EAAE,SAAS,EACnB,SAAS,EAAE,KAAK,CAAC,oCAAoC,CAAC,GACrD,aAAa,CA8Kf;IAED,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAsErC,OAAc,cAAc,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAQ5F;CACF"}
|
package/dest/otel.js
CHANGED
|
@@ -5,9 +5,10 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
|
|
|
5
5
|
import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
|
|
6
6
|
import { HostMetrics } from '@opentelemetry/host-metrics';
|
|
7
7
|
import { ExplicitBucketHistogramAggregation, InstrumentType, MeterProvider, PeriodicExportingMetricReader, View } from '@opentelemetry/sdk-metrics';
|
|
8
|
-
import {
|
|
8
|
+
import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
|
|
9
9
|
import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
|
|
10
10
|
import { toMetricOptions } from './metric-utils.js';
|
|
11
|
+
import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
|
|
11
12
|
import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
|
|
12
13
|
import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
|
|
13
14
|
import { registerOtelLoggerProvider } from './otel_logger_provider.js';
|
|
@@ -53,6 +54,9 @@ export class OpenTelemetryClient {
|
|
|
53
54
|
nodejsMetricsMonitor;
|
|
54
55
|
meters;
|
|
55
56
|
tracers;
|
|
57
|
+
/** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
|
|
58
|
+
* so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
|
|
59
|
+
* "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */ stopPromise;
|
|
56
60
|
constructor(resource, meterProvider, traceProvider, loggerProvider, publicMetricExporter, log){
|
|
57
61
|
this.resource = resource;
|
|
58
62
|
this.meterProvider = meterProvider;
|
|
@@ -113,13 +117,20 @@ export class OpenTelemetryClient {
|
|
|
113
117
|
return true;
|
|
114
118
|
}
|
|
115
119
|
async flush() {
|
|
120
|
+
// Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
|
|
121
|
+
if (this.stopPromise) {
|
|
122
|
+
return;
|
|
123
|
+
}
|
|
116
124
|
await Promise.all([
|
|
117
125
|
this.meterProvider.forceFlush(),
|
|
118
126
|
this.loggerProvider?.forceFlush(),
|
|
119
127
|
this.traceProvider instanceof NodeTracerProvider ? this.traceProvider.forceFlush() : Promise.resolve()
|
|
120
128
|
]);
|
|
121
129
|
}
|
|
122
|
-
|
|
130
|
+
stop() {
|
|
131
|
+
return this.stopPromise ??= this.doStop();
|
|
132
|
+
}
|
|
133
|
+
async doStop() {
|
|
123
134
|
this.nodejsMetricsMonitor?.stop();
|
|
124
135
|
const flushAndShutdown = async (provider)=>{
|
|
125
136
|
if (!provider) {
|
|
@@ -209,6 +220,32 @@ export class OpenTelemetryClient {
|
|
|
209
220
|
1200
|
|
210
221
|
], true)
|
|
211
222
|
}),
|
|
223
|
+
// Pending-to-mined delay routinely exceeds the 1-minute ceiling of the generic `ms`
|
|
224
|
+
// view below under load, so it would saturate at 60s. Give this one metric wider
|
|
225
|
+
// buckets (1s to 10min). This must precede the generic `ms` view: when multiple views
|
|
226
|
+
// match an instrument, the SDK keeps the first-registered compatible storage, so the
|
|
227
|
+
// first view in this list wins the bucket boundaries.
|
|
228
|
+
new View({
|
|
229
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
230
|
+
instrumentName: 'aztec.mempool.tx_mined_delay',
|
|
231
|
+
instrumentUnit: 'ms',
|
|
232
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
233
|
+
1_000,
|
|
234
|
+
2_500,
|
|
235
|
+
5_000,
|
|
236
|
+
7_500,
|
|
237
|
+
10_000,
|
|
238
|
+
15_000,
|
|
239
|
+
30_000,
|
|
240
|
+
45_000,
|
|
241
|
+
60_000,
|
|
242
|
+
90_000,
|
|
243
|
+
120_000,
|
|
244
|
+
180_000,
|
|
245
|
+
300_000,
|
|
246
|
+
600_000
|
|
247
|
+
], true)
|
|
248
|
+
}),
|
|
212
249
|
new View({
|
|
213
250
|
instrumentType: InstrumentType.HISTOGRAM,
|
|
214
251
|
instrumentUnit: 'ms',
|
|
@@ -369,6 +406,61 @@ export class OpenTelemetryClient {
|
|
|
369
406
|
15_000_000,
|
|
370
407
|
30_000_000
|
|
371
408
|
], true)
|
|
409
|
+
}),
|
|
410
|
+
// L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
|
|
411
|
+
new View({
|
|
412
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
413
|
+
instrumentUnit: 'gwei',
|
|
414
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
415
|
+
0.1,
|
|
416
|
+
0.5,
|
|
417
|
+
1,
|
|
418
|
+
2,
|
|
419
|
+
5,
|
|
420
|
+
10,
|
|
421
|
+
20,
|
|
422
|
+
50,
|
|
423
|
+
100,
|
|
424
|
+
200,
|
|
425
|
+
500,
|
|
426
|
+
1_000
|
|
427
|
+
], true)
|
|
428
|
+
}),
|
|
429
|
+
// L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
|
|
430
|
+
new View({
|
|
431
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
432
|
+
instrumentUnit: 'gas',
|
|
433
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
434
|
+
10_000,
|
|
435
|
+
50_000,
|
|
436
|
+
100_000,
|
|
437
|
+
250_000,
|
|
438
|
+
500_000,
|
|
439
|
+
1_000_000,
|
|
440
|
+
2_000_000,
|
|
441
|
+
5_000_000,
|
|
442
|
+
10_000_000,
|
|
443
|
+
15_000_000,
|
|
444
|
+
30_000_000
|
|
445
|
+
], true)
|
|
446
|
+
}),
|
|
447
|
+
// L1 tx total fee in ETH: typically 0.001 - 1 ETH
|
|
448
|
+
new View({
|
|
449
|
+
instrumentType: InstrumentType.HISTOGRAM,
|
|
450
|
+
instrumentUnit: 'eth',
|
|
451
|
+
aggregation: new ExplicitBucketHistogramAggregation([
|
|
452
|
+
0.0001,
|
|
453
|
+
0.0005,
|
|
454
|
+
0.001,
|
|
455
|
+
0.005,
|
|
456
|
+
0.01,
|
|
457
|
+
0.05,
|
|
458
|
+
0.1,
|
|
459
|
+
0.5,
|
|
460
|
+
1,
|
|
461
|
+
5,
|
|
462
|
+
10
|
|
463
|
+
], true)
|
|
372
464
|
})
|
|
373
465
|
]
|
|
374
466
|
});
|
|
@@ -378,9 +470,12 @@ export class OpenTelemetryClient {
|
|
|
378
470
|
const tracerProvider = new NodeTracerProvider({
|
|
379
471
|
resource,
|
|
380
472
|
spanProcessors: config.tracesCollectorUrl ? [
|
|
381
|
-
new
|
|
473
|
+
new MonitoredBatchSpanProcessor(new OTLPTraceExporter({
|
|
382
474
|
url: config.tracesCollectorUrl.href
|
|
383
|
-
})
|
|
475
|
+
}), log, {
|
|
476
|
+
maxQueueSize: config.otelBspMaxQueueSize,
|
|
477
|
+
minTraceDurationMs: config.otelMinTraceDurationMs
|
|
478
|
+
})
|
|
384
479
|
] : []
|
|
385
480
|
});
|
|
386
481
|
tracerProvider.register({
|
|
@@ -1,3 +1,5 @@
|
|
|
1
|
+
import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
|
|
1
2
|
import type Koa from 'koa';
|
|
2
3
|
export declare function getOtelJsonRpcPropagationMiddleware(scope?: string): (ctx: Koa.Context, next: () => Promise<void>) => Promise<void>;
|
|
3
|
-
|
|
4
|
+
export declare function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware;
|
|
5
|
+
//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9wcm9wYWdhdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWxfcHJvcGFnYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxtQ0FBbUMsQ0FBQztBQUcvRSxPQUFPLEtBQUssR0FBRyxNQUFNLEtBQUssQ0FBQztBQVUzQix3QkFBZ0IsbUNBQW1DLENBQ2pELEtBQUssU0FBa0IsR0FDdEIsQ0FBQyxHQUFHLEVBQUUsR0FBRyxDQUFDLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLEtBQUssT0FBTyxDQUFDLElBQUksQ0FBQyxDQW9DaEU7QUFFRCx3QkFBZ0IsbUNBQW1DLElBQUkscUJBQXFCLENBNEIzRSJ9
|
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"
|
|
1
|
+
{"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG/E,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAU3B,wBAAgB,mCAAmC,CACjD,KAAK,SAAkB,GACtB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAoChE;AAED,wBAAgB,mCAAmC,IAAI,qBAAqB,CA4B3E"}
|
package/dest/otel_propagation.js
CHANGED
|
@@ -6,7 +6,7 @@ export function getOtelJsonRpcPropagationMiddleware(scope = 'JsonRpcServer') {
|
|
|
6
6
|
const tracer = getTelemetryClient().getTracer(scope);
|
|
7
7
|
const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
|
|
8
8
|
const method = ctx.request.body?.method;
|
|
9
|
-
return tracer.startActiveSpan(`JsonRpcServer.${method ?? '
|
|
9
|
+
return tracer.startActiveSpan(`JsonRpcServer.${method ?? 'batch'}`, {
|
|
10
10
|
kind: SpanKind.SERVER
|
|
11
11
|
}, context, async (span)=>{
|
|
12
12
|
if (ctx.id) {
|
|
@@ -42,3 +42,51 @@ export function getOtelJsonRpcPropagationMiddleware(scope = 'JsonRpcServer') {
|
|
|
42
42
|
});
|
|
43
43
|
};
|
|
44
44
|
}
|
|
45
|
+
export function getOtelJsonRpcDiagnosticsMiddleware() {
|
|
46
|
+
return function otelJsonRpcDiagnostics(ctx, next) {
|
|
47
|
+
const [namespace, method] = splitNamespace(ctx.method);
|
|
48
|
+
const scope = namespace ?? 'UnknownHandler';
|
|
49
|
+
const tracer = getTelemetryClient().getTracer(scope);
|
|
50
|
+
return tracer.startActiveSpan(`${scope}.${method}`, {
|
|
51
|
+
kind: SpanKind.INTERNAL,
|
|
52
|
+
attributes: {
|
|
53
|
+
[ATTR_JSONRPC_METHOD]: ctx.method
|
|
54
|
+
}
|
|
55
|
+
}, async (span)=>{
|
|
56
|
+
if (ctx.id !== null) {
|
|
57
|
+
span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
|
|
58
|
+
}
|
|
59
|
+
try {
|
|
60
|
+
await next();
|
|
61
|
+
span.setStatus({
|
|
62
|
+
code: SpanStatusCode.OK
|
|
63
|
+
});
|
|
64
|
+
} catch (err) {
|
|
65
|
+
span.setStatus({
|
|
66
|
+
code: SpanStatusCode.ERROR,
|
|
67
|
+
message: err instanceof Error ? err.message : String(err)
|
|
68
|
+
});
|
|
69
|
+
if (typeof err === 'string' || err instanceof Error) {
|
|
70
|
+
span.recordException(err);
|
|
71
|
+
}
|
|
72
|
+
throw err;
|
|
73
|
+
} finally{
|
|
74
|
+
span.end();
|
|
75
|
+
}
|
|
76
|
+
});
|
|
77
|
+
};
|
|
78
|
+
}
|
|
79
|
+
function splitNamespace(fullMethod) {
|
|
80
|
+
const idx = fullMethod.indexOf('_');
|
|
81
|
+
if (idx > -1) {
|
|
82
|
+
return [
|
|
83
|
+
fullMethod.slice(0, idx),
|
|
84
|
+
fullMethod.slice(idx + 1)
|
|
85
|
+
];
|
|
86
|
+
} else {
|
|
87
|
+
return [
|
|
88
|
+
undefined,
|
|
89
|
+
fullMethod
|
|
90
|
+
];
|
|
91
|
+
}
|
|
92
|
+
}
|