@aztec/telemetry-client 5.0.0-private.20260319 → 5.0.0-rc.2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (46) hide show
  1. package/dest/attributes.d.ts +5 -2
  2. package/dest/attributes.d.ts.map +1 -1
  3. package/dest/attributes.js +2 -1
  4. package/dest/config.d.ts +3 -1
  5. package/dest/config.d.ts.map +1 -1
  6. package/dest/config.js +17 -9
  7. package/dest/lmdb_metrics.d.ts +2 -2
  8. package/dest/lmdb_metrics.d.ts.map +1 -1
  9. package/dest/metrics.d.ts +23 -4
  10. package/dest/metrics.d.ts.map +1 -1
  11. package/dest/metrics.js +120 -15
  12. package/dest/monitored_batch_span_processor.d.ts +29 -0
  13. package/dest/monitored_batch_span_processor.d.ts.map +1 -0
  14. package/dest/monitored_batch_span_processor.js +75 -0
  15. package/dest/otel.d.ts +6 -1
  16. package/dest/otel.d.ts.map +1 -1
  17. package/dest/otel.js +73 -4
  18. package/dest/otel_propagation.d.ts +3 -1
  19. package/dest/otel_propagation.d.ts.map +1 -1
  20. package/dest/otel_propagation.js +49 -1
  21. package/dest/start.d.ts +1 -1
  22. package/dest/start.d.ts.map +1 -1
  23. package/dest/start.js +1 -1
  24. package/dest/telemetry.d.ts +2 -2
  25. package/dest/telemetry.d.ts.map +1 -1
  26. package/dest/wrappers/fetch.d.ts +3 -3
  27. package/dest/wrappers/fetch.d.ts.map +1 -1
  28. package/dest/wrappers/fetch.js +3 -2
  29. package/dest/wrappers/index.d.ts +1 -2
  30. package/dest/wrappers/index.d.ts.map +1 -1
  31. package/dest/wrappers/index.js +0 -1
  32. package/package.json +3 -3
  33. package/src/attributes.ts +4 -1
  34. package/src/config.ts +24 -9
  35. package/src/metrics.ts +125 -15
  36. package/src/monitored_batch_span_processor.ts +100 -0
  37. package/src/otel.ts +52 -3
  38. package/src/otel_propagation.ts +42 -1
  39. package/src/start.ts +6 -1
  40. package/src/telemetry.ts +0 -1
  41. package/src/wrappers/fetch.ts +9 -3
  42. package/src/wrappers/index.ts +0 -1
  43. package/dest/wrappers/l2_block_stream.d.ts +0 -16
  44. package/dest/wrappers/l2_block_stream.d.ts.map +0 -1
  45. package/dest/wrappers/l2_block_stream.js +0 -400
  46. package/src/wrappers/l2_block_stream.ts +0 -41
package/dest/metrics.js CHANGED
@@ -267,6 +267,12 @@ export const ARCHIVER_SYNC_PER_BLOCK = {
267
267
  unit: 'ms',
268
268
  valueType: ValueType.INT
269
269
  };
270
+ export const ARCHIVER_SYNC_PER_CHECKPOINT = {
271
+ name: 'aztec.archiver.checkpoint.sync_per_item_duration',
272
+ description: 'Duration to sync a checkpoint',
273
+ unit: 'ms',
274
+ valueType: ValueType.INT
275
+ };
270
276
  export const ARCHIVER_SYNC_BLOCK_COUNT = {
271
277
  name: 'aztec.archiver.block.sync_count',
272
278
  description: 'Number of blocks synced from L1',
@@ -310,6 +316,11 @@ export const ARCHIVER_CHECKPOINT_L1_INCLUSION_DELAY = {
310
316
  unit: 's',
311
317
  valueType: ValueType.INT
312
318
  };
319
+ export const ARCHIVER_CHECKPOINT_PROMOTED_COUNT = {
320
+ name: 'aztec.archiver.checkpoint_promoted_count',
321
+ description: 'Number of checkpoints promoted from proposed (blob fetch skipped)',
322
+ valueType: ValueType.INT
323
+ };
313
324
  export const NODE_RECEIVE_TX_DURATION = {
314
325
  name: 'aztec.node.receive_tx.duration',
315
326
  description: 'The duration of the receiveTx method',
@@ -338,6 +349,12 @@ export const SEQUENCER_STATE_TRANSITION_BUFFER_DURATION = {
338
349
  unit: 'ms',
339
350
  valueType: ValueType.INT
340
351
  };
352
+ export const SEQUENCER_STATE_DURATION = {
353
+ name: 'aztec.sequencer.state_duration',
354
+ description: 'Wall-clock time spent in each sequencer state, labelled by the state being left',
355
+ unit: 'ms',
356
+ valueType: ValueType.INT
357
+ };
341
358
  export const SEQUENCER_BLOCK_BUILD_DURATION = {
342
359
  name: 'aztec.sequencer.block.build_duration',
343
360
  description: 'Duration to build a block',
@@ -355,6 +372,12 @@ export const SEQUENCER_BLOCK_COUNT = {
355
372
  description: 'Number of blocks built by this sequencer',
356
373
  valueType: ValueType.INT
357
374
  };
375
+ export const SEQUENCER_BLOCK_INTER_BLOCK_TIME = {
376
+ name: 'aztec.sequencer.block.inter_block_time',
377
+ description: 'Wall-clock time elapsed between consecutive blocks being built by this sequencer',
378
+ unit: 'ms',
379
+ valueType: ValueType.INT
380
+ };
358
381
  export const SEQUENCER_CURRENT_SLOT_REWARDS = {
359
382
  name: 'aztec.sequencer.current_slot_rewards',
360
383
  description: 'The rewards earned per filled slot',
@@ -424,6 +447,24 @@ export const SEQUENCER_CHECKPOINT_BUILD_DURATION = {
424
447
  unit: 'ms',
425
448
  valueType: ValueType.INT
426
449
  };
450
+ export const SEQUENCER_CHECKPOINT_START_TO_FIRST_BLOCK_DURATION = {
451
+ name: 'aztec.sequencer.checkpoint.start_to_first_block_duration',
452
+ description: 'Time from starting checkpoint work to the first block finishing build',
453
+ unit: 'ms',
454
+ valueType: ValueType.INT
455
+ };
456
+ export const SEQUENCER_CHECKPOINT_LAST_BLOCK_TO_BROADCAST_DURATION = {
457
+ name: 'aztec.sequencer.checkpoint.last_block_to_broadcast_duration',
458
+ description: 'Time from the final block finishing build to the checkpoint proposal being broadcast',
459
+ unit: 'ms',
460
+ valueType: ValueType.INT
461
+ };
462
+ export const SEQUENCER_PIPELINED_CHECKPOINT_BUILD_START_OFFSET_FROM_SLOT_BOUNDARY = {
463
+ name: 'aztec.sequencer.pipelined_checkpoint.build_start_offset_from_slot_boundary',
464
+ description: 'Absolute offset from the wall-clock slot boundary when a pipelined checkpoint build starts. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
465
+ unit: 'ms',
466
+ valueType: ValueType.INT
467
+ };
427
468
  export const SEQUENCER_CHECKPOINT_BLOCK_COUNT = {
428
469
  name: 'aztec.sequencer.checkpoint.block_count',
429
470
  description: 'Number of blocks built in a checkpoint',
@@ -446,6 +487,11 @@ export const SEQUENCER_SLASHING_ATTEMPTS_COUNT = {
446
487
  description: 'The number of slashing action attempts',
447
488
  valueType: ValueType.INT
448
489
  };
490
+ export const SLASHER_ROUND_EXECUTED_COUNT = {
491
+ name: 'aztec.slasher.round.executed_count',
492
+ description: 'The number of slashing rounds executed',
493
+ valueType: ValueType.INT
494
+ };
449
495
  export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT = {
450
496
  name: 'aztec.sequencer.checkpoint.success_count',
451
497
  description: 'The number of times checkpoint publishing succeeded',
@@ -461,6 +507,11 @@ export const SEQUENCER_PIPELINE_DISCARDS_COUNT = {
461
507
  description: 'The number of times a pipeline was discarded',
462
508
  valueType: ValueType.INT
463
509
  };
510
+ export const SEQUENCER_PIPELINE_PARENT_CHECKPOINT_MISMATCH_COUNT = {
511
+ name: 'aztec.sequencer.pipeline.parent_checkpoint_mismatch_count',
512
+ description: 'The number of times a pipelined checkpoint was discarded because the parent did not match expectations',
513
+ valueType: ValueType.INT
514
+ };
464
515
  // Fisherman fee analysis metrics
465
516
  export const FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED = {
466
517
  name: 'aztec.fisherman.fee_analysis.would_be_included',
@@ -625,6 +676,24 @@ export const L1_PUBLISHER_TX_TOTAL_FEE = {
625
676
  unit: 'eth',
626
677
  valueType: ValueType.DOUBLE
627
678
  };
679
+ export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS = {
680
+ name: 'aztec.prover_node.estimated_submission.gas',
681
+ description: 'Estimated gas for a proof submission tx (proof publishing disabled, not actually sent)',
682
+ unit: 'gas',
683
+ valueType: ValueType.INT
684
+ };
685
+ export const PROVER_NODE_ESTIMATED_SUBMISSION_GAS_PRICE = {
686
+ name: 'aztec.prover_node.estimated_submission.gas_price',
687
+ description: 'Estimated effective gas price for a proof submission tx (proof publishing disabled, not actually sent)',
688
+ unit: 'gwei',
689
+ valueType: ValueType.DOUBLE
690
+ };
691
+ export const PROVER_NODE_ESTIMATED_SUBMISSION_TOTAL_FEE = {
692
+ name: 'aztec.prover_node.estimated_submission.total_fee',
693
+ description: 'Estimated total L1 fee for a proof submission tx (proof publishing disabled, not actually sent)',
694
+ unit: 'eth',
695
+ valueType: ValueType.DOUBLE
696
+ };
628
697
  export const L1_BLOCK_HEIGHT = {
629
698
  name: 'aztec.l1.block_height',
630
699
  description: 'The current L1 block height',
@@ -845,6 +914,11 @@ export const P2P_GOSSIP_AGG_MESSAGE_VALIDATION_DURATION_AVG = {
845
914
  unit: 'ms',
846
915
  valueType: ValueType.INT
847
916
  };
917
+ export const P2P_GOSSIP_SLOW_VALIDATION_COUNT = {
918
+ name: 'aztec.p2p.gossip.slow_validation_count',
919
+ description: 'Number of gossip validations that exceeded 75% of the mcache eviction window',
920
+ valueType: ValueType.INT
921
+ };
848
922
  export const PUBLIC_PROCESSOR_TX_DURATION = {
849
923
  name: 'aztec.public_processor.tx_duration',
850
924
  description: 'Duration to process a public transaction',
@@ -908,6 +982,17 @@ export const PUBLIC_PROCESSOR_TREE_INSERTION = {
908
982
  unit: 'ms',
909
983
  valueType: ValueType.INT
910
984
  };
985
+ export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_COUNT = {
986
+ name: 'aztec.public_processor.silently_skipped_count',
987
+ description: 'Public txs fully processed then skipped (e.g. blob-field limit); not counted as processed or failed',
988
+ valueType: ValueType.INT
989
+ };
990
+ export const PUBLIC_PROCESSOR_SILENTLY_SKIPPED_DURATION = {
991
+ name: 'aztec.public_processor.silently_skipped_duration',
992
+ description: 'Wall-clock time spent processing txs that were then silently skipped',
993
+ unit: 'ms',
994
+ valueType: ValueType.INT
995
+ };
911
996
  export const PUBLIC_EXECUTOR_PREFIX = 'aztec.public_executor.';
912
997
  export const PUBLIC_EXECUTOR_SIMULATION_COUNT = {
913
998
  name: 'aztec.public_executor.simulation_count',
@@ -1047,12 +1132,6 @@ export const PROVING_AGENT_IDLE = {
1047
1132
  unit: 's',
1048
1133
  valueType: ValueType.DOUBLE
1049
1134
  };
1050
- export const PROVER_NODE_EXECUTION_DURATION = {
1051
- name: 'aztec.prover_node.execution.duration',
1052
- description: 'Duration of execution of an epoch by the prover',
1053
- unit: 'ms',
1054
- valueType: ValueType.INT
1055
- };
1056
1135
  export const PROVER_NODE_JOB_DURATION = {
1057
1136
  name: 'aztec.prover_node.job_duration',
1058
1137
  description: 'Duration of proving job',
@@ -1080,12 +1159,6 @@ export const PROVER_NODE_BLOB_PROCESSING_LAST_DURATION = {
1080
1159
  unit: 'ms',
1081
1160
  valueType: ValueType.INT
1082
1161
  };
1083
- export const PROVER_NODE_CHONK_VERIFIER_LAST_DURATION = {
1084
- name: 'aztec.prover_node.chonk_verifier.last_duration',
1085
- description: 'Duration of chonk verifier enqueuing in epoch proving job',
1086
- unit: 'ms',
1087
- valueType: ValueType.INT
1088
- };
1089
1162
  export const PROVER_NODE_BLOCK_PROCESSING_DURATION = {
1090
1163
  name: 'aztec.prover_node.block_processing.duration',
1091
1164
  description: 'Duration of processing a single block in epoch proving job',
@@ -1098,12 +1171,32 @@ export const PROVER_NODE_CHECKPOINT_PROCESSING_DURATION = {
1098
1171
  unit: 'ms',
1099
1172
  valueType: ValueType.INT
1100
1173
  };
1101
- export const PROVER_NODE_ALL_CHECKPOINTS_PROCESSING_LAST_DURATION = {
1102
- name: 'aztec.prover_node.all_checkpoints_processing.last_duration',
1103
- description: 'Duration of processing all checkpoints in epoch proving job',
1174
+ export const PROVER_NODE_CHECKPOINT_BLOCKS = {
1175
+ name: 'aztec.prover_node.checkpoint_blocks',
1176
+ description: 'Number of blocks in a proven checkpoint',
1177
+ valueType: ValueType.INT
1178
+ };
1179
+ export const PROVER_NODE_CHECKPOINT_TRANSACTIONS = {
1180
+ name: 'aztec.prover_node.checkpoint_transactions',
1181
+ description: 'Number of transactions in a proven checkpoint',
1182
+ valueType: ValueType.INT
1183
+ };
1184
+ export const PROVER_NODE_CHECKPOINT_PROVING_DURATION = {
1185
+ name: 'aztec.prover_node.checkpoint_proving.duration',
1186
+ description: 'Duration from the start of checkpoint processing to its block proofs being ready (excludes tx gathering)',
1104
1187
  unit: 'ms',
1105
1188
  valueType: ValueType.INT
1106
1189
  };
1190
+ export const PROVER_NODE_ACTIVE_CHECKPOINTS = {
1191
+ name: 'aztec.prover_node.active_checkpoints',
1192
+ description: 'Current number of canonical CheckpointProvers in the store (i.e. checkpoints currently being proven)',
1193
+ valueType: ValueType.INT
1194
+ };
1195
+ export const PROVER_NODE_ACTIVE_EPOCH_SESSIONS = {
1196
+ name: 'aztec.prover_node.active_epoch_sessions',
1197
+ description: 'Current number of live EpochSessions, broken down by kind (full|partial)',
1198
+ valueType: ValueType.INT
1199
+ };
1107
1200
  export const PROVER_NODE_REWARDS_TOTAL = {
1108
1201
  name: 'aztec.prover_node.rewards_total',
1109
1202
  description: 'The rewards earned (total)',
@@ -1214,6 +1307,18 @@ export const VALIDATOR_RE_EXECUTION_TX_COUNT = {
1214
1307
  unit: 'tx',
1215
1308
  valueType: ValueType.INT
1216
1309
  };
1310
+ export const VALIDATOR_CHECKPOINT_PROPOSAL_TO_PIPELINED_STATE_DURATION = {
1311
+ name: 'aztec.validator.checkpoint_proposal_to_pipelined_state_duration',
1312
+ description: 'Time from receiving a checkpoint proposal to setting proposed checkpoint state for pipelining',
1313
+ unit: 'ms',
1314
+ valueType: ValueType.INT
1315
+ };
1316
+ export const VALIDATOR_CHECKPOINT_PROPOSAL_RECEIVE_OFFSET_FROM_NEXT_SLOT_BOUNDARY = {
1317
+ name: 'aztec.validator.checkpoint.proposal_receive_offset_from_next_slot_boundary',
1318
+ description: 'Absolute offset from the next slot boundary when a foreign checkpoint proposal is received. Use aztec.slot_boundary_side to distinguish before vs after the boundary.',
1319
+ unit: 'ms',
1320
+ valueType: ValueType.INT
1321
+ };
1217
1322
  export const VALIDATOR_FAILED_REEXECUTION_COUNT = {
1218
1323
  name: 'aztec.validator.failed_reexecution_count',
1219
1324
  description: 'The number of failed re-executions',
@@ -0,0 +1,29 @@
1
+ import type { Logger } from '@aztec/foundation/log';
2
+ import { type Context } from '@opentelemetry/api';
3
+ import type { SpanExporter } from '@opentelemetry/sdk-trace-base';
4
+ import { BatchSpanProcessor, type BufferConfig, type ReadableSpan, type Span } from '@opentelemetry/sdk-trace-node';
5
+ export type MonitoredBatchSpanProcessorConfig = BufferConfig & {
6
+ minTraceDurationMs?: number;
7
+ };
8
+ /**
9
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
10
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
11
+ * maxQueueSize, making telemetry data loss invisible to operators.
12
+ */
13
+ export declare class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
14
+ private readonly maxQueueSize;
15
+ private readonly minTraceDurationMs;
16
+ private readonly log;
17
+ private approxQueueSize;
18
+ private droppedSinceLastWarning;
19
+ private totalDropped;
20
+ private lastWarningTime;
21
+ constructor(exporter: SpanExporter, log: Logger, config?: MonitoredBatchSpanProcessorConfig);
22
+ onStart(span: Span, parentContext: Context): void;
23
+ onEnd(span: ReadableSpan): void;
24
+ forceFlush(): Promise<void>;
25
+ shutdown(): Promise<void>;
26
+ private shouldDropShortSpan;
27
+ private maybeLogDropWarning;
28
+ }
29
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvbW9uaXRvcmVkX2JhdGNoX3NwYW5fcHJvY2Vzc29yLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sS0FBSyxFQUFFLE1BQU0sRUFBRSxNQUFNLHVCQUF1QixDQUFDO0FBRXBELE9BQU8sRUFBRSxLQUFLLE9BQU8sRUFBa0IsTUFBTSxvQkFBb0IsQ0FBQztBQUVsRSxPQUFPLEtBQUssRUFBRSxZQUFZLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUNsRSxPQUFPLEVBQUUsa0JBQWtCLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxZQUFZLEVBQUUsS0FBSyxJQUFJLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQWFwSCxNQUFNLE1BQU0saUNBQWlDLEdBQUcsWUFBWSxHQUFHO0lBQzdELGtCQUFrQixDQUFDLEVBQUUsTUFBTSxDQUFDO0NBQzdCLENBQUM7QUFFRjs7OztHQUlHO0FBQ0gscUJBQWEsMkJBQTRCLFNBQVEsa0JBQWtCO0lBQ2pFLE9BQU8sQ0FBQyxRQUFRLENBQUMsWUFBWSxDQUFTO0lBQ3RDLE9BQU8sQ0FBQyxRQUFRLENBQUMsa0JBQWtCLENBQVM7SUFDNUMsT0FBTyxDQUFDLFFBQVEsQ0FBQyxHQUFHLENBQVM7SUFFN0IsT0FBTyxDQUFDLGVBQWUsQ0FBSztJQUM1QixPQUFPLENBQUMsdUJBQXVCLENBQUs7SUFDcEMsT0FBTyxDQUFDLFlBQVksQ0FBSztJQUN6QixPQUFPLENBQUMsZUFBZSxDQUFLO0lBRTVCLFlBQVksUUFBUSxFQUFFLFlBQVksRUFBRSxHQUFHLEVBQUUsTUFBTSxFQUFFLE1BQU0sQ0FBQyxFQUFFLGlDQUFpQyxFQU8xRjtJQUVRLE9BQU8sQ0FBQyxJQUFJLEVBQUUsSUFBSSxFQUFFLGFBQWEsRUFBRSxPQUFPLEdBQUcsSUFBSSxDQUV6RDtJQUVRLEtBQUssQ0FBQyxJQUFJLEVBQUUsWUFBWSxHQUFHLElBQUksQ0FhdkM7SUFFYyxVQUFVLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUd6QztJQUVjLFFBQVEsSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBT3ZDO0lBRUQsT0FBTyxDQUFDLG1CQUFtQjtJQVEzQixPQUFPLENBQUMsbUJBQW1CO0NBWTVCIn0=
@@ -0,0 +1 @@
1
+ {"version":3,"file":"monitored_batch_span_processor.d.ts","sourceRoot":"","sources":["../src/monitored_batch_span_processor.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,MAAM,EAAE,MAAM,uBAAuB,CAAC;AAEpD,OAAO,EAAE,KAAK,OAAO,EAAkB,MAAM,oBAAoB,CAAC;AAElE,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,+BAA+B,CAAC;AAClE,OAAO,EAAE,kBAAkB,EAAE,KAAK,YAAY,EAAE,KAAK,YAAY,EAAE,KAAK,IAAI,EAAE,MAAM,+BAA+B,CAAC;AAapH,MAAM,MAAM,iCAAiC,GAAG,YAAY,GAAG;IAC7D,kBAAkB,CAAC,EAAE,MAAM,CAAC;CAC7B,CAAC;AAEF;;;;GAIG;AACH,qBAAa,2BAA4B,SAAQ,kBAAkB;IACjE,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAS;IACtC,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAS;IAC5C,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAE7B,OAAO,CAAC,eAAe,CAAK;IAC5B,OAAO,CAAC,uBAAuB,CAAK;IACpC,OAAO,CAAC,YAAY,CAAK;IACzB,OAAO,CAAC,eAAe,CAAK;IAE5B,YAAY,QAAQ,EAAE,YAAY,EAAE,GAAG,EAAE,MAAM,EAAE,MAAM,CAAC,EAAE,iCAAiC,EAO1F;IAEQ,OAAO,CAAC,IAAI,EAAE,IAAI,EAAE,aAAa,EAAE,OAAO,GAAG,IAAI,CAEzD;IAEQ,KAAK,CAAC,IAAI,EAAE,YAAY,GAAG,IAAI,CAavC;IAEc,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC,CAGzC;IAEc,QAAQ,IAAI,OAAO,CAAC,IAAI,CAAC,CAOvC;IAED,OAAO,CAAC,mBAAmB;IAQ3B,OAAO,CAAC,mBAAmB;CAY5B"}
@@ -0,0 +1,75 @@
1
+ import { SpanStatusCode } from '@opentelemetry/api';
2
+ import { hrTimeToMilliseconds } from '@opentelemetry/core';
3
+ import { BatchSpanProcessor } from '@opentelemetry/sdk-trace-node';
4
+ /** Minimum interval between drop warnings to avoid log spam. */ const DROP_WARNING_INTERVAL_MS = 30_000;
5
+ const DEFAULT_MIN_TRACE_DURATION_MS = 10;
6
+ const DEFAULT_MAX_QUEUE_SIZE = 16384;
7
+ /** Cap on the per-export batch size, so a large queue can actually be drained instead of dribbling out
8
+ * at the SDK default of 512 spans per scheduled export. Kept <= maxQueueSize per the BatchSpanProcessor contract. */ const DEFAULT_MAX_EXPORT_BATCH_SIZE = 2048;
9
+ /**
10
+ * Wraps BatchSpanProcessor to emit warnings when spans are dropped due to a full queue.
11
+ * The standard BatchSpanProcessor silently discards spans when its internal queue reaches
12
+ * maxQueueSize, making telemetry data loss invisible to operators.
13
+ */ export class MonitoredBatchSpanProcessor extends BatchSpanProcessor {
14
+ maxQueueSize;
15
+ minTraceDurationMs;
16
+ log;
17
+ approxQueueSize = 0;
18
+ droppedSinceLastWarning = 0;
19
+ totalDropped = 0;
20
+ lastWarningTime = 0;
21
+ constructor(exporter, log, config){
22
+ const maxQueueSize = config?.maxQueueSize ?? DEFAULT_MAX_QUEUE_SIZE;
23
+ const maxExportBatchSize = Math.min(config?.maxExportBatchSize ?? DEFAULT_MAX_EXPORT_BATCH_SIZE, maxQueueSize);
24
+ super(exporter, {
25
+ ...config,
26
+ maxQueueSize,
27
+ maxExportBatchSize
28
+ });
29
+ this.maxQueueSize = maxQueueSize;
30
+ this.minTraceDurationMs = Math.max(0, config?.minTraceDurationMs ?? DEFAULT_MIN_TRACE_DURATION_MS);
31
+ this.log = log;
32
+ }
33
+ onStart(span, parentContext) {
34
+ super.onStart(span, parentContext);
35
+ }
36
+ onEnd(span) {
37
+ if (this.shouldDropShortSpan(span)) {
38
+ return;
39
+ }
40
+ if (this.approxQueueSize >= this.maxQueueSize) {
41
+ this.droppedSinceLastWarning++;
42
+ this.totalDropped++;
43
+ this.maybeLogDropWarning();
44
+ } else {
45
+ this.approxQueueSize++;
46
+ }
47
+ super.onEnd(span);
48
+ }
49
+ async forceFlush() {
50
+ await super.forceFlush();
51
+ this.approxQueueSize = 0;
52
+ }
53
+ async shutdown() {
54
+ if (this.totalDropped > 0) {
55
+ this.log.warn(`BatchSpanProcessor shutting down with ${this.totalDropped} total spans dropped`, {
56
+ totalDropped: this.totalDropped
57
+ });
58
+ }
59
+ await super.shutdown();
60
+ }
61
+ shouldDropShortSpan(span) {
62
+ return this.minTraceDurationMs > 0 && span.status.code !== SpanStatusCode.ERROR && hrTimeToMilliseconds(span.duration) < this.minTraceDurationMs;
63
+ }
64
+ maybeLogDropWarning() {
65
+ const now = Date.now();
66
+ if (now - this.lastWarningTime >= DROP_WARNING_INTERVAL_MS) {
67
+ this.log.warn(`BatchSpanProcessor dropping spans: queue full (maxQueueSize=${this.maxQueueSize}). ` + `${this.droppedSinceLastWarning} dropped since last warning, ${this.totalDropped} total.`, {
68
+ droppedSinceLastWarning: this.droppedSinceLastWarning,
69
+ totalDropped: this.totalDropped
70
+ });
71
+ this.droppedSinceLastWarning = 0;
72
+ this.lastWarningTime = now;
73
+ }
74
+ }
75
+ }
package/dest/otel.d.ts CHANGED
@@ -20,6 +20,10 @@ export declare class OpenTelemetryClient implements TelemetryClient {
20
20
  nodejsMetricsMonitor: NodejsMetricsMonitor | undefined;
21
21
  private meters;
22
22
  private tracers;
23
+ /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
24
+ * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
25
+ * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */
26
+ private stopPromise;
23
27
  protected constructor(resource: IResource, meterProvider: MeterProvider, traceProvider: TracerProvider, loggerProvider: LoggerProvider | undefined, publicMetricExporter: PublicOtelFilterMetricExporter | undefined, log: Logger);
24
28
  setExportedPublicTelemetry(metrics: string[]): void;
25
29
  setPublicTelemetryCollectFrom(roles: string[]): void;
@@ -29,10 +33,11 @@ export declare class OpenTelemetryClient implements TelemetryClient {
29
33
  isEnabled(): boolean;
30
34
  flush(): Promise<void>;
31
35
  stop(): Promise<void>;
36
+ private doStop;
32
37
  getTraceContext(): string | undefined;
33
38
  extractPropagatedContext(traceContext: string): Context;
34
39
  static createMeterProvider(resource: IResource, exporters: Array<PeriodicExportingMetricReaderOptions>): MeterProvider;
35
40
  private static getCustomClientFactory;
36
41
  static createAndStart(config: TelemetryClientConfig, log: Logger): OpenTelemetryClient;
37
42
  }
38
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFnQixLQUFLLE1BQU0sRUFBcUIsTUFBTSx1QkFBdUIsQ0FBQztBQUVyRixPQUFPLEVBQ0wsS0FBSyxPQUFPLEVBS1osS0FBSyxNQUFNLEVBQ1gsS0FBSyxjQUFjLEVBTXBCLE1BQU0sb0JBQW9CLENBQUM7QUFJNUIsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzlELE9BQU8sRUFHTCxhQUFhLEVBRWIsS0FBSyxvQ0FBb0MsRUFFMUMsTUFBTSw0QkFBNEIsQ0FBQztBQUlwQyxPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUd6RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNuRSxPQUFPLEVBQTRCLDhCQUE4QixFQUFFLE1BQU0sa0NBQWtDLENBQUM7QUFHNUcsT0FBTyxLQUFLLEVBR1YsS0FBSyxFQUdMLGVBQWUsRUFFaEIsTUFBTSxnQkFBZ0IsQ0FBQztBQXlDeEIsTUFBTSxNQUFNLDBCQUEwQixHQUFHLENBQUMsUUFBUSxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsTUFBTSxLQUFLLG1CQUFtQixDQUFDO0FBRW5HLHFCQUFhLG1CQUFvQixZQUFXLGVBQWU7SUFPdkQsT0FBTyxDQUFDLFFBQVE7SUFDaEIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGNBQWM7SUFDdEIsT0FBTyxDQUFDLG9CQUFvQjtJQUM1QixPQUFPLENBQUMsR0FBRztJQVhiLFdBQVcsRUFBRSxXQUFXLEdBQUcsU0FBUyxDQUFDO0lBQ3JDLG9CQUFvQixFQUFFLG9CQUFvQixHQUFHLFNBQVMsQ0FBQztJQUN2RCxPQUFPLENBQUMsTUFBTSxDQUE4RDtJQUM1RSxPQUFPLENBQUMsT0FBTyxDQUFrRDtJQUVqRSxTQUFTLGFBQ0MsUUFBUSxFQUFFLFNBQVMsRUFDbkIsYUFBYSxFQUFFLGFBQWEsRUFDNUIsYUFBYSxFQUFFLGNBQWMsRUFDN0IsY0FBYyxFQUFFLGNBQWMsR0FBRyxTQUFTLEVBQzFDLG9CQUFvQixFQUFFLDhCQUE4QixHQUFHLFNBQVMsRUFDaEUsR0FBRyxFQUFFLE1BQU0sRUFDakI7SUFFSiwwQkFBMEIsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUVsRDtJQUVELDZCQUE2QixDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBRW5EO0lBRUQsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEdBQUcsS0FBSyxDQVE1QjtJQUVELFNBQVMsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FPOUI7SUFFTSxLQUFLLFNBOEJYO0lBRU0sU0FBUyxZQUVmO0lBRVksS0FBSyxrQkFNakI7SUFFWSxJQUFJLGtCQWdCaEI7SUFFTSxlQUFlLElBQUksTUFBTSxHQUFHLFNBQVMsQ0FJM0M7SUFFTSx3QkFBd0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FLN0Q7SUFFRCxPQUFjLG1CQUFtQixDQUMvQixRQUFRLEVBQUUsU0FBUyxFQUNuQixTQUFTLEVBQUUsS0FBSyxDQUFDLG9DQUFvQyxDQUFDLEdBQ3JELGFBQWEsQ0ErSGY7SUFFRCxPQUFPLENBQUMsTUFBTSxDQUFDLHNCQUFzQjtJQWlFckMsT0FBYyxjQUFjLENBQUMsTUFBTSxFQUFFLHFCQUFxQixFQUFFLEdBQUcsRUFBRSxNQUFNLEdBQUcsbUJBQW1CLENBUTVGO0NBQ0YifQ==
43
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbC5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWwudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxFQUFnQixLQUFLLE1BQU0sRUFBcUIsTUFBTSx1QkFBdUIsQ0FBQztBQUVyRixPQUFPLEVBQ0wsS0FBSyxPQUFPLEVBS1osS0FBSyxNQUFNLEVBQ1gsS0FBSyxjQUFjLEVBTXBCLE1BQU0sb0JBQW9CLENBQUM7QUFJNUIsT0FBTyxFQUFFLFdBQVcsRUFBRSxNQUFNLDZCQUE2QixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLFNBQVMsRUFBRSxNQUFNLDBCQUEwQixDQUFDO0FBQzFELE9BQU8sS0FBSyxFQUFFLGNBQWMsRUFBRSxNQUFNLHlCQUF5QixDQUFDO0FBQzlELE9BQU8sRUFHTCxhQUFhLEVBRWIsS0FBSyxvQ0FBb0MsRUFFMUMsTUFBTSw0QkFBNEIsQ0FBQztBQUlwQyxPQUFPLEtBQUssRUFBRSxxQkFBcUIsRUFBRSxNQUFNLGFBQWEsQ0FBQztBQUl6RCxPQUFPLEVBQUUsb0JBQW9CLEVBQUUsTUFBTSw2QkFBNkIsQ0FBQztBQUNuRSxPQUFPLEVBQTRCLDhCQUE4QixFQUFFLE1BQU0sa0NBQWtDLENBQUM7QUFHNUcsT0FBTyxLQUFLLEVBR1YsS0FBSyxFQUdMLGVBQWUsRUFFaEIsTUFBTSxnQkFBZ0IsQ0FBQztBQXlDeEIsTUFBTSxNQUFNLDBCQUEwQixHQUFHLENBQUMsUUFBUSxFQUFFLFNBQVMsRUFBRSxHQUFHLEVBQUUsTUFBTSxLQUFLLG1CQUFtQixDQUFDO0FBRW5HLHFCQUFhLG1CQUFvQixZQUFXLGVBQWU7SUFZdkQsT0FBTyxDQUFDLFFBQVE7SUFDaEIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGFBQWE7SUFDckIsT0FBTyxDQUFDLGNBQWM7SUFDdEIsT0FBTyxDQUFDLG9CQUFvQjtJQUM1QixPQUFPLENBQUMsR0FBRztJQWhCYixXQUFXLEVBQUUsV0FBVyxHQUFHLFNBQVMsQ0FBQztJQUNyQyxvQkFBb0IsRUFBRSxvQkFBb0IsR0FBRyxTQUFTLENBQUM7SUFDdkQsT0FBTyxDQUFDLE1BQU0sQ0FBOEQ7SUFDNUUsT0FBTyxDQUFDLE9BQU8sQ0FBa0Q7SUFFakU7O3lIQUVxSDtJQUNySCxPQUFPLENBQUMsV0FBVyxDQUE0QjtJQUUvQyxTQUFTLGFBQ0MsUUFBUSxFQUFFLFNBQVMsRUFDbkIsYUFBYSxFQUFFLGFBQWEsRUFDNUIsYUFBYSxFQUFFLGNBQWMsRUFDN0IsY0FBYyxFQUFFLGNBQWMsR0FBRyxTQUFTLEVBQzFDLG9CQUFvQixFQUFFLDhCQUE4QixHQUFHLFNBQVMsRUFDaEUsR0FBRyxFQUFFLE1BQU0sRUFDakI7SUFFSiwwQkFBMEIsQ0FBQyxPQUFPLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUVsRDtJQUVELDZCQUE2QixDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBRW5EO0lBRUQsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEdBQUcsS0FBSyxDQVE1QjtJQUVELFNBQVMsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FPOUI7SUFFTSxLQUFLLFNBOEJYO0lBRU0sU0FBUyxZQUVmO0lBRVksS0FBSyxrQkFVakI7SUFFTSxJQUFJLGtCQUVWO1lBRWEsTUFBTTtJQWtCYixlQUFlLElBQUksTUFBTSxHQUFHLFNBQVMsQ0FJM0M7SUFFTSx3QkFBd0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxHQUFHLE9BQU8sQ0FLN0Q7SUFFRCxPQUFjLG1CQUFtQixDQUMvQixRQUFRLEVBQUUsU0FBUyxFQUNuQixTQUFTLEVBQUUsS0FBSyxDQUFDLG9DQUFvQyxDQUFDLEdBQ3JELGFBQWEsQ0E2SmY7SUFFRCxPQUFPLENBQUMsTUFBTSxDQUFDLHNCQUFzQjtJQXNFckMsT0FBYyxjQUFjLENBQUMsTUFBTSxFQUFFLHFCQUFxQixFQUFFLEdBQUcsRUFBRSxNQUFNLEdBQUcsbUJBQW1CLENBUTVGO0NBQ0YifQ==
@@ -1 +1 @@
1
- {"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,EACL,KAAK,OAAO,EAKZ,KAAK,MAAM,EACX,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAGL,aAAa,EAEb,KAAK,oCAAoC,EAE1C,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAGzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAA4B,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAG5G,OAAO,KAAK,EAGV,KAAK,EAGL,eAAe,EAEhB,MAAM,gBAAgB,CAAC;AAyCxB,MAAM,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAEnG,qBAAa,mBAAoB,YAAW,eAAe;IAOvD,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,GAAG;IAXb,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACvD,OAAO,CAAC,MAAM,CAA8D;IAC5E,OAAO,CAAC,OAAO,CAAkD;IAEjE,SAAS,aACC,QAAQ,EAAE,SAAS,EACnB,aAAa,EAAE,aAAa,EAC5B,aAAa,EAAE,cAAc,EAC7B,cAAc,EAAE,cAAc,GAAG,SAAS,EAC1C,oBAAoB,EAAE,8BAA8B,GAAG,SAAS,EAChE,GAAG,EAAE,MAAM,EACjB;IAEJ,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAElD;IAED,6BAA6B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAEnD;IAED,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAQ5B;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO9B;IAEM,KAAK,SA8BX;IAEM,SAAS,YAEf;IAEY,KAAK,kBAMjB;IAEY,IAAI,kBAgBhB;IAEM,eAAe,IAAI,MAAM,GAAG,SAAS,CAI3C;IAEM,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAK7D;IAED,OAAc,mBAAmB,CAC/B,QAAQ,EAAE,SAAS,EACnB,SAAS,EAAE,KAAK,CAAC,oCAAoC,CAAC,GACrD,aAAa,CA+Hf;IAED,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAiErC,OAAc,cAAc,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAQ5F;CACF"}
1
+ {"version":3,"file":"otel.d.ts","sourceRoot":"","sources":["../src/otel.ts"],"names":[],"mappings":"AAAA,OAAO,EAAgB,KAAK,MAAM,EAAqB,MAAM,uBAAuB,CAAC;AAErF,OAAO,EACL,KAAK,OAAO,EAKZ,KAAK,MAAM,EACX,KAAK,cAAc,EAMpB,MAAM,oBAAoB,CAAC;AAI5B,OAAO,EAAE,WAAW,EAAE,MAAM,6BAA6B,CAAC;AAC1D,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,KAAK,EAAE,cAAc,EAAE,MAAM,yBAAyB,CAAC;AAC9D,OAAO,EAGL,aAAa,EAEb,KAAK,oCAAoC,EAE1C,MAAM,4BAA4B,CAAC;AAIpC,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAIzD,OAAO,EAAE,oBAAoB,EAAE,MAAM,6BAA6B,CAAC;AACnE,OAAO,EAA4B,8BAA8B,EAAE,MAAM,kCAAkC,CAAC;AAG5G,OAAO,KAAK,EAGV,KAAK,EAGL,eAAe,EAEhB,MAAM,gBAAgB,CAAC;AAyCxB,MAAM,MAAM,0BAA0B,GAAG,CAAC,QAAQ,EAAE,SAAS,EAAE,GAAG,EAAE,MAAM,KAAK,mBAAmB,CAAC;AAEnG,qBAAa,mBAAoB,YAAW,eAAe;IAYvD,OAAO,CAAC,QAAQ;IAChB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,aAAa;IACrB,OAAO,CAAC,cAAc;IACtB,OAAO,CAAC,oBAAoB;IAC5B,OAAO,CAAC,GAAG;IAhBb,WAAW,EAAE,WAAW,GAAG,SAAS,CAAC;IACrC,oBAAoB,EAAE,oBAAoB,GAAG,SAAS,CAAC;IACvD,OAAO,CAAC,MAAM,CAA8D;IAC5E,OAAO,CAAC,OAAO,CAAkD;IAEjE;;yHAEqH;IACrH,OAAO,CAAC,WAAW,CAA4B;IAE/C,SAAS,aACC,QAAQ,EAAE,SAAS,EACnB,aAAa,EAAE,aAAa,EAC5B,aAAa,EAAE,cAAc,EAC7B,cAAc,EAAE,cAAc,GAAG,SAAS,EAC1C,oBAAoB,EAAE,8BAA8B,GAAG,SAAS,EAChE,GAAG,EAAE,MAAM,EACjB;IAEJ,0BAA0B,CAAC,OAAO,EAAE,MAAM,EAAE,GAAG,IAAI,CAElD;IAED,6BAA6B,CAAC,KAAK,EAAE,MAAM,EAAE,GAAG,IAAI,CAEnD;IAED,QAAQ,CAAC,IAAI,EAAE,MAAM,GAAG,KAAK,CAQ5B;IAED,SAAS,CAAC,IAAI,EAAE,MAAM,GAAG,MAAM,CAO9B;IAEM,KAAK,SA8BX;IAEM,SAAS,YAEf;IAEY,KAAK,kBAUjB;IAEM,IAAI,kBAEV;YAEa,MAAM;IAkBb,eAAe,IAAI,MAAM,GAAG,SAAS,CAI3C;IAEM,wBAAwB,CAAC,YAAY,EAAE,MAAM,GAAG,OAAO,CAK7D;IAED,OAAc,mBAAmB,CAC/B,QAAQ,EAAE,SAAS,EACnB,SAAS,EAAE,KAAK,CAAC,oCAAoC,CAAC,GACrD,aAAa,CA6Jf;IAED,OAAO,CAAC,MAAM,CAAC,sBAAsB;IAsErC,OAAc,cAAc,CAAC,MAAM,EAAE,qBAAqB,EAAE,GAAG,EAAE,MAAM,GAAG,mBAAmB,CAQ5F;CACF"}
package/dest/otel.js CHANGED
@@ -5,9 +5,10 @@ import { OTLPMetricExporter } from '@opentelemetry/exporter-metrics-otlp-http';
5
5
  import { OTLPTraceExporter } from '@opentelemetry/exporter-trace-otlp-http';
6
6
  import { HostMetrics } from '@opentelemetry/host-metrics';
7
7
  import { ExplicitBucketHistogramAggregation, InstrumentType, MeterProvider, PeriodicExportingMetricReader, View } from '@opentelemetry/sdk-metrics';
8
- import { BatchSpanProcessor, NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
8
+ import { NodeTracerProvider } from '@opentelemetry/sdk-trace-node';
9
9
  import { ATTR_SERVICE_NAME, ATTR_SERVICE_VERSION } from '@opentelemetry/semantic-conventions';
10
10
  import { toMetricOptions } from './metric-utils.js';
11
+ import { MonitoredBatchSpanProcessor } from './monitored_batch_span_processor.js';
11
12
  import { NodejsMetricsMonitor } from './nodejs_metrics_monitor.js';
12
13
  import { OtelFilterMetricExporter, PublicOtelFilterMetricExporter } from './otel_filter_metric_exporter.js';
13
14
  import { registerOtelLoggerProvider } from './otel_logger_provider.js';
@@ -53,6 +54,9 @@ export class OpenTelemetryClient {
53
54
  nodejsMetricsMonitor;
54
55
  meters;
55
56
  tracers;
57
+ /** Memoized shutdown promise. The telemetry client is shared between the aztec-node and an embedded prover-node,
58
+ * so stop() can be invoked more than once; the providers throw "shutdown may only be called once" and
59
+ * "invalid attempt to force flush after shutdown" if that happens. Guarding here makes stop()/flush() idempotent. */ stopPromise;
56
60
  constructor(resource, meterProvider, traceProvider, loggerProvider, publicMetricExporter, log){
57
61
  this.resource = resource;
58
62
  this.meterProvider = meterProvider;
@@ -113,13 +117,20 @@ export class OpenTelemetryClient {
113
117
  return true;
114
118
  }
115
119
  async flush() {
120
+ // Flushing after the providers have been shut down throws "invalid attempt to force flush after shutdown".
121
+ if (this.stopPromise) {
122
+ return;
123
+ }
116
124
  await Promise.all([
117
125
  this.meterProvider.forceFlush(),
118
126
  this.loggerProvider?.forceFlush(),
119
127
  this.traceProvider instanceof NodeTracerProvider ? this.traceProvider.forceFlush() : Promise.resolve()
120
128
  ]);
121
129
  }
122
- async stop() {
130
+ stop() {
131
+ return this.stopPromise ??= this.doStop();
132
+ }
133
+ async doStop() {
123
134
  this.nodejsMetricsMonitor?.stop();
124
135
  const flushAndShutdown = async (provider)=>{
125
136
  if (!provider) {
@@ -369,6 +380,61 @@ export class OpenTelemetryClient {
369
380
  15_000_000,
370
381
  30_000_000
371
382
  ], true)
383
+ }),
384
+ // L1 gas prices in gwei: priority fees ~0.01-10, base fees ~1-500, spikes to 1000+
385
+ new View({
386
+ instrumentType: InstrumentType.HISTOGRAM,
387
+ instrumentUnit: 'gwei',
388
+ aggregation: new ExplicitBucketHistogramAggregation([
389
+ 0.1,
390
+ 0.5,
391
+ 1,
392
+ 2,
393
+ 5,
394
+ 10,
395
+ 20,
396
+ 50,
397
+ 100,
398
+ 200,
399
+ 500,
400
+ 1_000
401
+ ], true)
402
+ }),
403
+ // L1 gas consumption: tx gas 100k-30M, calldata/blob gas varies
404
+ new View({
405
+ instrumentType: InstrumentType.HISTOGRAM,
406
+ instrumentUnit: 'gas',
407
+ aggregation: new ExplicitBucketHistogramAggregation([
408
+ 10_000,
409
+ 50_000,
410
+ 100_000,
411
+ 250_000,
412
+ 500_000,
413
+ 1_000_000,
414
+ 2_000_000,
415
+ 5_000_000,
416
+ 10_000_000,
417
+ 15_000_000,
418
+ 30_000_000
419
+ ], true)
420
+ }),
421
+ // L1 tx total fee in ETH: typically 0.001 - 1 ETH
422
+ new View({
423
+ instrumentType: InstrumentType.HISTOGRAM,
424
+ instrumentUnit: 'eth',
425
+ aggregation: new ExplicitBucketHistogramAggregation([
426
+ 0.0001,
427
+ 0.0005,
428
+ 0.001,
429
+ 0.005,
430
+ 0.01,
431
+ 0.05,
432
+ 0.1,
433
+ 0.5,
434
+ 1,
435
+ 5,
436
+ 10
437
+ ], true)
372
438
  })
373
439
  ]
374
440
  });
@@ -378,9 +444,12 @@ export class OpenTelemetryClient {
378
444
  const tracerProvider = new NodeTracerProvider({
379
445
  resource,
380
446
  spanProcessors: config.tracesCollectorUrl ? [
381
- new BatchSpanProcessor(new OTLPTraceExporter({
447
+ new MonitoredBatchSpanProcessor(new OTLPTraceExporter({
382
448
  url: config.tracesCollectorUrl.href
383
- }))
449
+ }), log, {
450
+ maxQueueSize: config.otelBspMaxQueueSize,
451
+ minTraceDurationMs: config.otelMinTraceDurationMs
452
+ })
384
453
  ] : []
385
454
  });
386
455
  tracerProvider.register({
@@ -1,3 +1,5 @@
1
+ import type { DiagnosticsMiddleware } from '@aztec/foundation/json-rpc/server';
1
2
  import type Koa from 'koa';
2
3
  export declare function getOtelJsonRpcPropagationMiddleware(scope?: string): (ctx: Koa.Context, next: () => Promise<void>) => Promise<void>;
3
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9wcm9wYWdhdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWxfcHJvcGFnYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQ0EsT0FBTyxLQUFLLEdBQUcsTUFBTSxLQUFLLENBQUM7QUFVM0Isd0JBQWdCLG1DQUFtQyxDQUNqRCxLQUFLLFNBQWtCLEdBQ3RCLENBQUMsR0FBRyxFQUFFLEdBQUcsQ0FBQyxPQUFPLEVBQUUsSUFBSSxFQUFFLE1BQU0sT0FBTyxDQUFDLElBQUksQ0FBQyxLQUFLLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FvQ2hFIn0=
4
+ export declare function getOtelJsonRpcDiagnosticsMiddleware(): DiagnosticsMiddleware;
5
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoib3RlbF9wcm9wYWdhdGlvbi5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vc3JjL290ZWxfcHJvcGFnYXRpb24udHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxtQ0FBbUMsQ0FBQztBQUcvRSxPQUFPLEtBQUssR0FBRyxNQUFNLEtBQUssQ0FBQztBQVUzQix3QkFBZ0IsbUNBQW1DLENBQ2pELEtBQUssU0FBa0IsR0FDdEIsQ0FBQyxHQUFHLEVBQUUsR0FBRyxDQUFDLE9BQU8sRUFBRSxJQUFJLEVBQUUsTUFBTSxPQUFPLENBQUMsSUFBSSxDQUFDLEtBQUssT0FBTyxDQUFDLElBQUksQ0FBQyxDQW9DaEU7QUFFRCx3QkFBZ0IsbUNBQW1DLElBQUkscUJBQXFCLENBNEIzRSJ9
@@ -1 +1 @@
1
- {"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAU3B,wBAAgB,mCAAmC,CACjD,KAAK,SAAkB,GACtB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAoChE"}
1
+ {"version":3,"file":"otel_propagation.d.ts","sourceRoot":"","sources":["../src/otel_propagation.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,mCAAmC,CAAC;AAG/E,OAAO,KAAK,GAAG,MAAM,KAAK,CAAC;AAU3B,wBAAgB,mCAAmC,CACjD,KAAK,SAAkB,GACtB,CAAC,GAAG,EAAE,GAAG,CAAC,OAAO,EAAE,IAAI,EAAE,MAAM,OAAO,CAAC,IAAI,CAAC,KAAK,OAAO,CAAC,IAAI,CAAC,CAoChE;AAED,wBAAgB,mCAAmC,IAAI,qBAAqB,CA4B3E"}
@@ -6,7 +6,7 @@ export function getOtelJsonRpcPropagationMiddleware(scope = 'JsonRpcServer') {
6
6
  const tracer = getTelemetryClient().getTracer(scope);
7
7
  const context = propagation.extract(ROOT_CONTEXT, ctx.request.headers);
8
8
  const method = ctx.request.body?.method;
9
- return tracer.startActiveSpan(`JsonRpcServer.${method ?? 'unknown'}`, {
9
+ return tracer.startActiveSpan(`JsonRpcServer.${method ?? 'batch'}`, {
10
10
  kind: SpanKind.SERVER
11
11
  }, context, async (span)=>{
12
12
  if (ctx.id) {
@@ -42,3 +42,51 @@ export function getOtelJsonRpcPropagationMiddleware(scope = 'JsonRpcServer') {
42
42
  });
43
43
  };
44
44
  }
45
+ export function getOtelJsonRpcDiagnosticsMiddleware() {
46
+ return function otelJsonRpcDiagnostics(ctx, next) {
47
+ const [namespace, method] = splitNamespace(ctx.method);
48
+ const scope = namespace ?? 'UnknownHandler';
49
+ const tracer = getTelemetryClient().getTracer(scope);
50
+ return tracer.startActiveSpan(`${scope}.${method}`, {
51
+ kind: SpanKind.INTERNAL,
52
+ attributes: {
53
+ [ATTR_JSONRPC_METHOD]: ctx.method
54
+ }
55
+ }, async (span)=>{
56
+ if (ctx.id !== null) {
57
+ span.setAttribute(ATTR_JSONRPC_REQUEST_ID, ctx.id);
58
+ }
59
+ try {
60
+ await next();
61
+ span.setStatus({
62
+ code: SpanStatusCode.OK
63
+ });
64
+ } catch (err) {
65
+ span.setStatus({
66
+ code: SpanStatusCode.ERROR,
67
+ message: err instanceof Error ? err.message : String(err)
68
+ });
69
+ if (typeof err === 'string' || err instanceof Error) {
70
+ span.recordException(err);
71
+ }
72
+ throw err;
73
+ } finally{
74
+ span.end();
75
+ }
76
+ });
77
+ };
78
+ }
79
+ function splitNamespace(fullMethod) {
80
+ const idx = fullMethod.indexOf('_');
81
+ if (idx > -1) {
82
+ return [
83
+ fullMethod.slice(0, idx),
84
+ fullMethod.slice(idx + 1)
85
+ ];
86
+ } else {
87
+ return [
88
+ undefined,
89
+ fullMethod
90
+ ];
91
+ }
92
+ }
package/dest/start.d.ts CHANGED
@@ -4,4 +4,4 @@ import type { TelemetryClient } from './telemetry.js';
4
4
  export * from './config.js';
5
5
  export declare function initTelemetryClient(config: TelemetryClientConfig, bindings?: LoggerBindings): Promise<TelemetryClient>;
6
6
  export declare function getTelemetryClient(): TelemetryClient;
7
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RhcnQuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9zdGFydC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsS0FBSyxjQUFjLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFFMUUsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFekQsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFdEQsY0FBYyxhQUFhLENBQUM7QUFLNUIsd0JBQXNCLG1CQUFtQixDQUN2QyxNQUFNLEVBQUUscUJBQXFCLEVBQzdCLFFBQVEsQ0FBQyxFQUFFLGNBQWMsR0FDeEIsT0FBTyxDQUFDLGVBQWUsQ0FBQyxDQWtCMUI7QUFFRCx3QkFBZ0Isa0JBQWtCLElBQUksZUFBZSxDQUVwRCJ9
7
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoic3RhcnQuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy9zdGFydC50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEVBQUUsS0FBSyxjQUFjLEVBQWdCLE1BQU0sdUJBQXVCLENBQUM7QUFFMUUsT0FBTyxLQUFLLEVBQUUscUJBQXFCLEVBQUUsTUFBTSxhQUFhLENBQUM7QUFFekQsT0FBTyxLQUFLLEVBQUUsZUFBZSxFQUFFLE1BQU0sZ0JBQWdCLENBQUM7QUFFdEQsY0FBYyxhQUFhLENBQUM7QUFLNUIsd0JBQXNCLG1CQUFtQixDQUN2QyxNQUFNLEVBQUUscUJBQXFCLEVBQzdCLFFBQVEsQ0FBQyxFQUFFLGNBQWMsR0FDeEIsT0FBTyxDQUFDLGVBQWUsQ0FBQyxDQXVCMUI7QUFFRCx3QkFBZ0Isa0JBQWtCLElBQUksZUFBZSxDQUVwRCJ9
@@ -1 +1 @@
1
- {"version":3,"file":"start.d.ts","sourceRoot":"","sources":["../src/start.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AAE1E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,cAAc,aAAa,CAAC;AAK5B,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,qBAAqB,EAC7B,QAAQ,CAAC,EAAE,cAAc,GACxB,OAAO,CAAC,eAAe,CAAC,CAkB1B;AAED,wBAAgB,kBAAkB,IAAI,eAAe,CAEpD"}
1
+ {"version":3,"file":"start.d.ts","sourceRoot":"","sources":["../src/start.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,KAAK,cAAc,EAAgB,MAAM,uBAAuB,CAAC;AAE1E,OAAO,KAAK,EAAE,qBAAqB,EAAE,MAAM,aAAa,CAAC;AAEzD,OAAO,KAAK,EAAE,eAAe,EAAE,MAAM,gBAAgB,CAAC;AAEtD,cAAc,aAAa,CAAC;AAK5B,wBAAsB,mBAAmB,CACvC,MAAM,EAAE,qBAAqB,EAC7B,QAAQ,CAAC,EAAE,cAAc,GACxB,OAAO,CAAC,eAAe,CAAC,CAuB1B;AAED,wBAAgB,kBAAkB,IAAI,eAAe,CAEpD"}
package/dest/start.js CHANGED
@@ -9,7 +9,7 @@ export async function initTelemetryClient(config, bindings) {
9
9
  log.warn('Telemetry client has already been initialized once');
10
10
  return telemetry;
11
11
  }
12
- if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl) {
12
+ if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl || config.tracesCollectorUrl || config.logsCollectorUrl) {
13
13
  log.info(`Using OpenTelemetry client with custom collector`);
14
14
  // Lazy load OpenTelemetry to avoid loading heavy deps at startup
15
15
  const { OpenTelemetryClient } = await import('./otel.js');
@@ -12,7 +12,7 @@ type AttributeNames = ValuesOf<typeof Attributes>;
12
12
  *
13
13
  * Think twice before removing an attribute from this list.
14
14
  */
15
- type BannedMetricAttributeNames = (typeof Attributes)['BLOCK_NUMBER' | 'BLOCK_ARCHIVE' | 'SLOT_NUMBER' | 'BLOCK_PARENT' | 'BLOCK_CANDIDATE_TXS_COUNT' | 'BLOCK_TXS_COUNT' | 'BLOCK_SIZE' | 'EPOCH_SIZE' | 'EPOCH_NUMBER' | 'TX_HASH' | 'PROVING_JOB_ID' | 'P2P_ID' | 'P2P_REQ_RESP_BATCH_REQUESTS_COUNT' | 'TARGET_ADDRESS' | 'MANA_USED' | 'TOTAL_INSTRUCTIONS'];
15
+ type BannedMetricAttributeNames = (typeof Attributes)['BLOCK_NUMBER' | 'BLOCK_ARCHIVE' | 'SLOT_NUMBER' | 'BLOCK_PARENT' | 'BLOCK_CANDIDATE_TXS_COUNT' | 'BLOCK_TXS_COUNT' | 'BLOCK_SIZE' | 'EPOCH_SIZE' | 'EPOCH_NUMBER' | 'TX_HASH' | 'PROVING_JOB_ID' | 'P2P_ID' | 'TARGET_ADDRESS' | 'MANA_USED' | 'TOTAL_INSTRUCTIONS'];
16
16
  /** Global registry of attributes */
17
17
  export type AttributesType = Partial<Record<AttributeNames, AttributeValue>>;
18
18
  export type AllowedAttributeNames = Exclude<AttributeNames, BannedMetricAttributeNames>;
@@ -144,4 +144,4 @@ export declare function runInSpan<A extends any[], R>(tracer: Tracer | string, s
144
144
  * Unlike runInSpan which returns a reusable function, this executes right away.
145
145
  */
146
146
  export declare function execInSpan<R>(tracer: Tracer | string, spanName: string, callback: (span: Span) => Promise<R>): Promise<R>;
147
- //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVsZW1ldHJ5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvdGVsZW1ldHJ5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFDTCxLQUFLLGNBQWMsRUFDbkIsS0FBSyx1QkFBdUIsRUFDNUIsS0FBSyxPQUFPLEVBQ1osS0FBSyxhQUFhLEVBQ2xCLEtBQUssVUFBVSxFQUNmLEtBQUsscUJBQXFCLElBQUkseUJBQXlCLEVBQ3ZELEtBQUssS0FBSyxJQUFJLFNBQVMsRUFDdkIsS0FBSyxTQUFTLElBQUksYUFBYSxFQUMvQixLQUFLLGVBQWUsSUFBSSxtQkFBbUIsRUFDM0MsS0FBSyxnQkFBZ0IsSUFBSSxvQkFBb0IsRUFDN0MsS0FBSyx1QkFBdUIsSUFBSSwyQkFBMkIsRUFDM0QsS0FBSyxhQUFhLElBQUksaUJBQWlCLEVBQ3ZDLEtBQUssSUFBSSxFQUVULEtBQUssTUFBTSxFQUNaLE1BQU0sb0JBQW9CLENBQUM7QUFFNUIsT0FBTyxLQUFLLEtBQUssVUFBVSxNQUFNLGlCQUFpQixDQUFDO0FBQ25ELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBR3JELE9BQU8sRUFBRSxlQUFlLEVBQUUsOEJBQThCLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUVwRixPQUFPLEVBQUUsS0FBSyxJQUFJLEVBQUUsY0FBYyxFQUFFLFNBQVMsRUFBRSxLQUFLLE9BQU8sRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBRXhGLEtBQUssUUFBUSxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsTUFBTSxDQUFDLE1BQU0sRUFBRSxNQUFNLENBQUMsQ0FBQyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUM7QUFFakUsS0FBSyxjQUFjLEdBQUcsUUFBUSxDQUFDLE9BQU8sVUFBVSxDQUFDLENBQUM7QUFFbEQ7Ozs7OztHQU1HO0FBQ0gsS0FBSywwQkFBMEIsR0FBRyxDQUFDLE9BQU8sVUFBVSxDQUFDLENBQ2pELGNBQWMsR0FDZCxlQUFlLEdBQ2YsYUFBYSxHQUNiLGNBQWMsR0FDZCwyQkFBMkIsR0FDM0IsaUJBQWlCLEdBQ2pCLFlBQVksR0FDWixZQUFZLEdBQ1osY0FBYyxHQUNkLFNBQVMsR0FDVCxnQkFBZ0IsR0FDaEIsUUFBUSxHQUNSLG1DQUFtQyxHQUNuQyxnQkFBZ0IsR0FDaEIsV0FBVyxHQUNYLG9CQUFvQixDQUFDLENBQUM7QUFFMUIsb0NBQW9DO0FBQ3BDLE1BQU0sTUFBTSxjQUFjLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxjQUFjLEVBQUUsY0FBYyxDQUFDLENBQUMsQ0FBQztBQUU3RSxNQUFNLE1BQU0scUJBQXFCLEdBQUcsT0FBTyxDQUFDLGNBQWMsRUFBRSwwQkFBMEIsQ0FBQyxDQUFDO0FBRXhGLDBEQUEwRDtBQUMxRCxNQUFNLE1BQU0sb0JBQW9CLEdBQUcsT0FBTyxDQUFDLE1BQU0sQ0FBQyxxQkFBcUIsRUFBRSxjQUFjLENBQUMsQ0FBQyxDQUFDO0FBRTFGLGlEQUFpRDtBQUNqRCxZQUFZLEVBQUUsZ0JBQWdCLEVBQUUsTUFBTSxjQUFjLENBQUM7QUFFckQsTUFBTSxNQUFNLEtBQUssR0FBRyxTQUFTLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUNwRCxNQUFNLE1BQU0sU0FBUyxHQUFHLGFBQWEsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDO0FBQzVELE1BQU0sTUFBTSxhQUFhLEdBQUcsaUJBQWlCLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUNwRSxNQUFNLE1BQU0sZUFBZSxHQUFHLG1CQUFtQixDQUFDLG9CQUFvQixDQUFDLENBQUM7QUFDeEUsTUFBTSxNQUFNLHVCQUF1QixHQUFHLDJCQUEyQixDQUFDLG9CQUFvQixDQUFDLENBQUM7QUFDeEYsTUFBTSxNQUFNLGdCQUFnQixHQUFHLG9CQUFvQixDQUFDLG9CQUFvQixDQUFDLENBQUM7QUFDMUUsTUFBTSxNQUFNLHFCQUFxQixHQUFHLHlCQUF5QixDQUFDLG9CQUFvQixDQUFDLENBQUM7QUFFcEYsWUFBWSxFQUFFLE1BQU0sRUFBRSxDQUFDO0FBR3ZCOztHQUVHO0FBQ0gsTUFBTSxXQUFXLEtBQUs7SUFDcEI7OztPQUdHO0lBQ0gsV0FBVyxDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsR0FBRyxLQUFLLENBQUM7SUFFN0M7OztPQUdHO0lBQ0gscUJBQXFCLENBQUMsTUFBTSxFQUFFLGdCQUFnQixHQUFHLGVBQWUsQ0FBQztJQUVqRSwwQkFBMEIsQ0FDeEIsUUFBUSxFQUFFLHVCQUF1QixDQUFDLGNBQWMsQ0FBQyxFQUNqRCxXQUFXLEVBQUUsVUFBVSxDQUFDLGNBQWMsQ0FBQyxFQUFFLEdBQ3hDLElBQUksQ0FBQztJQUVSLDZCQUE2QixDQUMzQixRQUFRLEVBQUUsdUJBQXVCLENBQUMsY0FBYyxDQUFDLEVBQ2pELFdBQVcsRUFBRSxVQUFVLENBQUMsY0FBYyxDQUFDLEVBQUUsR0FDeEMsSUFBSSxDQUFDO0lBRVI7Ozs7T0FJRztJQUNILGVBQWUsQ0FBQyxNQUFNLEVBQUUsZ0JBQWdCLEVBQUUsWUFBWSxDQUFDLEVBQUUsT0FBTyxDQUFDLGFBQWEsQ0FBQyxHQUFHLFNBQVMsQ0FBQztJQUU1Rjs7O09BR0c7SUFDSCxtQkFBbUIsQ0FBQyxNQUFNLEVBQUUsZ0JBQWdCLEdBQUcsYUFBYSxDQUFDO0lBRTdEOzs7T0FHRztJQUNILDZCQUE2QixDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsR0FBRyx1QkFBdUIsQ0FBQztDQUNsRjtBQUVEOztHQUVHO0FBQ0gsTUFBTSxXQUFXLGVBQWU7SUFDOUI7O1FBRUk7SUFDSixTQUFTLElBQUksT0FBTyxDQUFDO0lBQ3JCOzs7T0FHRztJQUNILFFBQVEsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLEtBQUssQ0FBQztJQUU5Qjs7O09BR0c7SUFDSCxTQUFTLENBQUMsSUFBSSxFQUFFLE1BQU0sR0FBRyxNQUFNLENBQUM7SUFFaEM7O09BRUc7SUFDSCxJQUFJLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXRCOztPQUVHO0lBQ0gsS0FBSyxJQUFJLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FBQztJQUV2Qjs7T0FFRztJQUNILDBCQUEwQixDQUFDLFFBQVEsRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBQUM7SUFFckQ7O09BRUc7SUFDSCw2QkFBNkIsQ0FBQyxLQUFLLEVBQUUsTUFBTSxFQUFFLEdBQUcsSUFBSSxDQUFDO0lBRXJEOzs7T0FHRztJQUNILGVBQWUsSUFBSSxNQUFNLEdBQUcsU0FBUyxDQUFDO0lBRXRDOztPQUVHO0lBQ0gsd0JBQXdCLENBQUMsWUFBWSxFQUFFLE1BQU0sR0FBRyxPQUFPLEdBQUcsU0FBUyxDQUFDO0NBQ3JFO0FBRUQsK0RBQStEO0FBQy9ELE1BQU0sV0FBVyxTQUFTO0lBQ3hCLE1BQU0sRUFBRSxNQUFNLENBQUM7Q0FDaEI7QUFFRCxLQUFLLGFBQWEsQ0FBQyxDQUFDLFNBQVMsU0FBUyxFQUFFLENBQUMsU0FBUyxDQUFDLEdBQUcsSUFBSSxFQUFFLEdBQUcsRUFBRSxLQUFLLEdBQUcsSUFBSSxDQUMzRSxjQUFjLEVBQUUsQ0FBQyxFQUNqQixPQUFPLEVBQUUsMkJBQTJCLENBQUMsQ0FBQyxDQUFDLEtBQ3BDLENBQUMsQ0FBQztBQUVQOzs7Ozs7Ozs7Ozs7OztHQWNHO0FBQ0gsd0JBQWdCLFNBQVMsQ0FBQyxDQUFDLFNBQVMsU0FBUyxFQUFFLENBQUMsU0FBUyxDQUFDLEdBQUcsSUFBSSxFQUFFLEdBQUcsRUFBRSxLQUFLLEdBQUcsRUFDOUUsUUFBUSxFQUFFLE1BQU0sR0FBRyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsRUFBRSxHQUFHLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQyxDQUFDLEtBQUssTUFBTSxDQUFDLEVBQ2hFLFVBQVUsQ0FBQyxFQUFFLGNBQWMsR0FBRyxDQUFDLENBQUMsSUFBSSxFQUFFLENBQUMsRUFBRSxHQUFHLElBQUksRUFBRSxVQUFVLENBQUMsQ0FBQyxDQUFDLEtBQUssT0FBTyxDQUFDLGNBQWMsQ0FBQyxHQUFHLGNBQWMsQ0FBQyxFQUM3RyxlQUFlLENBQUMsRUFBRSxDQUFDLElBQUksRUFBRSxDQUFDLEVBQUUsV0FBVyxFQUFFLE9BQU8sQ0FBQyxVQUFVLENBQUMsQ0FBQyxDQUFDLENBQUMsS0FBSyxjQUFjLEdBQ2pGLGFBQWEsQ0FBQyxDQUFDLEVBQUUsQ0FBQyxDQUFDLENBNkNyQjtBQUVEOzs7Ozs7Ozs7O0dBVUc7QUFDSCx3QkFBZ0Isa0JBQWtCLENBQUMsQ0FBQyxTQUFTLENBQUMsR0FBRyxJQUFJLEVBQUUsR0FBRyxFQUFFLEtBQUssR0FBRyxFQUNsRSxNQUFNLEVBQUUsTUFBTSxFQUNkLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLFVBQVUsRUFBRSxjQUFjLEVBQzFCLFFBQVEsRUFBRSxDQUFDLEdBQ1YsQ0FBQyxDQWtCSDtBQUVELHdCQUFnQixTQUFTLENBQUMsQ0FBQyxTQUFTLEdBQUcsRUFBRSxFQUFFLENBQUMsRUFDMUMsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLEVBQ3ZCLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLFFBQVEsRUFBRSxDQUFDLElBQUksRUFBRSxJQUFJLEVBQUUsR0FBRyxJQUFJLEVBQUUsQ0FBQyxLQUFLLE9BQU8sQ0FBQyxDQUFDLENBQUMsR0FDL0MsQ0FBQyxHQUFHLElBQUksRUFBRSxDQUFDLEtBQUssT0FBTyxDQUFDLENBQUMsQ0FBQyxDQXlCNUI7QUFFRDs7O0dBR0c7QUFDSCx3QkFBZ0IsVUFBVSxDQUFDLENBQUMsRUFDMUIsTUFBTSxFQUFFLE1BQU0sR0FBRyxNQUFNLEVBQ3ZCLFFBQVEsRUFBRSxNQUFNLEVBQ2hCLFFBQVEsRUFBRSxDQUFDLElBQUksRUFBRSxJQUFJLEtBQUssT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUNuQyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBaUJaIn0=
147
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidGVsZW1ldHJ5LmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvdGVsZW1ldHJ5LnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUFBLE9BQU8sRUFDTCxLQUFLLGNBQWMsRUFDbkIsS0FBSyx1QkFBdUIsRUFDNUIsS0FBSyxPQUFPLEVBQ1osS0FBSyxhQUFhLEVBQ2xCLEtBQUssVUFBVSxFQUNmLEtBQUsscUJBQXFCLElBQUkseUJBQXlCLEVBQ3ZELEtBQUssS0FBSyxJQUFJLFNBQVMsRUFDdkIsS0FBSyxTQUFTLElBQUksYUFBYSxFQUMvQixLQUFLLGVBQWUsSUFBSSxtQkFBbUIsRUFDM0MsS0FBSyxnQkFBZ0IsSUFBSSxvQkFBb0IsRUFDN0MsS0FBSyx1QkFBdUIsSUFBSSwyQkFBMkIsRUFDM0QsS0FBSyxhQUFhLElBQUksaUJBQWlCLEVBQ3ZDLEtBQUssSUFBSSxFQUVULEtBQUssTUFBTSxFQUNaLE1BQU0sb0JBQW9CLENBQUM7QUFFNUIsT0FBTyxLQUFLLEtBQUssVUFBVSxNQUFNLGlCQUFpQixDQUFDO0FBQ25ELE9BQU8sS0FBSyxFQUFFLGdCQUFnQixFQUFFLE1BQU0sY0FBYyxDQUFDO0FBR3JELE9BQU8sRUFBRSxlQUFlLEVBQUUsOEJBQThCLEVBQUUsTUFBTSxtQkFBbUIsQ0FBQztBQUVwRixPQUFPLEVBQUUsS0FBSyxJQUFJLEVBQUUsY0FBYyxFQUFFLFNBQVMsRUFBRSxLQUFLLE9BQU8sRUFBRSxNQUFNLG9CQUFvQixDQUFDO0FBRXhGLEtBQUssUUFBUSxDQUFDLENBQUMsSUFBSSxDQUFDLFNBQVMsTUFBTSxDQUFDLE1BQU0sRUFBRSxNQUFNLENBQUMsQ0FBQyxHQUFHLENBQUMsR0FBRyxLQUFLLENBQUM7QUFFakUsS0FBSyxjQUFjLEdBQUcsUUFBUSxDQUFDLE9BQU8sVUFBVSxDQUFDLENBQUM7QUFFbEQ7Ozs7OztHQU1HO0FBQ0gsS0FBSywwQkFBMEIsR0FBRyxDQUFDLE9BQU8sVUFBVSxDQUFDLENBQ2pELGNBQWMsR0FDZCxlQUFlLEdBQ2YsYUFBYSxHQUNiLGNBQWMsR0FDZCwyQkFBMkIsR0FDM0IsaUJBQWlCLEdBQ2pCLFlBQVksR0FDWixZQUFZLEdBQ1osY0FBYyxHQUNkLFNBQVMsR0FDVCxnQkFBZ0IsR0FDaEIsUUFBUSxHQUNSLGdCQUFnQixHQUNoQixXQUFXLEdBQ1gsb0JBQW9CLENBQUMsQ0FBQztBQUUxQixvQ0FBb0M7QUFDcEMsTUFBTSxNQUFNLGNBQWMsR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLGNBQWMsRUFBRSxjQUFjLENBQUMsQ0FBQyxDQUFDO0FBRTdFLE1BQU0sTUFBTSxxQkFBcUIsR0FBRyxPQUFPLENBQUMsY0FBYyxFQUFFLDBCQUEwQixDQUFDLENBQUM7QUFFeEYsMERBQTBEO0FBQzFELE1BQU0sTUFBTSxvQkFBb0IsR0FBRyxPQUFPLENBQUMsTUFBTSxDQUFDLHFCQUFxQixFQUFFLGNBQWMsQ0FBQyxDQUFDLENBQUM7QUFFMUYsaURBQWlEO0FBQ2pELFlBQVksRUFBRSxnQkFBZ0IsRUFBRSxNQUFNLGNBQWMsQ0FBQztBQUVyRCxNQUFNLE1BQU0sS0FBSyxHQUFHLFNBQVMsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDO0FBQ3BELE1BQU0sTUFBTSxTQUFTLEdBQUcsYUFBYSxDQUFDLG9CQUFvQixDQUFDLENBQUM7QUFDNUQsTUFBTSxNQUFNLGFBQWEsR0FBRyxpQkFBaUIsQ0FBQyxvQkFBb0IsQ0FBQyxDQUFDO0FBQ3BFLE1BQU0sTUFBTSxlQUFlLEdBQUcsbUJBQW1CLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUN4RSxNQUFNLE1BQU0sdUJBQXVCLEdBQUcsMkJBQTJCLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUN4RixNQUFNLE1BQU0sZ0JBQWdCLEdBQUcsb0JBQW9CLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUMxRSxNQUFNLE1BQU0scUJBQXFCLEdBQUcseUJBQXlCLENBQUMsb0JBQW9CLENBQUMsQ0FBQztBQUVwRixZQUFZLEVBQUUsTUFBTSxFQUFFLENBQUM7QUFHdkI7O0dBRUc7QUFDSCxNQUFNLFdBQVcsS0FBSztJQUNwQjs7O09BR0c7SUFDSCxXQUFXLENBQUMsTUFBTSxFQUFFLGdCQUFnQixHQUFHLEtBQUssQ0FBQztJQUU3Qzs7O09BR0c7SUFDSCxxQkFBcUIsQ0FBQyxNQUFNLEVBQUUsZ0JBQWdCLEdBQUcsZUFBZSxDQUFDO0lBRWpFLDBCQUEwQixDQUN4QixRQUFRLEVBQUUsdUJBQXVCLENBQUMsY0FBYyxDQUFDLEVBQ2pELFdBQVcsRUFBRSxVQUFVLENBQUMsY0FBYyxDQUFDLEVBQUUsR0FDeEMsSUFBSSxDQUFDO0lBRVIsNkJBQTZCLENBQzNCLFFBQVEsRUFBRSx1QkFBdUIsQ0FBQyxjQUFjLENBQUMsRUFDakQsV0FBVyxFQUFFLFVBQVUsQ0FBQyxjQUFjLENBQUMsRUFBRSxHQUN4QyxJQUFJLENBQUM7SUFFUjs7OztPQUlHO0lBQ0gsZUFBZSxDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsRUFBRSxZQUFZLENBQUMsRUFBRSxPQUFPLENBQUMsYUFBYSxDQUFDLEdBQUcsU0FBUyxDQUFDO0lBRTVGOzs7T0FHRztJQUNILG1CQUFtQixDQUFDLE1BQU0sRUFBRSxnQkFBZ0IsR0FBRyxhQUFhLENBQUM7SUFFN0Q7OztPQUdHO0lBQ0gsNkJBQTZCLENBQUMsTUFBTSxFQUFFLGdCQUFnQixHQUFHLHVCQUF1QixDQUFDO0NBQ2xGO0FBRUQ7O0dBRUc7QUFDSCxNQUFNLFdBQVcsZUFBZTtJQUM5Qjs7UUFFSTtJQUNKLFNBQVMsSUFBSSxPQUFPLENBQUM7SUFDckI7OztPQUdHO0lBQ0gsUUFBUSxDQUFDLElBQUksRUFBRSxNQUFNLEdBQUcsS0FBSyxDQUFDO0lBRTlCOzs7T0FHRztJQUNILFNBQVMsQ0FBQyxJQUFJLEVBQUUsTUFBTSxHQUFHLE1BQU0sQ0FBQztJQUVoQzs7T0FFRztJQUNILElBQUksSUFBSSxPQUFPLENBQUMsSUFBSSxDQUFDLENBQUM7SUFFdEI7O09BRUc7SUFDSCxLQUFLLElBQUksT0FBTyxDQUFDLElBQUksQ0FBQyxDQUFDO0lBRXZCOztPQUVHO0lBQ0gsMEJBQTBCLENBQUMsUUFBUSxFQUFFLE1BQU0sRUFBRSxHQUFHLElBQUksQ0FBQztJQUVyRDs7T0FFRztJQUNILDZCQUE2QixDQUFDLEtBQUssRUFBRSxNQUFNLEVBQUUsR0FBRyxJQUFJLENBQUM7SUFFckQ7OztPQUdHO0lBQ0gsZUFBZSxJQUFJLE1BQU0sR0FBRyxTQUFTLENBQUM7SUFFdEM7O09BRUc7SUFDSCx3QkFBd0IsQ0FBQyxZQUFZLEVBQUUsTUFBTSxHQUFHLE9BQU8sR0FBRyxTQUFTLENBQUM7Q0FDckU7QUFFRCwrREFBK0Q7QUFDL0QsTUFBTSxXQUFXLFNBQVM7SUFDeEIsTUFBTSxFQUFFLE1BQU0sQ0FBQztDQUNoQjtBQUVELEtBQUssYUFBYSxDQUFDLENBQUMsU0FBUyxTQUFTLEVBQUUsQ0FBQyxTQUFTLENBQUMsR0FBRyxJQUFJLEVBQUUsR0FBRyxFQUFFLEtBQUssR0FBRyxJQUFJLENBQzNFLGNBQWMsRUFBRSxDQUFDLEVBQ2pCLE9BQU8sRUFBRSwyQkFBMkIsQ0FBQyxDQUFDLENBQUMsS0FDcEMsQ0FBQyxDQUFDO0FBRVA7Ozs7Ozs7Ozs7Ozs7O0dBY0c7QUFDSCx3QkFBZ0IsU0FBUyxDQUFDLENBQUMsU0FBUyxTQUFTLEVBQUUsQ0FBQyxTQUFTLENBQUMsR0FBRyxJQUFJLEVBQUUsR0FBRyxFQUFFLEtBQUssR0FBRyxFQUM5RSxRQUFRLEVBQUUsTUFBTSxHQUFHLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLEdBQUcsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFDLENBQUMsS0FBSyxNQUFNLENBQUMsRUFDaEUsVUFBVSxDQUFDLEVBQUUsY0FBYyxHQUFHLENBQUMsQ0FBQyxJQUFJLEVBQUUsQ0FBQyxFQUFFLEdBQUcsSUFBSSxFQUFFLFVBQVUsQ0FBQyxDQUFDLENBQUMsS0FBSyxPQUFPLENBQUMsY0FBYyxDQUFDLEdBQUcsY0FBYyxDQUFDLEVBQzdHLGVBQWUsQ0FBQyxFQUFFLENBQUMsSUFBSSxFQUFFLENBQUMsRUFBRSxXQUFXLEVBQUUsT0FBTyxDQUFDLFVBQVUsQ0FBQyxDQUFDLENBQUMsQ0FBQyxLQUFLLGNBQWMsR0FDakYsYUFBYSxDQUFDLENBQUMsRUFBRSxDQUFDLENBQUMsQ0E2Q3JCO0FBRUQ7Ozs7Ozs7Ozs7R0FVRztBQUNILHdCQUFnQixrQkFBa0IsQ0FBQyxDQUFDLFNBQVMsQ0FBQyxHQUFHLElBQUksRUFBRSxHQUFHLEVBQUUsS0FBSyxHQUFHLEVBQ2xFLE1BQU0sRUFBRSxNQUFNLEVBQ2QsUUFBUSxFQUFFLE1BQU0sRUFDaEIsVUFBVSxFQUFFLGNBQWMsRUFDMUIsUUFBUSxFQUFFLENBQUMsR0FDVixDQUFDLENBa0JIO0FBRUQsd0JBQWdCLFNBQVMsQ0FBQyxDQUFDLFNBQVMsR0FBRyxFQUFFLEVBQUUsQ0FBQyxFQUMxQyxNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sRUFDdkIsUUFBUSxFQUFFLE1BQU0sRUFDaEIsUUFBUSxFQUFFLENBQUMsSUFBSSxFQUFFLElBQUksRUFBRSxHQUFHLElBQUksRUFBRSxDQUFDLEtBQUssT0FBTyxDQUFDLENBQUMsQ0FBQyxHQUMvQyxDQUFDLEdBQUcsSUFBSSxFQUFFLENBQUMsS0FBSyxPQUFPLENBQUMsQ0FBQyxDQUFDLENBeUI1QjtBQUVEOzs7R0FHRztBQUNILHdCQUFnQixVQUFVLENBQUMsQ0FBQyxFQUMxQixNQUFNLEVBQUUsTUFBTSxHQUFHLE1BQU0sRUFDdkIsUUFBUSxFQUFFLE1BQU0sRUFDaEIsUUFBUSxFQUFFLENBQUMsSUFBSSxFQUFFLElBQUksS0FBSyxPQUFPLENBQUMsQ0FBQyxDQUFDLEdBQ25DLE9BQU8sQ0FBQyxDQUFDLENBQUMsQ0FpQloifQ==