@aztec/telemetry-client 4.0.0-nightly.20250907 → 4.0.0-nightly.20260107

package/src/metrics.ts

@@ -34,6 +34,9 @@ export const CIRCUIT_SIZE = 'aztec.circuit.size';
 
 export const MEMPOOL_TX_COUNT = 'aztec.mempool.tx_count';
 export const MEMPOOL_TX_SIZE = 'aztec.mempool.tx_size';
+export const MEMPOOL_TX_ADDED_COUNT = 'aztec.mempool.tx_added_count';
+export const MEMPOOL_TX_MINED_DELAY = 'aztec.mempool.tx_mined_delay';
+
 export const DB_NUM_ITEMS = 'aztec.db.num_items';
 export const DB_MAP_SIZE = 'aztec.db.map_size';
 export const DB_PHYSICAL_FILE_SIZE = 'aztec.db.physical_file_size';
@@ -41,6 +44,8 @@ export const DB_USED_SIZE = 'aztec.db.used_size';
 
 export const MEMPOOL_ATTESTATIONS_COUNT = 'aztec.mempool.attestations_count';
 export const MEMPOOL_ATTESTATIONS_SIZE = 'aztec.mempool.attestations_size';
+export const MEMPOOL_ATTESTATIONS_ADDED_COUNT = 'aztec.mempool.attestations_added_count';
+export const MEMPOOL_ATTESTATIONS_MINED_DELAY = 'aztec.mempool.attestations_mined_delay';
 
 export const ARCHIVER_L1_BLOCK_HEIGHT = 'aztec.archiver.l1_block_height';
 export const ARCHIVER_BLOCK_HEIGHT = 'aztec.archiver.block_height';
@@ -60,27 +65,50 @@ export const ARCHIVER_PRUNE_COUNT = 'aztec.archiver.prune_count';
 
 export const ARCHIVER_TOTAL_TXS = 'aztec.archiver.tx_count';
 
+export const ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT = 'aztec.archiver.block_proposal_tx_target_count';
+
 export const NODE_RECEIVE_TX_DURATION = 'aztec.node.receive_tx.duration';
 export const NODE_RECEIVE_TX_COUNT = 'aztec.node.receive_tx.count';
 
+export const NODE_SNAPSHOT_DURATION = 'aztec.node.snapshot_duration';
+export const NODE_SNAPSHOT_ERROR_COUNT = 'aztec.node.snapshot_error_count';
+
 export const SEQUENCER_STATE_TRANSITION_BUFFER_DURATION = 'aztec.sequencer.state_transition_buffer.duration';
 export const SEQUENCER_BLOCK_BUILD_DURATION = 'aztec.sequencer.block.build_duration';
 export const SEQUENCER_BLOCK_BUILD_MANA_PER_SECOND = 'aztec.sequencer.block.build_mana_per_second';
 export const SEQUENCER_BLOCK_COUNT = 'aztec.sequencer.block.count';
-export const SEQUENCER_CURRENT_STATE = 'aztec.sequencer.current.state';
-export const SEQUENCER_CURRENT_BLOCK_NUMBER = 'aztec.sequencer.current.block_number';
-export const SEQUENCER_CURRENT_BLOCK_SIZE = 'aztec.sequencer.current.block_size';
-export const SEQUENCER_BLOCK_BUILD_INSERTION_TIME = 'aztec.sequencer.block_builder_tree_insertion_duration';
 export const SEQUENCER_CURRENT_BLOCK_REWARDS = 'aztec.sequencer.current_block_rewards';
 export const SEQUENCER_SLOT_COUNT = 'aztec.sequencer.slot.total_count';
 export const SEQUENCER_FILLED_SLOT_COUNT = 'aztec.sequencer.slot.filled_count';
-export const SEQUENCER_MISSED_SLOT_COUNT = 'aztec.sequencer.slot.missed_count';
+
+export const SEQUENCER_BLOCK_ATTESTATION_DELAY = 'aztec.sequencer.block.attestation_delay';
 
 export const SEQUENCER_COLLECTED_ATTESTATIONS_COUNT = 'aztec.sequencer.attestations.collected_count';
 export const SEQUENCER_REQUIRED_ATTESTATIONS_COUNT = 'aztec.sequencer.attestations.required_count';
 export const SEQUENCER_COLLECT_ATTESTATIONS_DURATION = 'aztec.sequencer.attestations.collect_duration';
 export const SEQUENCER_COLLECT_ATTESTATIONS_TIME_ALLOWANCE = 'aztec.sequencer.attestations.collect_allowance';
 
+export const SEQUENCER_BLOCK_PROPOSAL_FAILED_COUNT = 'aztec.sequencer.block.proposal_failed_count';
+export const SEQUENCER_BLOCK_PROPOSAL_SUCCESS_COUNT = 'aztec.sequencer.block.proposal_success_count';
+export const SEQUENCER_BLOCK_PROPOSAL_PRECHECK_FAILED_COUNT = 'aztec.sequencer.block.proposal_precheck_failed_count';
+export const SEQUENCER_SLASHING_ATTEMPTS_COUNT = 'aztec.sequencer.slashing.attempts_count';
+export const SEQUENCER_CHECKPOINT_SUCCESS_COUNT = 'aztec.sequencer.checkpoint.success_count';
+
+// Fisherman fee analysis metrics
+export const FISHERMAN_FEE_ANALYSIS_WOULD_BE_INCLUDED = 'aztec.fisherman.fee_analysis.would_be_included';
+export const FISHERMAN_FEE_ANALYSIS_TIME_BEFORE_BLOCK = 'aztec.fisherman.fee_analysis.time_before_block';
+export const FISHERMAN_FEE_ANALYSIS_PENDING_BLOB_TX_COUNT = 'aztec.fisherman.fee_analysis.pending_blob_tx_count';
+export const FISHERMAN_FEE_ANALYSIS_INCLUDED_BLOB_TX_COUNT = 'aztec.fisherman.fee_analysis.included_blob_tx_count';
+export const FISHERMAN_FEE_ANALYSIS_CALCULATED_PRIORITY_FEE = 'aztec.fisherman.fee_analysis.calculated_priority_fee';
+export const FISHERMAN_FEE_ANALYSIS_PRIORITY_FEE_DELTA = 'aztec.fisherman.fee_analysis.priority_fee_delta';
+export const FISHERMAN_FEE_ANALYSIS_ESTIMATED_COST = 'aztec.fisherman.fee_analysis.estimated_cost';
+export const FISHERMAN_FEE_ANALYSIS_ESTIMATED_OVERPAYMENT = 'aztec.fisherman.fee_analysis.estimated_overpayment';
+export const FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_PRIORITY_FEE =
+  'aztec.fisherman.fee_analysis.mined_blob_tx_priority_fee';
+export const FISHERMAN_FEE_ANALYSIS_MINED_BLOB_TX_TOTAL_COST = 'aztec.fisherman.fee_analysis.mined_blob_tx_total_cost';
+
+export const VALIDATOR_INVALID_ATTESTATION_RECEIVED_COUNT = 'aztec.validator.invalid_attestation_received_count';
+
 export const L1_PUBLISHER_GAS_PRICE = 'aztec.l1_publisher.gas_price';
 export const L1_PUBLISHER_TX_COUNT = 'aztec.l1_publisher.tx_count';
 export const L1_PUBLISHER_TX_DURATION = 'aztec.l1_publisher.tx_duration';
@@ -101,9 +129,22 @@ export const L1_BALANCE_ETH = 'aztec.l1.balance';
 export const L1_GAS_PRICE_WEI = 'aztec.l1.gas_price';
 export const L1_BLOB_BASE_FEE_WEI = 'aztec.l1.blob_base_fee';
 
+export const L1_TX_MINED_DURATION = 'aztec.l1_tx.mined_duration';
+export const L1_TX_MINED_COUNT = 'aztec.l1_tx.mined_count';
+export const L1_TX_REVERTED_COUNT = 'aztec.l1_tx.reverted_count';
+export const L1_TX_CANCELLED_COUNT = 'aztec.l1_tx.cancelled_count';
+export const L1_TX_NOT_MINED_COUNT = 'aztec.l1_tx.not_mined_count';
+export const L1_TX_ATTEMPTS_UNTIL_MINED = 'aztec.l1_tx.attempts_until_mined';
+export const L1_TX_MAX_PRIORITY_FEE = 'aztec.l1_tx.max_priority_fee';
+export const L1_TX_MAX_FEE = 'aztec.l1_tx.max_fee';
+export const L1_TX_BLOB_FEE = 'aztec.l1_tx.blob_fee';
+
 export const PEER_MANAGER_GOODBYES_SENT = 'aztec.peer_manager.goodbyes_sent';
 export const PEER_MANAGER_GOODBYES_RECEIVED = 'aztec.peer_manager.goodbyes_received';
 export const PEER_MANAGER_PEER_COUNT = 'aztec.peer_manager.peer_count';
+export const PEER_MANAGER_LOW_SCORE_DISCONNECTS = 'aztec.peer_manager.low_score_disconnects';
+export const PEER_MANAGER_PEER_CONNECTION_DURATION = 'aztec.peer_manager.peer_connection_duration';
+export const P2P_PEER_STATE_COUNT = 'aztec.p2p.peer_state_count';
 
 export const P2P_REQ_RESP_SENT_REQUESTS = 'aztec.p2p.req_resp.sent_requests';
 export const P2P_REQ_RESP_RECEIVED_REQUESTS = 'aztec.p2p.req_resp.received_requests';
@@ -113,6 +154,7 @@ export const P2P_REQ_RESP_FAILED_INBOUND_REQUESTS = 'aztec.p2p.req_resp.failed_i
 export const P2P_GOSSIP_MESSAGE_VALIDATION_DURATION = 'aztec.p2p.gossip.message_validation_duration';
 export const P2P_GOSSIP_MESSAGE_PREVALIDATION_COUNT = 'aztec.p2p.gossip.message_validation_count';
 export const P2P_GOSSIP_MESSAGE_LATENCY = 'aztec.p2p.gossip.message_latency';
+export const P2P_GOSSIP_TX_RECEIVED_COUNT = 'aztec.p2p.gossip.tx_received_count';
 
 export const P2P_GOSSIP_AGG_MESSAGE_LATENCY_MIN = 'aztec.p2p.gossip.agg_message_latency_min';
 export const P2P_GOSSIP_AGG_MESSAGE_LATENCY_MAX = 'aztec.p2p.gossip.agg_message_latency_max';
@@ -151,8 +193,6 @@ export const PUBLIC_EXECUTOR_SIMULATION_BYTECODE_SIZE = 'aztec.public_executor.s
 export const PROVING_ORCHESTRATOR_BASE_ROLLUP_INPUTS_DURATION =
   'aztec.proving_orchestrator.base_rollup.inputs_duration';
 
-export const PROVING_ORCHESTRATOR_AVM_FALLBACK_COUNT = 'aztec.proving_orchestrator.avm.fallback_count';
-
 export const PROVING_QUEUE_JOB_SIZE = 'aztec.proving_queue.job_size';
 export const PROVING_QUEUE_SIZE = 'aztec.proving_queue.size';
 export const PROVING_QUEUE_TOTAL_JOBS = 'aztec.proving_queue.enqueued_jobs_count';
@@ -172,6 +212,7 @@ export const PROVING_AGENT_IDLE = 'aztec.proving_queue.agent.idle';
 
 export const PROVER_NODE_EXECUTION_DURATION = 'aztec.prover_node.execution.duration';
 export const PROVER_NODE_JOB_DURATION = 'aztec.prover_node.job_duration';
+export const PROVER_NODE_JOB_CHECKPOINTS = 'aztec.prover_node.job_checkpoints';
 export const PROVER_NODE_JOB_BLOCKS = 'aztec.prover_node.job_blocks';
 export const PROVER_NODE_JOB_TRANSACTIONS = 'aztec.prover_node.job_transactions';
 export const PROVER_NODE_REWARDS_TOTAL = 'aztec.prover_node.rewards_total';
@@ -199,8 +240,9 @@ export const VALIDATOR_RE_EXECUTION_MANA = 'aztec.validator.re_execution_mana';
 export const VALIDATOR_RE_EXECUTION_TX_COUNT = 'aztec.validator.re_execution_tx_count';
 
 export const VALIDATOR_FAILED_REEXECUTION_COUNT = 'aztec.validator.failed_reexecution_count';
-export const VALIDATOR_ATTESTATION_COUNT = 'aztec.validator.attestation_count';
-export const VALIDATOR_FAILED_ATTESTATION_COUNT = 'aztec.validator.failed_attestation_count';
+export const VALIDATOR_ATTESTATION_SUCCESS_COUNT = 'aztec.validator.attestation_success_count';
+export const VALIDATOR_ATTESTATION_FAILED_BAD_PROPOSAL_COUNT = 'aztec.validator.attestation_failed_bad_proposal_count';
+export const VALIDATOR_ATTESTATION_FAILED_NODE_ISSUE_COUNT = 'aztec.validator.attestation_failed_node_issue_count';
 
 export const NODEJS_EVENT_LOOP_DELAY_MIN = 'nodejs.eventloop.delay.min';
 export const NODEJS_EVENT_LOOP_DELAY_MEAN = 'nodejs.eventloop.delay.mean';
@@ -222,6 +264,8 @@ export const TX_PROVIDER_TXS_FROM_PROPOSALS_COUNT = 'aztec.tx_collector.txs_from
 export const TX_PROVIDER_TXS_FROM_MEMPOOL_COUNT = 'aztec.tx_collector.txs_from_mempool_count';
 export const TX_PROVIDER_TXS_FROM_P2P_COUNT = 'aztec.tx_collector.txs_from_p2p_count';
 export const TX_PROVIDER_MISSING_TXS_COUNT = 'aztec.tx_collector.missing_txs_count';
+export const TX_PROVIDER_P2P_TXS_REQUESTED_FRACTION = 'aztec.tx_collector.txs_requested_fraction';
+export const TX_PROVIDER_P2P_TXS_REQUEST_DELAY = 'aztec.tx_collector.txs_requested_delay';
 
 export const TX_COLLECTOR_COUNT = 'aztec.tx_collector.tx_count';
 export const TX_COLLECTOR_DURATION_PER_REQUEST = 'aztec.tx_collector.duration_per_request';

package/src/nodejs_metrics_monitor.ts

@@ -1,3 +1,4 @@
+import type { Observable } from '@opentelemetry/api';
 import { type EventLoopUtilization, type IntervalHistogram, monitorEventLoopDelay, performance } from 'node:perf_hooks';
 
 import * as Attributes from './attributes.js';
@@ -142,17 +143,26 @@ export class NodejsMetricsMonitor {
     // - https://youtu.be/WetXnEPraYM
     obs.observe(this.eventLoopUilization, delta.utilization);
 
-    this.eventLoopTime.add(Math.floor(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
-    this.eventLoopTime.add(Math.floor(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
+    this.eventLoopTime.add(Math.trunc(delta.idle), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'idle' });
+    this.eventLoopTime.add(Math.trunc(delta.active), { [Attributes.NODEJS_EVENT_LOOP_STATE]: 'active' });
 
-    obs.observe(this.eventLoopDelayGauges.min, Math.floor(this.eventLoopDelay.min));
-    obs.observe(this.eventLoopDelayGauges.mean, Math.floor(this.eventLoopDelay.mean));
-    obs.observe(this.eventLoopDelayGauges.max, Math.floor(this.eventLoopDelay.max));
-    obs.observe(this.eventLoopDelayGauges.stddev, Math.floor(this.eventLoopDelay.stddev));
-    obs.observe(this.eventLoopDelayGauges.p50, Math.floor(this.eventLoopDelay.percentile(50)));
-    obs.observe(this.eventLoopDelayGauges.p90, Math.floor(this.eventLoopDelay.percentile(90)));
-    obs.observe(this.eventLoopDelayGauges.p99, Math.floor(this.eventLoopDelay.percentile(99)));
+    safeObserveInt(obs, this.eventLoopDelayGauges.min, this.eventLoopDelay.min);
+    safeObserveInt(obs, this.eventLoopDelayGauges.mean, this.eventLoopDelay.mean);
+    safeObserveInt(obs, this.eventLoopDelayGauges.max, this.eventLoopDelay.max);
+    safeObserveInt(obs, this.eventLoopDelayGauges.stddev, this.eventLoopDelay.stddev);
+    safeObserveInt(obs, this.eventLoopDelayGauges.p50, this.eventLoopDelay.percentile(50));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p90, this.eventLoopDelay.percentile(90));
+    safeObserveInt(obs, this.eventLoopDelayGauges.p99, this.eventLoopDelay.percentile(99));
 
     this.eventLoopDelay.reset();
   };
 }
+
+function safeObserveInt(observer: BatchObservableResult, metric: Observable, value: number, attrs?: object) {
+  // discard NaN, Infinity, -Infinity
+  if (!Number.isFinite(value)) {
+    return;
+  }
+
+  observer.observe(metric, Math.trunc(value), attrs);
+}

package/src/otel.ts

@@ -283,11 +283,25 @@ export class OpenTelemetryClient implements TelemetryClient {
 
       const exporters: PeriodicExportingMetricReaderOptions[] = [];
       if (config.metricsCollectorUrl) {
+        // Default to a blacklist that is empty (allow all metrics)
+        let filter: string[] = [];
+        let mode: 'allow' | 'deny' = 'deny';
+        if (config.otelExcludeMetrics.length > 0) {
+          // Implement a blacklist as specified in config
+          log.info(`Excluding metrics from export: ${config.otelExcludeMetrics}`);
+          filter = config.otelExcludeMetrics;
+          mode = 'deny';
+        } else if (config.otelIncludeMetrics.length > 0) {
+          // Implement a whitelist as specified in config
+          log.info(`Including only specified metrics for export: ${config.otelIncludeMetrics}`);
+          filter = config.otelIncludeMetrics;
+          mode = 'allow';
+        }
         exporters.push({
           exporter: new OtelFilterMetricExporter(
             new OTLPMetricExporter({ url: config.metricsCollectorUrl.href }),
-            config.otelExcludeMetrics,
-            'deny',
+            filter,
+            mode,
           ),
           exportTimeoutMillis: config.otelExportTimeoutMs,
           exportIntervalMillis: config.otelCollectIntervalMs,

package/src/otel_resource.ts

@@ -37,6 +37,8 @@ const aztecNetworkDetectorSync: DetectorSync = {
       role = 'prover-node';
     } else if (process.argv.includes('--node')) {
       role = 'node';
+    } else if (process.argv.includes('--p2p-bootstrap')) {
+      role = 'bootnode';
     }
     const aztecAttributes = {
       // this gets overwritten by OTEL_RESOURCE_ATTRIBUTES (if set)

package/src/start.ts

@@ -2,7 +2,6 @@ import { createLogger } from '@aztec/foundation/log';
 
 import type { TelemetryClientConfig } from './config.js';
 import { NoopTelemetryClient } from './noop.js';
-import { OpenTelemetryClient } from './otel.js';
 import type { TelemetryClient } from './telemetry.js';
 
 export * from './config.js';
@@ -10,7 +9,7 @@ export * from './config.js';
 let initialized = false;
 let telemetry: TelemetryClient = new NoopTelemetryClient();
 
-export function initTelemetryClient(config: TelemetryClientConfig): TelemetryClient {
+export async function initTelemetryClient(config: TelemetryClientConfig): Promise<TelemetryClient> {
   const log = createLogger('telemetry:client');
   if (initialized) {
     log.warn('Telemetry client has already been initialized once');
@@ -19,6 +18,8 @@ export function initTelemetryClient(config: TelemetryClientConfig): TelemetryCli
 
   if (config.metricsCollectorUrl || config.publicMetricsCollectorUrl) {
     log.info(`Using OpenTelemetry client with custom collector`);
+    // Lazy load OpenTelemetry to avoid loading heavy deps at startup
+    const { OpenTelemetryClient } = await import('./otel.js');
     telemetry = OpenTelemetryClient.createAndStart(config, log);
   } else {
     log.info('Using NoopTelemetryClient');

package/src/telemetry.ts

@@ -204,24 +204,36 @@ export function trackSpan<T extends Traceable, F extends (...args: any[]) => any
       // "active" means the span will be alive for the duration of the function execution
       // and if any other spans are started during the execution of originalMethod, they will be children of this span
       // behind the scenes this uses AsyncLocalStorage https://nodejs.org/dist/latest-v22.x/docs/api/async_context.html
-      return this.tracer.startActiveSpan(name, async (span: Span) => {
-        span.setAttributes(currentAttrs ?? {});
+      return this.tracer.startActiveSpan(
+        name,
+        {
+          attributes: currentAttrs,
+        },
+        async (span: Span) => {
+          try {
+            const res = await originalMethod.call(this, ...args);
+            const extraAttrs = extraAttributes?.call(this, res);
+            span.setAttributes(extraAttrs ?? {});
+            span.setStatus({
+              code: SpanStatusCode.OK,
+            });
+            return res;
+          } catch (err) {
+            span.setStatus({
+              code: SpanStatusCode.ERROR,
+              message: String(err),
+            });
 
-        try {
-          const res = await originalMethod.call(this, ...args);
-          const extraAttrs = extraAttributes?.call(this, res);
-          span.setAttributes(extraAttrs ?? {});
-          return res;
-        } catch (err) {
-          span.setStatus({
-            code: SpanStatusCode.ERROR,
-            message: String(err),
-          });
-          throw err;
-        } finally {
-          span.end();
-        }
-      });
+            if (typeof err === 'string' || (err && err instanceof Error)) {
+              span.recordException(err);
+            }
+
+            throw err;
+          } finally {
+            span.end();
+          }
+        },
+      );
     } as F;
   };
 }

package/src/wrappers/l2_block_stream.ts

@@ -1,3 +1,4 @@
+import { BlockNumber } from '@aztec/foundation/branded-types';
 import { createLogger } from '@aztec/foundation/log';
 import {
   type L2BlockSource,
@@ -20,7 +21,7 @@ export class TraceableL2BlockStream extends L2BlockStream implements Traceable {
       proven?: boolean;
       pollIntervalMS?: number;
       batchSize?: number;
-      startingBlock?: number;
+      startingBlock?: BlockNumber;
     } = {},
   ) {
     super(l2BlockSource, localData, handler, log, opts);