@lodestar/beacon-node 1.47.0-dev.ec596194e2 → 1.47.0-dev.f0f26cd2b6

package/lib/metrics/metrics/lodestar.js

@@ -37,15 +37,15 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
             }),
             jobTime: register.histogram({
                 name: "lodestar_gossip_validation_queue_job_time_seconds",
-                help: "Time to process gossip validation queue job in seconds",
+                help: "Time to process gossip validation queue job in seconds (accepted messages only)",
                 labelNames: ["topic"],
-                buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+                buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
             }),
             jobWaitTime: register.histogram({
                 name: "lodestar_gossip_validation_queue_job_wait_time_seconds",
                 help: "Time from job added to the queue to starting the job in seconds",
                 labelNames: ["topic"],
-                buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+                buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
             }),
             concurrency: register.gauge({
                 name: "lodestar_gossip_validation_queue_concurrency",
@@ -61,6 +61,7 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
             queueTime: register.histogram({
                 name: "lodestar_gossip_validation_queue_time_seconds",
                 help: "Total time an item stays in queue until it is processed in seconds",
+                labelNames: ["topic"],
                 buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
             }),
         },
@@ -342,85 +343,107 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
         }),
         // BLS verifier thread pool and queue
         bls: {
-            aggregatedPubkeys: register.gauge({
+            aggregatedPubkeys: register.counter({
                 name: "lodestar_bls_aggregated_pubkeys_total",
-                help: "Total aggregated pubkeys for BLS validation",
+                help: "Total validator public keys used as inputs to aggregate signature sets",
+            }),
+            pubkeyCacheSize: register.gauge({
+                name: "lodestar_bls_pubkey_cache_size",
+                help: "Current number of validator public keys in the native cache",
+            }),
+            pubkeyCacheCapacity: register.gauge({
+                name: "lodestar_bls_pubkey_cache_capacity",
+                help: "Number of validator public keys the native cache can hold without growing",
             }),
         },
         blsThreadPool: {
-            jobsWorkerTime: register.gauge({
+            jobsWorkerTime: register.counter({
                 name: "lodestar_bls_thread_pool_time_seconds_sum",
-                help: "Total time spent verifying signature sets measured on the worker",
+                help: "Cumulative wall time spent processing BLS dispatch groups by worker",
                 labelNames: ["workerId"],
             }),
-            successJobsSignatureSetsCount: register.gauge({
+            successJobsSignatureSetsCount: register.counter({
                 name: "lodestar_bls_thread_pool_success_jobs_signature_sets_count",
-                help: "Count of total verified signature sets",
+                help: "Count of signature sets completed without an operational error",
             }),
-            errorAggregateSignatureSetsCount: register.gauge({
-                name: "lodestar_bls_thread_pool_error_aggregate_signature_sets_count",
-                help: "Count of error when aggregating pubkeys or signatures",
-                labelNames: ["type"],
-            }),
-            errorJobsSignatureSetsCount: register.gauge({
+            errorJobsSignatureSetsCount: register.counter({
                 name: "lodestar_bls_thread_pool_error_jobs_signature_sets_count",
-                help: "Count of total error-ed signature sets",
+                help: "Count of signature sets that ended in an input, binding, or worker error",
             }),
             jobWaitTime: register.histogram({
                 name: "lodestar_bls_thread_pool_queue_job_wait_time_seconds",
-                help: "Time from job added to the queue to starting the job in seconds",
-                buckets: [0.01, 0.02, 0.1, 0.3, 0.5, 1],
+                help: "Time from adding a BLS job until it is selected for a worker dispatch group",
+                labelNames: ["type"],
+                buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+            }),
+            jobDuration: register.histogram({
+                name: "lodestar_bls_thread_pool_job_duration_seconds",
+                help: "End-to-end duration of a BLS job from submission through completion",
+                labelNames: ["type"],
+                buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.2, 0.5, 1, 2],
+            }),
+            jobResults: register.counter({
+                name: "lodestar_bls_thread_pool_job_results_total",
+                help: "Count of completed BLS jobs by cryptographic or operational outcome",
+                labelNames: ["type", "outcome"],
             }),
             queueLength: register.gauge({
                 name: "lodestar_bls_thread_pool_queue_length",
-                help: "Count of total block processor queue length",
+                help: "Count of ready BLS jobs waiting for a worker",
+            }),
+            bufferedQueueLength: register.gauge({
+                name: "lodestar_bls_thread_pool_buffered_queue_length",
+                help: "Count of BLS jobs waiting in the intentional batching buffer",
             }),
             workersBusy: register.gauge({
                 name: "lodestar_bls_thread_pool_workers_busy",
                 help: "Count of current busy workers",
             }),
-            totalJobsGroupsStarted: register.gauge({
+            totalJobsGroupsStarted: register.counter({
                 name: "lodestar_bls_thread_pool_job_groups_started_total",
-                help: "Count of total jobs groups started in bls thread pool, job groups include +1 jobs",
+                help: "Count of worker dispatch groups started by the BLS thread pool",
             }),
-            totalJobsStarted: register.gauge({
+            totalJobsStarted: register.counter({
                 name: "lodestar_bls_thread_pool_jobs_started_total",
-                help: "Count of total jobs started in bls thread pool, jobs include +1 signature sets",
+                help: "Count of jobs started by the BLS thread pool",
                 labelNames: ["type"],
             }),
-            totalSigSetsStarted: register.gauge({
+            totalSigSetsStarted: register.counter({
                 name: "lodestar_bls_thread_pool_sig_sets_started_total",
-                help: "Count of total signature sets started in bls thread pool, sig sets include 1 pk, msg, sig",
+                help: "Count of signature sets started by the BLS thread pool",
                 labelNames: ["type"],
             }),
             // Re-verifying a batch means doing double work. This number must be very low or it can be a waste of CPU resources
-            batchRetries: register.gauge({
+            batchRetries: register.counter({
                 name: "lodestar_bls_thread_pool_batch_retries_total",
-                help: "Count of total batches that failed and had to be verified again.",
+                help: "Count of combined batches that failed and required per-job fallback",
             }),
             // To count how many sigs are being validated with the optimization of batching them
-            batchSigsSuccess: register.gauge({
+            batchSigsSuccess: register.counter({
                 name: "lodestar_bls_thread_pool_batch_sigs_success_total",
-                help: "Count of total batches that failed and had to be verified again.",
+                help: "Count of signature sets accepted by combined batch verification without fallback",
             }),
-            sameMessageRetryJobs: register.gauge({
-                name: "lodestar_bls_thread_pool_same_message_jobs_retries_total",
-                help: "Count of total same message jobs that failed and had to be verified again.",
+            bufferFlushes: register.counter({
+                name: "lodestar_bls_thread_pool_buffer_flushes_total",
+                help: "Count of batching-buffer flushes by trigger",
+                labelNames: ["reason"],
             }),
-            sameMessageRetrySets: register.gauge({
-                name: "lodestar_bls_thread_pool_same_message_sets_retries_total",
-                help: "Count of total same message sets that failed and had to be verified again.",
+            bufferedSignatureSets: register.histogram({
+                name: "lodestar_bls_thread_pool_buffered_signature_sets",
+                help: "Count of signature sets included in each batching-buffer flush",
+                buckets: [1, 8, 16, 24, 28, 32, 36, 40, 44, 48, 64, 128],
             }),
             // To measure the time cost of main thread <-> worker message passing
             latencyToWorker: register.histogram({
                 name: "lodestar_bls_thread_pool_latency_to_worker",
                 help: "Time from sending the job to the worker and the worker receiving it",
-                buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+                // The dashboard treats 20ms as slow; values above 100ms only need severe-stall classification.
+                buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
             }),
             latencyFromWorker: register.histogram({
                 name: "lodestar_bls_thread_pool_latency_from_worker",
                 help: "Time from the worker sending the result and the main thread receiving it",
-                buckets: [0.001, 0.003, 0.01, 0.03, 0.1],
+                buckets: [0.00025, 0.001, 0.005, 0.02, 0.1],
             }),
             mainThreadDurationInThreadPool: register.histogram({
                 name: "lodestar_bls_thread_pool_main_thread_time_seconds",
@@ -430,32 +453,40 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
             }),
             timePerSigSet: register.histogram({
                 name: "lodestar_bls_worker_thread_time_per_sigset_seconds",
-                help: "Time to verify each sigset with worker thread mode",
+                help: "Average BLS worker dispatch wall time per original signature set",
                 // Time per sig ~0.9ms on good machines
-                buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3],
-            }),
-            totalSigSets: register.gauge({
+                buckets: [0.5e-3, 0.75e-3, 1e-3, 1.5e-3, 2e-3, 5e-3, 10e-3, 20e-3, 50e-3, 100e-3],
+            }),
+            workRequestPreparationDuration: register.histogram({
+                name: "lodestar_bls_thread_pool_work_request_preparation_duration_seconds",
+                help: "Main-thread time spent converting one worker dispatch group to binding inputs",
+                // Normal requests take microseconds; the dashboard treats 1ms as a main-thread stall.
+                buckets: [0.000005, 0.000025, 0.0001, 0.0005, 0.001, 0.005, 0.025],
+            }),
+            verificationCallDuration: register.histogram({
+                name: "lodestar_bls_thread_pool_verification_call_duration_seconds",
+                help: "Duration of one worker-side lodestar-z verification binding call, including input conversion",
+                labelNames: ["operation"],
+                // Verification ranges from sub-millisecond calls to large batches; above 500ms is uniformly severe.
+                buckets: [0.00025, 0.001, 0.005, 0.02, 0.1, 0.5],
+            }),
+            verificationCallSignatureSets: register.counter({
+                name: "lodestar_bls_thread_pool_verification_call_signature_sets_total",
+                help: "Count of signature sets passed to worker-side lodestar-z verification binding calls",
+                labelNames: ["operation"],
+            }),
+            totalSigSets: register.counter({
                 name: "lodestar_bls_thread_pool_sig_sets_total",
                 help: "Count of total signature sets",
             }),
-            prioritizedSigSets: register.gauge({
+            prioritizedSigSets: register.counter({
                 name: "lodestar_bls_thread_pool_prioritized_sig_sets_total",
                 help: "Count of total prioritized signature sets",
             }),
-            batchableSigSets: register.gauge({
+            batchableSigSets: register.counter({
                 name: "lodestar_bls_thread_pool_batchable_sig_sets_total",
                 help: "Count of total batchable signature sets",
             }),
-            aggregateWithRandomnessAsyncDuration: register.histogram({
-                name: "lodestar_bls_thread_pool_aggregate_with_randomness_async_time_seconds",
-                help: "Total time performing aggregateWithRandomness async",
-                buckets: [0.001, 0.005, 0.01, 0.1, 0.3],
-            }),
-            pubkeysAggregationMainThreadDuration: register.histogram({
-                name: "lodestar_bls_thread_pool_pubkeys_aggregation_main_thread_time_seconds",
-                help: "Total time spent aggregating pubkeys on main thread",
-                buckets: [0.001, 0.005, 0.01, 0.1],
-            }),
         },
         // BLS time on single thread mode
         blsSingleThread: {
@@ -585,7 +616,13 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
             }),
             removedBlocks: register.gauge({
                 name: "lodestar_sync_unknown_block_removed_blocks_total",
-                help: "Total number of removed bad blocks in UnknownBlockSync",
+                help: "Total pending blocks dropped from BlockInputSync without completing, by reason",
+                labelNames: ["reason"],
+            }),
+            removedPayloads: register.gauge({
+                name: "lodestar_sync_unknown_block_removed_payloads_total",
+                help: "Total pending payloads dropped from BlockInputSync without completing, by reason",
+                labelNames: ["reason"],
             }),
             elapsedTimeTillReceived: register.histogram({
                 name: "lodestar_sync_unknown_block_elapsed_time_till_received",
@@ -810,6 +847,11 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 help: "Count of errors, by error type, while processing blocks",
                 labelNames: ["error"],
             }),
+            preStateSource: register.counter({
+                name: "lodestar_gossip_block_validation_pre_state_source_total",
+                help: "Source of the pre-state used for gossip block validation",
+                labelNames: ["source"],
+            }),
         },
         gossipBlob: {
             recvToValidation: register.histogram({
@@ -841,6 +883,22 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 labelNames: ["error"],
             }),
         },
+        gossipExecutionPayloadBid: {
+            elapsedTimeTillReceived: register.histogram({
+                name: "lodestar_gossip_execution_payload_bid_elapsed_time_till_received_seconds",
+                help: "Time elapsed between the bid's slot start and the time the execution payload bid was received (negative = received before its slot, broadcast in the previous slot; positive = received during its own slot, late)",
+                labelNames: ["source"],
+                buckets: [-6, -3, -1, 0, 3],
+            }),
+        },
+        gossipPayloadAttestationMessage: {
+            elapsedTimeTillReceived: register.histogram({
+                name: "lodestar_gossip_payload_attestation_message_elapsed_time_till_received_seconds",
+                help: "Time elapsed between slot time and the time the payload attestation message was received",
+                labelNames: ["source"],
+                buckets: [3, 6, 9, 12],
+            }),
+        },
         // recovery in the case of specific blob rows required
         recoverBlobSidecars: {
             blobsReconstructed: register.counter({
@@ -1107,6 +1165,12 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 name: "lodestar_oppool_attester_slashing_pool_size",
                 help: "Current size of the AttesterSlashingPool",
             }),
+            deferredVoluntaryExitPool: {
+                size: register.gauge({
+                    name: "lodestar_oppool_deferred_voluntary_exit_pool_size",
+                    help: "Current size of the DeferredVoluntaryExitPool",
+                }),
+            },
             proposerSlashingPoolSize: register.gauge({
                 name: "lodestar_oppool_proposer_slashing_pool_size",
                 help: "Current size of the ProposerSlashingPool",
@@ -1217,6 +1281,11 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                     help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
                     labelNames: ["insertOutcome"],
                 }),
+                apiValidationTime: register.histogram({
+                    name: "lodestar_api_execution_payload_bid_validation_time_seconds",
+                    help: "Time elapsed for signed execution payload bid validation - api path",
+                    buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.5],
+                }),
             },
         },
         chain: {
@@ -1305,6 +1374,16 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 help: "Checkpoint state cache size",
                 labelNames: ["type"],
             }),
+            persistentTierEpochs: register.gauge({
+                name: "lodestar_cp_state_cache_persistent_tier_epochs",
+                help: "Number of epoch keys retained in each on-disk checkpoint state retention tier",
+                labelNames: ["tier"],
+            }),
+            persistentTierStates: register.gauge({
+                name: "lodestar_cp_state_cache_persistent_tier_states",
+                help: "Number of persisted checkpoint states retained in each on-disk retention tier",
+                labelNames: ["tier"],
+            }),
             reads: register.avgMinMax({
                 name: "lodestar_cp_state_epoch_reads",
                 help: "Avg min max of all state cache items total read count",
@@ -1518,7 +1597,13 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 }),
                 created: register.counter({
                     name: "lodestar_seen_payload_envelope_input_cache_items_created_total",
-                    help: "Number of PayloadEnvelopeInputs created",
+                    help: "Number of PayloadEnvelopeInputs created by source",
+                    labelNames: ["source"],
+                }),
+                pruned: register.counter({
+                    name: "lodestar_seen_payload_envelope_input_cache_items_pruned_total",
+                    help: "Number of PayloadEnvelopeInputs evicted by reason",
+                    labelNames: ["reason"],
                 }),
             },
         },
@@ -1624,6 +1709,10 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 name: "lodestar_precompute_next_epoch_transition_waste_total",
                 help: "Total number of precomputing next epoch transition wasted",
             }),
+            predictedReorg: register.counter({
+                name: "lodestar_precompute_next_epoch_transition_predicted_reorg_total",
+                help: "Predicted epoch-boundary reorgs where the strong parent was dialed instead of the weak head",
+            }),
             duration: register.histogram({
                 name: "lodestar_precompute_next_epoch_transition_duration_seconds",
                 help: "Duration of precomputeNextEpochTransition, including epoch transition and hashTreeRoot",
@@ -1854,6 +1943,39 @@ export function createLodestarMetrics(register, metadata, genesisTime) {
                 labelNames: ["urlIndex", "baseUrl"],
             }),
         },
+        builderApi: {
+            statusChecks: register.counter({
+                name: "lodestar_builder_api_status_checks_total",
+                help: "Total count of status checks sent to external builders ahead of a proposal",
+                labelNames: ["status"],
+            }),
+            bidRequests: register.counter({
+                name: "lodestar_builder_api_bid_requests_total",
+                help: "Total count of execution payload bid requests sent to external builders",
+            }),
+            bidRequestErrors: register.counter({
+                name: "lodestar_builder_api_bid_request_errors_total",
+                help: "Total count of failed execution payload bid requests to external builders",
+            }),
+            bidsReceived: register.counter({
+                name: "lodestar_builder_api_bids_received_total",
+                help: "Total count of execution payload bids received from external builders",
+            }),
+            bidsDiscarded: register.counter({
+                name: "lodestar_builder_api_bids_discarded_total",
+                help: "Total count of execution payload bids from external builders discarded due to failed validation",
+            }),
+            blockSubmissions: register.counter({
+                name: "lodestar_builder_api_block_submissions_total",
+                help: "Total count of signed beacon blocks submitted to external builders",
+                labelNames: ["status"],
+            }),
+            preferencesForwarded: register.counter({
+                name: "lodestar_builder_api_preferences_forwarded_total",
+                help: "Total count of builder preferences forwarded to external builders",
+                labelNames: ["status"],
+            }),
+        },
         db: {
             dbReadReq: register.gauge({
                 name: "lodestar_db_read_req_total",