@lodestar/beacon-node 1.41.0-dev.253ed58dd1 → 1.41.0-dev.26ed5ad353

package/src/chain/validation/dataColumnSidecar.ts

@@ -12,6 +12,7 @@ import {
 } from "@lodestar/state-transition";
 import {DataColumnSidecar, Root, Slot, SubnetID, fulu, ssz} from "@lodestar/types";
 import {byteArrayEquals, toRootHex, verifyMerkleBranch} from "@lodestar/utils";
+import {BeaconMetrics} from "../../metrics/metrics/beacon.js";
 import {Metrics} from "../../metrics/metrics.js";
 import {kzg} from "../../util/kzg.js";
 import {
@@ -73,7 +74,7 @@ export async function validateGossipDataColumnSidecar(
   // 6) [IGNORE] The sidecar's block's parent (defined by block_header.parent_root) has been seen (via gossip
   //             or non-gossip sources)
   const parentRoot = toRootHex(blockHeader.parentRoot);
-  const parentBlock = chain.forkChoice.getBlockHex(parentRoot);
+  const parentBlock = chain.forkChoice.getBlockHexDefaultStatus(parentRoot);
   if (parentBlock === null) {
     // If fork choice does *not* consider the parent to be a descendant of the finalized block,
     // then there are two more cases:
@@ -177,6 +178,7 @@ export async function validateGossipDataColumnSidecar(
     });
   }
 
+  // single data column is being verified here
   const kzgProofTimer = metrics?.peerDas.dataColumnSidecarKzgProofsVerificationTime.startTimer();
   // 11) [REJECT] The sidecar's column data is valid as verified by verify_data_column_sidecar_kzg_proofs
   try {
@@ -297,159 +299,177 @@ export async function validateBlockDataColumnSidecars(
   blockSlot: Slot,
   blockRoot: Root,
   blockBlobCount: number,
-  dataColumnSidecars: fulu.DataColumnSidecars
+  dataColumnSidecars: fulu.DataColumnSidecars,
+  metrics?: BeaconMetrics["peerDas"] | null
 ): Promise<void> {
-  if (dataColumnSidecars.length === 0) {
-    return;
-  }
+  metrics?.dataColumnSidecarProcessingRequests.inc(dataColumnSidecars.length);
+  const verificationTimer = metrics?.dataColumnSidecarGossipVerificationTime.startTimer();
+  try {
+    if (dataColumnSidecars.length === 0) {
+      return;
+    }
 
-  if (blockBlobCount === 0) {
-    throw new DataColumnSidecarValidationError(
-      {
-        code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT,
-        slot: blockSlot,
-        expected: 0,
-        actual: dataColumnSidecars.length,
-      },
-      "Block has no blob commitments but data column sidecars were provided"
-    );
-  }
-  // Hash the first sidecar block header and compare the rest via (cheaper) equality
-  const firstSidecarSignedBlockHeader = dataColumnSidecars[0].signedBlockHeader;
-  const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message;
-  const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader);
-  if (!byteArrayEquals(blockRoot, firstBlockRoot)) {
-    throw new DataColumnSidecarValidationError(
-      {
-        code: DataColumnSidecarErrorCode.INCORRECT_BLOCK,
-        slot: blockSlot,
-        columnIndex: 0,
-        expected: toRootHex(blockRoot),
-        actual: toRootHex(firstBlockRoot),
-      },
-      "DataColumnSidecar doesn't match corresponding block"
-    );
-  }
+    if (blockBlobCount === 0) {
+      throw new DataColumnSidecarValidationError(
+        {
+          code: DataColumnSidecarErrorCode.INCORRECT_SIDECAR_COUNT,
+          slot: blockSlot,
+          expected: 0,
+          actual: dataColumnSidecars.length,
+        },
+        "Block has no blob commitments but data column sidecars were provided"
+      );
+    }
+    // Hash the first sidecar block header and compare the rest via (cheaper) equality
+    const firstSidecarSignedBlockHeader = dataColumnSidecars[0].signedBlockHeader;
+    const firstSidecarBlockHeader = firstSidecarSignedBlockHeader.message;
+    const firstBlockRoot = ssz.phase0.BeaconBlockHeader.hashTreeRoot(firstSidecarBlockHeader);
+    if (!byteArrayEquals(blockRoot, firstBlockRoot)) {
+      throw new DataColumnSidecarValidationError(
+        {
+          code: DataColumnSidecarErrorCode.INCORRECT_BLOCK,
+          slot: blockSlot,
+          columnIndex: 0,
+          expected: toRootHex(blockRoot),
+          actual: toRootHex(firstBlockRoot),
+        },
+        "DataColumnSidecar doesn't match corresponding block"
+      );
+    }
+
+    if (chain !== null) {
+      const rootHex = toRootHex(blockRoot);
+      const slot = firstSidecarSignedBlockHeader.message.slot;
+      const signature = firstSidecarSignedBlockHeader.signature;
+      if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) {
+        const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(
+          chain.config,
+          firstSidecarSignedBlockHeader
+        );
+
+        if (
+          !(await chain.bls.verifySignatureSets([signatureSet], {
+            verifyOnMainThread: true,
+          }))
+        ) {
+          throw new DataColumnSidecarValidationError({
+            code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
+            blockRoot: rootHex,
+            slot: blockSlot,
+            index: dataColumnSidecars[0].index,
+          });
+        }
+
+        chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature);
+      }
+    }
 
-  if (chain !== null) {
-    const rootHex = toRootHex(blockRoot);
-    const slot = firstSidecarSignedBlockHeader.message.slot;
-    const signature = firstSidecarSignedBlockHeader.signature;
-    if (!chain.seenBlockInputCache.isVerifiedProposerSignature(slot, rootHex, signature)) {
-      const signatureSet = getBlockHeaderProposerSignatureSetByHeaderSlot(chain.config, firstSidecarSignedBlockHeader);
+    const commitments: Uint8Array[] = [];
+    const cellIndices: number[] = [];
+    const cells: Uint8Array[] = [];
+    const proofs: Uint8Array[] = [];
+    for (let i = 0; i < dataColumnSidecars.length; i++) {
+      const columnSidecar = dataColumnSidecars[i];
 
       if (
-        !(await chain.bls.verifySignatureSets([signatureSet], {
-          verifyOnMainThread: true,
-        }))
+        i !== 0 &&
+        !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader)
       ) {
         throw new DataColumnSidecarValidationError({
-          code: DataColumnSidecarErrorCode.PROPOSAL_SIGNATURE_INVALID,
-          blockRoot: rootHex,
+          code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT,
           slot: blockSlot,
-          index: dataColumnSidecars[0].index,
+          expected: toRootHex(blockRoot),
+          actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)),
         });
       }
 
-      chain.seenBlockInputCache.markVerifiedProposerSignature(slot, rootHex, signature);
-    }
-  }
+      if (columnSidecar.index >= NUMBER_OF_COLUMNS) {
+        throw new DataColumnSidecarValidationError(
+          {
+            code: DataColumnSidecarErrorCode.INVALID_INDEX,
+            slot: blockSlot,
+            columnIndex: columnSidecar.index,
+          },
+          "DataColumnSidecar has invalid index"
+        );
+      }
 
-  const commitments: Uint8Array[] = [];
-  const cellIndices: number[] = [];
-  const cells: Uint8Array[] = [];
-  const proofs: Uint8Array[] = [];
-  for (let i = 0; i < dataColumnSidecars.length; i++) {
-    const columnSidecar = dataColumnSidecars[i];
+      if (columnSidecar.column.length !== blockBlobCount) {
+        throw new DataColumnSidecarValidationError({
+          code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT,
+          slot: blockSlot,
+          columnIndex: columnSidecar.index,
+          expected: blockBlobCount,
+          actual: columnSidecar.column.length,
+        });
+      }
 
-    if (
-      i !== 0 &&
-      !ssz.phase0.SignedBeaconBlockHeader.equals(firstSidecarSignedBlockHeader, columnSidecar.signedBlockHeader)
-    ) {
-      throw new DataColumnSidecarValidationError({
-        code: DataColumnSidecarErrorCode.INCORRECT_HEADER_ROOT,
-        slot: blockSlot,
-        expected: toRootHex(blockRoot),
-        actual: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(columnSidecar.signedBlockHeader.message)),
-      });
-    }
+      if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) {
+        throw new DataColumnSidecarValidationError({
+          code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT,
+          slot: blockSlot,
+          columnIndex: columnSidecar.index,
+          expected: columnSidecar.column.length,
+          actual: columnSidecar.kzgCommitments.length,
+        });
+      }
 
-    if (columnSidecar.index >= NUMBER_OF_COLUMNS) {
-      throw new DataColumnSidecarValidationError(
-        {
-          code: DataColumnSidecarErrorCode.INVALID_INDEX,
+      if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) {
+        throw new DataColumnSidecarValidationError({
+          code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT,
           slot: blockSlot,
           columnIndex: columnSidecar.index,
-        },
-        "DataColumnSidecar has invalid index"
-      );
-    }
+          expected: columnSidecar.column.length,
+          actual: columnSidecar.kzgProofs.length,
+        });
+      }
 
-    if (columnSidecar.column.length !== blockBlobCount) {
-      throw new DataColumnSidecarValidationError({
-        code: DataColumnSidecarErrorCode.INCORRECT_CELL_COUNT,
-        slot: blockSlot,
-        columnIndex: columnSidecar.index,
-        expected: blockBlobCount,
-        actual: columnSidecar.column.length,
-      });
-    }
+      const inclusionProofTimer = metrics?.dataColumnSidecarInclusionProofVerificationTime.startTimer();
+      const validInclusionProof = verifyDataColumnSidecarInclusionProof(columnSidecar);
+      inclusionProofTimer?.();
+      if (!validInclusionProof) {
+        throw new DataColumnSidecarValidationError(
+          {
+            code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
+            slot: blockSlot,
+            columnIndex: columnSidecar.index,
+          },
+          "DataColumnSidecar has invalid inclusion proof"
+        );
+      }
 
-    if (columnSidecar.column.length !== columnSidecar.kzgCommitments.length) {
-      throw new DataColumnSidecarValidationError({
-        code: DataColumnSidecarErrorCode.INCORRECT_KZG_COMMITMENTS_COUNT,
-        slot: blockSlot,
-        columnIndex: columnSidecar.index,
-        expected: columnSidecar.column.length,
-        actual: columnSidecar.kzgCommitments.length,
-      });
+      commitments.push(...columnSidecar.kzgCommitments);
+      cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index));
+      cells.push(...columnSidecar.column);
+      proofs.push(...columnSidecar.kzgProofs);
     }
 
-    if (columnSidecar.column.length !== columnSidecar.kzgProofs.length) {
-      throw new DataColumnSidecarValidationError({
-        code: DataColumnSidecarErrorCode.INCORRECT_KZG_PROOF_COUNT,
-        slot: blockSlot,
-        columnIndex: columnSidecar.index,
-        expected: columnSidecar.column.length,
-        actual: columnSidecar.kzgProofs.length,
-      });
+    let reason: string | undefined;
+    // batch verification for the cases: downloadByRange and downloadByRoot
+    const kzgVerificationTimer = metrics?.kzgVerificationDataColumnBatchTime.startTimer();
+    try {
+      const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs);
+      if (!valid) {
+        reason = "Invalid KZG proof batch";
+      }
+    } catch (e) {
+      reason = (e as Error).message;
+    } finally {
+      kzgVerificationTimer?.();
     }
-
-    if (!verifyDataColumnSidecarInclusionProof(columnSidecar)) {
+    if (reason !== undefined) {
       throw new DataColumnSidecarValidationError(
         {
-          code: DataColumnSidecarErrorCode.INCLUSION_PROOF_INVALID,
+          code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH,
           slot: blockSlot,
-          columnIndex: columnSidecar.index,
+          reason,
         },
-        "DataColumnSidecar has invalid inclusion proof"
+        "DataColumnSidecar has invalid KZG proof batch"
       );
     }
-
-    commitments.push(...columnSidecar.kzgCommitments);
-    cellIndices.push(...Array.from({length: columnSidecar.column.length}, () => columnSidecar.index));
-    cells.push(...columnSidecar.column);
-    proofs.push(...columnSidecar.kzgProofs);
-  }
-
-  let reason: string | undefined;
-  try {
-    const valid = await kzg.asyncVerifyCellKzgProofBatch(commitments, cellIndices, cells, proofs);
-    if (!valid) {
-      reason = "Invalid KZG proof batch";
-    }
-  } catch (e) {
-    reason = (e as Error).message;
-  }
-  if (reason !== undefined) {
-    throw new DataColumnSidecarValidationError(
-      {
-        code: DataColumnSidecarErrorCode.INVALID_KZG_PROOF_BATCH,
-        slot: blockSlot,
-        reason,
-      },
-      "DataColumnSidecar has invalid KZG proof batch"
-    );
+    metrics?.dataColumnSidecarProcessingSuccesses.inc();
+  } finally {
+    verificationTimer?.();
   }
 }
 

package/src/chain/validation/executionPayloadBid.ts

@@ -113,8 +113,7 @@ async function validateExecutionPayloadBid(
 
   // [IGNORE] `bid.parent_block_root` is the hash tree root of a known beacon
   // block in fork choice.
-  const block = chain.forkChoice.getBlock(bid.parentBlockRoot);
-  if (block === null) {
+  if (!chain.forkChoice.hasBlock(bid.parentBlockRoot)) {
     throw new ExecutionPayloadBidError(GossipAction.IGNORE, {
       code: ExecutionPayloadBidErrorCode.UNKNOWN_BLOCK_ROOT,
       parentBlockRoot: parentBlockRootHex,

package/src/chain/validation/executionPayloadEnvelope.ts

@@ -37,7 +37,7 @@ async function validateExecutionPayloadEnvelope(
   // the block is retrieved).
   // TODO GLOAS: Need to review this, we should queue the envelope for later
   // processing if the block is not yet known, otherwise we would ignore it here
-  const block = chain.forkChoice.getBlock(envelope.beaconBlockRoot);
+  const block = chain.forkChoice.getBlockDefaultStatus(envelope.beaconBlockRoot);
   if (block === null) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
       code: ExecutionPayloadEnvelopeErrorCode.BLOCK_ROOT_UNKNOWN,
@@ -79,7 +79,7 @@ async function validateExecutionPayloadEnvelope(
     });
   }
 
-  if (block.builderIndex === undefined || block.blockHashHex === undefined) {
+  if (block.builderIndex == null || block.blockHashFromBid == null) {
     // This indicates this block is a pre-gloas block which is wrong
     throw new ExecutionPayloadEnvelopeError(GossipAction.IGNORE, {
       code: ExecutionPayloadEnvelopeErrorCode.CACHE_FAIL,
@@ -97,11 +97,11 @@ async function validateExecutionPayloadEnvelope(
   }
 
   // [REJECT] `payload.block_hash == bid.block_hash`
-  if (toRootHex(payload.blockHash) !== block.blockHashHex) {
+  if (toRootHex(payload.blockHash) !== block.blockHashFromBid) {
     throw new ExecutionPayloadEnvelopeError(GossipAction.REJECT, {
       code: ExecutionPayloadEnvelopeErrorCode.BLOCK_HASH_MISMATCH,
       envelopeBlockHash: toRootHex(payload.blockHash),
-      bidBlockHash: block.blockHashHex,
+      bidBlockHash: block.blockHashFromBid,
     });
   }
 

package/src/chain/validation/payloadAttestationMessage.ts

@@ -59,8 +59,7 @@ async function validatePayloadAttestationMessage(
   // [IGNORE] The message's block `data.beacon_block_root` has been seen (via
   // gossip or non-gossip sources) (a client MAY queue attestation for processing
   // once the block is retrieved. Note a client might want to request payload after).
-  const block = chain.forkChoice.getBlock(data.beaconBlockRoot);
-  if (block === null) {
+  if (!chain.forkChoice.hasBlock(data.beaconBlockRoot)) {
     throw new PayloadAttestationError(GossipAction.IGNORE, {
       code: PayloadAttestationErrorCode.UNKNOWN_BLOCK_ROOT,
       blockRoot: toRootHex(data.beaconBlockRoot),
@@ -87,8 +86,16 @@ async function validatePayloadAttestationMessage(
   }
 
   // [REJECT] `payload_attestation_message.signature` is valid with respect to the validator's public key.
+  const validatorPubkey = chain.pubkeyCache.get(validatorIndex);
+  if (!validatorPubkey) {
+    throw new PayloadAttestationError(GossipAction.REJECT, {
+      code: PayloadAttestationErrorCode.INVALID_ATTESTER,
+      attesterIndex: validatorIndex,
+    });
+  }
+
   const signatureSet = createSingleSignatureSetFromComponents(
-    chain.index2pubkey[validatorIndex],
+    validatorPubkey,
     getPayloadAttestationDataSigningRoot(chain.config, data),
     payloadAttestationMessage.signature
   );

package/src/chain/validation/proposerSlashing.ts

@@ -37,7 +37,7 @@ async function validateProposerSlashing(
   try {
     const proposer = state.validators.getReadonly(proposerSlashing.signedHeader1.message.proposerIndex);
     // verifySignature = false, verified in batch below
-    assertValidProposerSlashing(chain.config, chain.index2pubkey, state.slot, proposerSlashing, proposer, false);
+    assertValidProposerSlashing(chain.config, chain.pubkeyCache, state.slot, proposerSlashing, proposer, false);
   } catch (e) {
     throw new ProposerSlashingError(GossipAction.REJECT, {
       code: ProposerSlashingErrorCode.INVALID,

package/src/chain/validation/syncCommitteeContributionAndProof.ts

@@ -106,7 +106,7 @@ export async function validateSyncCommitteeGossipContributionAndProof(
 /**
  * Retrieve pubkeys in contribution aggregate using epochCtx:
  * - currSyncCommitteeIndexes cache
- * - index2pubkey cache
+ * - pubkeyCache
  */
 function getContributionIndices(
   state: CachedBeaconStateAltair,

package/src/db/index.ts

@@ -1,2 +1,3 @@
 export {BeaconDb} from "./beacon.js";
+export {Bucket} from "./buckets.js";
 export type {IBeaconDb} from "./interface.js";

package/src/db/repositories/blockArchive.ts

@@ -1,4 +1,3 @@
-import all from "it-all";
 import {ChainForkConfig} from "@lodestar/config";
 import {Db, FilterOptions, KeyValue, Repository} from "@lodestar/db";
 import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
@@ -121,7 +120,7 @@ export class BlockArchiveRepository extends Repository<Slot, SignedBeaconBlock>
   }
 
   async values(opts?: BlockFilterOptions): Promise<SignedBeaconBlock[]> {
-    return all(this.valuesStream(opts));
+    return await Array.fromAsync(this.valuesStream(opts));
   }
 
   // INDEX

package/src/execution/engine/http.ts

@@ -128,6 +128,7 @@ const getClientVersionOpts: ReqOpts = {routeId: "getClientVersion"};
  */
 export class ExecutionEngineHttp implements IExecutionEngine {
   private logger: Logger;
+  private metrics: Metrics | null;
 
   // The default state is ONLINE, it will be updated to SYNCING once we receive the first payload
   // This assumption is better than the OFFLINE state, since we can't be sure if the EL is offline and being offline may trigger some notifications
@@ -167,6 +168,7 @@ export class ExecutionEngineHttp implements IExecutionEngine {
       metrics?.engineHttpProcessorQueue
     );
     this.logger = logger;
+    this.metrics = metrics ?? null;
 
     this.rpc.emitter.on(JsonRpcHttpClientEvent.ERROR, ({error}) => {
       this.updateEngineState(getExecutionEngineState({payloadError: error, oldState: this.state}));
@@ -369,6 +371,7 @@ export class ExecutionEngineHttp implements IExecutionEngine {
     } = await request;
 
     this.updateEngineState(getExecutionEngineState({payloadStatus: status, oldState: this.state}));
+    this.metrics?.engineNotifyForkchoiceUpdateResult.inc({result: status});
 
     switch (status) {
       case ExecutionPayloadStatus.VALID:

package/src/metrics/metrics/beacon.ts

@@ -333,11 +333,13 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         help: "Time taken to verify data_column sidecar inclusion proof",
         buckets: [0.002, 0.004, 0.006, 0.008, 0.01, 0.05, 1, 2],
       }),
+      // single verification
       dataColumnSidecarKzgProofsVerificationTime: register.histogram({
         name: "beacon_data_column_sidecar_kzg_proofs_verification_seconds",
-        help: "Time taken to verify data_column sidecar kzg proofs",
+        help: "Time taken to verify single data_column sidecar kzg proofs",
         buckets: [0.01, 0.02, 0.03, 0.04, 0.05, 0.1, 0.2, 0.5, 1],
       }),
+      // batch verification
       kzgVerificationDataColumnBatchTime: register.histogram({
         name: "beacon_kzg_verification_data_column_batch_seconds",
         help: "Runtime of batched data column kzg verification",
@@ -361,10 +363,14 @@ export function createBeaconMetrics(register: RegistryMetricCreator) {
         help: "Duration of engine_getBlobsV2 requests",
         buckets: [0.01, 0.05, 0.1, 0.5, 1, 2.5, 5, 7.5],
       }),
-      targetCustodyGroupCount: register.gauge({
-        name: "beacon_target_custody_group_count",
+      custodyGroupCount: register.gauge({
+        name: "beacon_custody_groups",
         help: "Total number of custody groups within a node",
       }),
+      custodyGroupsBackfilled: register.gauge({
+        name: "beacon_custody_groups_backfilled",
+        help: "Total number of custody groups backfilled by a node",
+      }),
       reconstructedColumns: register.counter({
         name: "beacon_data_availability_reconstructed_columns_total",
         help: "Total count of reconstructed columns",

package/src/metrics/metrics/lodestar.ts

@@ -836,20 +836,23 @@ export function createLodestarMetrics(
         buckets: [0.5, 1, 2, 4, 6, 12],
       }),
     },
-    recoverDataColumnSidecars: {
-      recoverTime: register.histogram({
-        name: "lodestar_recover_data_column_sidecar_recover_time_seconds",
-        help: "Time elapsed to recover data column sidecar",
-        buckets: [0.5, 1.0, 1.5, 2],
+    // recovery in the case of specific blob rows required
+    recoverBlobSidecars: {
+      blobsReconstructed: register.counter({
+        name: "lodestar_blobs_reconstructed_total",
+        help: "Total count of reconstructed blobs",
+      }),
+      reconstructionTime: register.histogram({
+        name: "lodestar_blob_reconstruction_seconds",
+        help: "Time taken to reconstruct blobs",
+        buckets: [0.2, 0.4, 0.6, 0.8, 1.0, 1.2, 2, 5],
       }),
+    },
+    recoverDataColumnSidecars: {
       custodyBeforeReconstruction: register.gauge({
         name: "lodestar_data_columns_in_custody_before_reconstruction",
         help: "Number of data columns in custody before reconstruction",
       }),
-      numberOfColumnsRecovered: register.gauge({
-        name: "lodestar_recover_data_column_sidecar_recovered_columns_total",
-        help: "Total number of columns that were recovered",
-      }),
       reconstructionResult: register.counter<{result: DataColumnReconstructionCode}>({
         name: "lodestar_data_column_sidecars_reconstruction_result",
         help: "Data column sidecars reconstruction result",
@@ -857,6 +860,10 @@ export function createLodestarMetrics(
       }),
     },
     dataColumns: {
+      alreadyAdded: register.counter({
+        name: "lodestar_data_column_sidecar_already_added",
+        help: "Total number of columns that were already added by other sources while waiting",
+      }),
       bySource: register.gauge<{source: BlockInputSource}>({
         name: "lodestar_data_columns_by_source",
         help: "Number of received data columns by source",
@@ -912,11 +919,6 @@ export function createLodestarMetrics(
         help: "Total number of imported blobs by source",
         labelNames: ["blobsSource"],
       }),
-      columnsBySource: register.gauge<{source: BlockInputSource}>({
-        name: "lodestar_import_columns_by_source_total",
-        help: "Total number of imported columns (sampled columns) by source",
-        labelNames: ["source"],
-      }),
       notOverrideFcuReason: register.counter<{reason: NotReorgedReason}>({
         name: "lodestar_import_block_not_override_fcu_reason_total",
         help: "Reason why the fcu call is not suppressed during block import",
@@ -928,6 +930,11 @@ export function createLodestarMetrics(
       help: "The total result of calling notifyNewPayload execution engine api",
       labelNames: ["result"],
     }),
+    engineNotifyForkchoiceUpdateResult: register.gauge<{result: ExecutionPayloadStatus}>({
+      name: "lodestar_execution_engine_notify_forkchoice_update_result_total",
+      help: "The total result of calling notifyForkchoiceUpdate execution engine api",
+      labelNames: ["result"],
+    }),
     backfillSync: {
       backfilledTillSlot: register.gauge({
         name: "lodestar_backfill_till_slot",
@@ -1456,29 +1463,37 @@ export function createLodestarMetrics(
           name: "lodestar_seen_block_input_cache_size",
           help: "Number of cached BlockInputs",
         }),
-        duplicateBlockCount: register.gauge<{source: BlockInputSource}>({
-          name: "lodestar_seen_block_input_cache_duplicate_block_count",
+        serializedObjectRefs: register.gauge({
+          name: "lodestar_seen_block_input_cache_serialized_object_refs",
+          help: "Number of serialized-cache object refs retained by cached BlockInputs",
+        }),
+        duplicateBlockCount: register.counter<{source: BlockInputSource}>({
+          name: "lodestar_seen_block_input_cache_duplicate_block_total",
           help: "Total number of duplicate blocks that pass validation and attempt to be cached but are known",
           labelNames: ["source"],
         }),
-        duplicateBlobCount: register.gauge<{source: BlockInputSource}>({
-          name: "lodestar_seen_block_input_cache_duplicate_blob_count",
+        duplicateBlobCount: register.counter<{source: BlockInputSource}>({
+          name: "lodestar_seen_block_input_cache_duplicate_blob_total",
           help: "Total number of duplicate blobs that pass validation and attempt to be cached but are known",
           labelNames: ["source"],
         }),
-        duplicateColumnCount: register.gauge<{source: BlockInputSource}>({
-          name: "lodestar_seen_block_input_cache_duplicate_column_count",
+        duplicateColumnCount: register.counter<{source: BlockInputSource}>({
+          name: "lodestar_seen_block_input_cache_duplicate_column_total",
           help: "Total number of duplicate columns that pass validation and attempt to be cached but are known",
           labelNames: ["source"],
         }),
-        createdByBlock: register.gauge({
-          name: "lodestar_seen_block_input_cache_items_created_by_block",
+        createdByBlock: register.counter({
+          name: "lodestar_seen_block_input_cache_items_created_by_block_total",
           help: "Number of BlockInputs created via a block being seen first",
         }),
-        createdByBlob: register.gauge({
-          name: "lodestar_seen_block_input_cache_items_created_by_blob",
+        createdByBlob: register.counter({
+          name: "lodestar_seen_block_input_cache_items_created_by_blob_total",
           help: "Number of BlockInputs created via a blob being seen first",
         }),
+        createdByColumn: register.counter({
+          name: "lodestar_seen_block_input_cache_items_created_by_column_total",
+          help: "Number of BlockInputs created via a data column being seen first",
+        }),
       },
     },
 

package/src/monitoring/service.ts

@@ -89,9 +89,9 @@ export class MonitoringService {
   }
 
   /**
-   * Stop sending client stats
+   * Stop sending client stats and wait for any pending request to complete
    */
-  close(): void {
+  async close(): Promise<void> {
     if (this.status === Status.Closed) return;
     this.status = Status.Closed;
 
@@ -103,6 +103,7 @@ export class MonitoringService {
     }
     if (this.pendingRequest) {
       this.fetchAbortController?.abort(FetchAbortReason.Close);
+      await this.pendingRequest;
     }
   }
 

package/src/network/core/networkCore.ts

@@ -1,8 +1,8 @@
-import {Connection, PrivateKey} from "@libp2p/interface";
+import type {PeerScoreStatsDump} from "@libp2p/gossipsub/score";
+import type {PublishOpts} from "@libp2p/gossipsub/types";
+import type {Connection, PrivateKey} from "@libp2p/interface";
 import {peerIdFromPrivateKey} from "@libp2p/peer-id";
 import {multiaddr} from "@multiformats/multiaddr";
-import {PeerScoreStatsDump} from "@chainsafe/libp2p-gossipsub/score";
-import {PublishOpts} from "@chainsafe/libp2p-gossipsub/types";
 import {routes} from "@lodestar/api";
 import {BeaconConfig, ForkBoundary} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";