@lodestar/beacon-node 1.48.0-dev.fd70e5fc15 → 1.48.0-dev.ff9749be2c

package/src/db/repositories/blockArchiveIndex.ts

@@ -1,7 +1,6 @@
 import {Db, encodeKey} from "@lodestar/db";
 import {ForkAll} from "@lodestar/params";
 import {Root, SSZTypesFor, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
-import {intToBytes} from "@lodestar/utils";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
 export const rootIndexBucketId = getBucketNameByValue(Bucket.index_blockArchiveRootIndex);
@@ -15,14 +14,6 @@ export async function getParentRootIndex(db: Db, parentRoot: Root): Promise<Uint
   return db.get(getParentRootIndexKey(parentRoot), {bucketId: parentRootIndexBucketId});
 }
 
-export async function storeRootIndex(db: Db, slot: Slot, blockRoot: Root): Promise<void> {
-  return db.put(getRootIndexKey(blockRoot), intToBytes(slot, 8, "be"), {bucketId: rootIndexBucketId});
-}
-
-export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root): Promise<void> {
-  return db.put(getParentRootIndexKey(parentRoot), intToBytes(slot, 8, "be"), {bucketId: parentRootIndexBucketId});
-}
-
 export async function deleteRootIndex(
   db: Db,
   signedBeaconBlockType: SSZTypesFor<ForkAll, "SignedBeaconBlock">,
@@ -43,3 +34,7 @@ export function getParentRootIndexKey(parentRoot: Root): Uint8Array {
 export function getRootIndexKey(root: Root): Uint8Array {
   return encodeKey(Bucket.index_blockArchiveRootIndex, root);
 }
+
+export function getSlotIndexKey(slot: Slot): Uint8Array {
+  return encodeKey(Bucket.index_mainChain, slot);
+}

package/src/metrics/metrics/lodestar.ts

@@ -21,6 +21,7 @@ import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
+import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
@@ -2099,7 +2100,7 @@ export function createLodestarMetrics(
       }),
       dbSizeTotal: register.gauge({
         name: "lodestar_db_size_bytes_total",
-        help: "Approximate number of bytes of file system space used by db",
+        help: "Approximate number of bytes of file system space used by LevelDB",
       }),
       dbApproximateSizeTime: register.histogram({
         name: "lodestar_db_approximate_size_time_seconds",
@@ -2108,6 +2109,61 @@ export function createLodestarMetrics(
       }),
     },
 
+    flatFileStore: {
+      operationDuration: {
+        read: register.histogram({
+          name: "lodestar_flat_file_store_read_duration_seconds",
+          help: "Duration of flat file store reads in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        write: register.histogram({
+          name: "lodestar_flat_file_store_write_duration_seconds",
+          help: "Duration of flat file store writes in seconds",
+          buckets: [0.005, 0.01, 0.05, 0.1, 0.5, 1],
+        }),
+        delete: register.histogram({
+          name: "lodestar_flat_file_store_delete_duration_seconds",
+          help: "Duration of flat file store deletions in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        prune: register.histogram({
+          name: "lodestar_flat_file_store_prune_duration_seconds",
+          help: "Duration of flat file store pruning in seconds",
+          buckets: [0.0005, 0.01, 0.05, 0.5, 5, 30],
+        }),
+      },
+      operationErrors: register.counter<{operation: FlatFileStoreOperation}>({
+        name: "lodestar_flat_file_store_operation_errors_total",
+        help: "Total count of failed flat file store operations",
+        labelNames: ["operation"],
+      }),
+      readBytes: register.counter({
+        name: "lodestar_flat_file_store_read_bytes_total",
+        help: "Total bytes read from flat file storage",
+      }),
+      writeBytes: register.counter({
+        name: "lodestar_flat_file_store_write_bytes_total",
+        help: "Total bytes written to flat file storage",
+      }),
+      prunedDirectories: register.counter({
+        name: "lodestar_flat_file_store_pruned_directories_total",
+        help: "Total count of slot directories pruned from flat file storage",
+      }),
+      slotIndexSize: register.gauge({
+        name: "lodestar_flat_file_store_indexed_slots",
+        help: "Count of indexed flat file slot directories, including empty directories awaiting pruning",
+      }),
+      startupDuration: register.histogram({
+        name: "lodestar_flat_file_store_startup_duration_seconds",
+        help: "Duration of flat file store slot index reconstruction in seconds",
+        buckets: [0.1, 0.5, 1, 5, 10, 30, 60, 300],
+      }),
+      startupErrors: register.counter({
+        name: "lodestar_flat_file_store_startup_errors_total",
+        help: "Total count of flat file store startup failures",
+      }),
+    },
+
     pruneHistory: {
       pruneCount: register.gauge({
         name: "lodestar_prune_history_prune_count_total",

package/src/monitoring/clientStats.ts

@@ -107,7 +107,7 @@ function createBeaconNodeStats(): ClientStats {
       metricName: "lodestar_db_size_bytes_total",
       jsonType: JsonType.Number,
       defaultValue: 0,
-      description: "Amount of bytes consumed on disk by the beacon node's database",
+      description: "Amount of bytes consumed on disk by the beacon node's LevelDB database",
     }),
     networkLibp2pBytesTotalReceive: new MetricProperty({
       jsonKey: "network_libp2p_bytes_total_receive",

package/src/network/gossip/encoding.ts

@@ -1,7 +1,6 @@
 import type {Message} from "@libp2p/gossipsub";
 import type {RPC} from "@libp2p/gossipsub/message";
 import type {DataTransform} from "@libp2p/gossipsub/types";
-// snappyjs is better for compression for smaller payloads
 import xxhashFactory from "xxhash-wasm";
 import {digest} from "@chainsafe/as-sha256";
 import snappyWasm from "@chainsafe/snappy-wasm";
@@ -123,9 +122,6 @@ export class DataTransformSnappy implements DataTransform {
     if (uncompressedDataLength > maxSize) {
       throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
     }
-    if (uncompressedDataLength > sszType.maxSize) {
-      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
-    }
 
     // Only after sanity length checks, we can decompress the data
     // Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
@@ -146,9 +142,6 @@ export class DataTransformSnappy implements DataTransform {
     if (data.length > maxSize) {
       throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);
     }
-    if (data.length > sszType.maxSize) {
-      throw Error(`ssz_snappy encoded data length ${data.length} > ${sszType.maxSize}`);
-    }
 
     // Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
     const compressedData = Buffer.alloc(snappyWasm.max_compress_len(data.length));

package/src/network/gossip/topic.ts

@@ -4,17 +4,11 @@ import {
   ATTESTATION_SUBNET_COUNT,
   ForkName,
   ForkSeq,
-  MAX_ATTESTER_SLASHING_SIZE,
-  MAX_DATA_COLUMN_SIDECAR_SIZE,
-  MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
-  MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE,
-  MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE,
   SYNC_COMMITTEE_SUBNET_COUNT,
   isForkPostAltair,
   isForkPostElectra,
   isForkPostFulu,
   isForkPostGloas,
-  isForkPostHeze,
 } from "@lodestar/params";
 import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
 import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -139,27 +133,11 @@ export function getGossipSSZType(topic: GossipTopic) {
 }
 
 /**
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object.
+ * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
+ * or MAX_PAYLOAD_SIZE, whichever is smaller.
  */
 export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
-  const {fork} = topic.boundary;
-  // Gloas progressive containers have broad theoretical SSZ max sizes; use the preset p2p bounds instead.
-  switch (topic.type) {
-    case GossipType.beacon_block:
-      return maxPayloadSize;
-    case GossipType.beacon_aggregate_and_proof:
-      return isForkPostGloas(fork) ? MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.attester_slashing:
-      return isForkPostGloas(fork) ? MAX_ATTESTER_SLASHING_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.data_column_sidecar:
-      return isForkPostGloas(fork) ? MAX_DATA_COLUMN_SIDECAR_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.execution_payload:
-      return maxPayloadSize;
-    case GossipType.execution_payload_bid:
-      return isForkPostHeze(fork) ? MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE : MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE;
-    default:
-      return (sszType ?? getGossipSSZType(topic)).maxSize;
-  }
+  return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
 }
 
 /**

package/src/network/libp2p/index.ts

@@ -1,11 +1,13 @@
 import {getCiphers} from "node:crypto";
 import {bootstrap} from "@libp2p/bootstrap";
 import {identify} from "@libp2p/identify";
-import type {PrivateKey} from "@libp2p/interface";
+import type {Address, PrivateKey} from "@libp2p/interface";
 import {mdns} from "@libp2p/mdns";
 import {mplex} from "@libp2p/mplex";
 import {prometheusMetrics} from "@libp2p/prometheus-metrics";
 import {tcp} from "@libp2p/tcp";
+import {loopbackAddressLast, publicAddressesFirst, reliableTransportsFirst} from "@libp2p/utils";
+import type {Multiaddr} from "@multiformats/multiaddr";
 import {Libp2pInit, createLibp2p} from "libp2p";
 import {Registry} from "prom-client";
 import {ENR} from "@chainsafe/enr";
@@ -24,6 +26,29 @@ const noiseCrypto = getCiphers().includes("chacha20-poly1305")
       chaCha20Poly1305Decrypt: asCrypto.chaCha20Poly1305Decrypt,
     };
 
+function quicAddressesFirst(a: Multiaddr, b: Multiaddr): -1 | 0 | 1 {
+  const aQuic = a.getComponents().some((c) => c.name === "quic-v1");
+  const bQuic = b.getComponents().some((c) => c.name === "quic-v1");
+  if (aQuic === bQuic) return 0;
+  return aQuic ? -1 : 1;
+}
+
+/**
+ * Custom sorter for libp2p's `addressSorter`.
+ *
+ * Prefers QUIC > TCP addresses before dialing, per p2p spec.
+ *
+ * Spec: https://github.com/ethereum/consensus-specs/blob/f21dac06e99743b1e98bb80297897b96520c942b/specs/phase0/p2p-interface.md#transport
+ */
+export function quicFirstAddressSorter(a: Address, b: Address): -1 | 0 | 1 {
+  return (
+    loopbackAddressLast(a.multiaddr, b.multiaddr) ||
+    publicAddressesFirst(a.multiaddr, b.multiaddr) ||
+    quicAddressesFirst(a.multiaddr, b.multiaddr) ||
+    reliableTransportsFirst(a.multiaddr, b.multiaddr)
+  );
+}
+
 export type NodeJsLibp2pOpts = {
   peerStoreDir?: string;
   disablePeerDiscovery?: boolean;
@@ -150,6 +175,8 @@ export async function createNodeJsLibp2p(
       // the maximum number of pending connections libp2p will accept before it starts rejecting incoming connections.
       // make it the same to backlog option above
       maxIncomingPendingConnections: 5,
+      // Prefer dialing QUIC > TCP for a peer address if QUIC is enabled
+      addressSorter: quicEnabled ? quicFirstAddressSorter : undefined,
     },
     // rely on lodestar's peer manager to ping peers
     connectionMonitor: {

package/src/network/processor/gossipHandlers.ts

@@ -27,7 +27,7 @@ import {
   ssz,
   sszTypesFor,
 } from "@lodestar/types";
-import {LogLevel, Logger, prettyBytes, toHex, toRootHex} from "@lodestar/utils";
+import {LogLevel, Logger, toHex, toRootHex} from "@lodestar/utils";
 import {
   BlockInput,
   BlockInputColumns,
@@ -157,7 +157,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     const slot = signedBlock.message.slot;
     const forkTypes = config.getForkTypes(slot);
     const blockRootHex = toRootHex(forkTypes.BeaconBlock.hashTreeRoot(signedBlock.message));
-    const blockShortHex = prettyBytes(blockRootHex);
     const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
     const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
 
@@ -167,6 +166,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
 
     const logCtx = {
       slot,
+      root: blockRootHex,
       currentSlot: chain.clock.currentSlot,
       peerId: peerIdStr,
       delaySec,
@@ -227,7 +227,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       return blockInput;
     } catch (e) {
       if (e instanceof BlockGossipError) {
-        logger.debug("Gossip block has error", {slot, root: blockShortHex, code: e.type.code});
+        logger.debug("Gossip block has error", {slot, root: blockRootHex, code: e.type.code});
         if (
           (e.type.code === BlockErrorCode.PARENT_BLOCK_UNKNOWN ||
             e.type.code === BlockErrorCode.PARENT_PAYLOAD_UNKNOWN) &&
@@ -249,7 +249,12 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           throw e;
         }
 
-        chain.persistInvalidSszValue(forkTypes.SignedBeaconBlock, signedBlock, `gossip_reject_slot_${slot}`);
+        chain.persistInvalidSszValue(
+          forkTypes.SignedBeaconBlock,
+          signedBlock,
+          `gossip_reject_slot_${slot}`,
+          blockRootHex
+        );
       }
 
       // REJECT or unexpected (non-BlockGossipError) error: drop the optimistically-added entries from
@@ -279,7 +284,6 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
     const slot = blobBlockHeader.slot;
     const fork = config.getForkName(slot);
     const blockRootHex = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(blobBlockHeader));
-    const blockShortHex = prettyBytes(blockRootHex);
 
     const delaySec = chain.clock.secFromSlot(slot, seenTimestampSec);
     const recvToValLatency = Date.now() / 1000 - seenTimestampSec;
@@ -332,7 +336,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
       if (e instanceof BlobSidecarGossipError) {
         // Don't trigger this yet if full block and blobs haven't arrived yet
         if (e.type.code === BlobSidecarErrorCode.PARENT_UNKNOWN) {
-          logger.debug("Gossip blob has error", {slot, root: blockShortHex, code: e.type.code});
+          logger.debug("Gossip blob has error", {slot, root: blockRootHex, code: e.type.code});
           // no need to trigger `unknownBlockParent` event here, as we already did it in `validateBeaconBlock()`
           //
           // TODO(fulu): is this note above correct? Could have random blob that we see that could trigger
@@ -696,11 +700,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
           logLevel = LogLevel.error;
         }
         metrics?.gossipBlock.processBlockErrors.inc({error: e instanceof BlockError ? e.type.code : "NOT_BLOCK_ERROR"});
-        logger[logLevel](
-          "Error processing block",
-          {slot, peer: peerIdStr, blockRoot: prettyBytes(blockInput.blockRootHex)},
-          e as Error
-        );
+        logger[logLevel]("Error processing block", {slot, root: blockInput.blockRootHex, peer: peerIdStr}, e as Error);
         // TODO(fulu): Revisit when we prune block inputs
         chain.seenBlockInputCache.prune(blockInput.blockRootHex);
       });