@lodestar/beacon-node 1.47.0-rc.0 → 1.48.0-dev.02b355c4db

package/src/db/repositories/blockArchiveIndex.ts

@@ -1,7 +1,6 @@
 import {Db, encodeKey} from "@lodestar/db";
 import {ForkAll} from "@lodestar/params";
 import {Root, SSZTypesFor, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
-import {intToBytes} from "@lodestar/utils";
 import {Bucket, getBucketNameByValue} from "../buckets.js";
 
 export const rootIndexBucketId = getBucketNameByValue(Bucket.index_blockArchiveRootIndex);
@@ -15,14 +14,6 @@ export async function getParentRootIndex(db: Db, parentRoot: Root): Promise<Uint
   return db.get(getParentRootIndexKey(parentRoot), {bucketId: parentRootIndexBucketId});
 }
 
-export async function storeRootIndex(db: Db, slot: Slot, blockRoot: Root): Promise<void> {
-  return db.put(getRootIndexKey(blockRoot), intToBytes(slot, 8, "be"), {bucketId: rootIndexBucketId});
-}
-
-export async function storeParentRootIndex(db: Db, slot: Slot, parentRoot: Root): Promise<void> {
-  return db.put(getParentRootIndexKey(parentRoot), intToBytes(slot, 8, "be"), {bucketId: parentRootIndexBucketId});
-}
-
 export async function deleteRootIndex(
   db: Db,
   signedBeaconBlockType: SSZTypesFor<ForkAll, "SignedBeaconBlock">,
@@ -43,3 +34,7 @@ export function getParentRootIndexKey(parentRoot: Root): Uint8Array {
 export function getRootIndexKey(root: Root): Uint8Array {
   return encodeKey(Bucket.index_blockArchiveRootIndex, root);
 }
+
+export function getSlotIndexKey(slot: Slot): Uint8Array {
+  return encodeKey(Bucket.index_mainChain, slot);
+}

package/src/execution/builder/apiClient.ts

@@ -201,6 +201,26 @@ export class BuilderApiClient {
     }
   }
 
+  /**
+   * Establish the connection to every known builder outside the bid deadline, a cold TCP/TLS handshake
+   * to a distant builder can exhaust `BUILDER_BID_DEADLINE_MS` on its own, and a long idle connection
+   * may be half-open and only fail once written to.
+   */
+  async checkStatus(): Promise<void> {
+    await Promise.all(
+      Array.from(this.clients.entries()).map(async ([url, client]) => {
+        try {
+          (await client.status()).assertOk();
+          this.metrics?.builderApi.statusChecks.inc({status: "success"});
+          this.logger?.debug("Builder is ready", {builder: toPrintableUrl(url)});
+        } catch (e) {
+          this.metrics?.builderApi.statusChecks.inc({status: "error"});
+          this.logger?.warn("Builder status check failed", {builder: toPrintableUrl(url)}, e as Error);
+        }
+      })
+    );
+  }
+
   private async getOrCreateClient(
     url: BuilderUrl,
     proposerPubkey: BLSPubkey,
@@ -237,7 +257,7 @@ export class BuilderApiClient {
         {config: this.config, metrics: this.metrics?.builderHttpClient, logger: this.logger}
       );
       this.clients.set(url, client);
-      this.logger?.info("External builder registered", {url: toPrintableUrl(url)});
+      this.logger?.debug("External builder registered", {url: toPrintableUrl(url)});
     }
     return client;
   }

package/src/execution/builder/validateBid.ts

@@ -37,10 +37,11 @@ export async function validateBuilderApiExecutionPayloadBid(
     parentBlockHash: RootHex;
     parentBlockRoot: RootHex;
     entry: routes.validator.BuilderEntry;
+    getParentExecutionRequests: () => Promise<gloas.ExecutionRequests>;
   }
 ): Promise<void> {
   const bid = signedExecutionPayloadBid.message;
-  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry} = request;
+  const {slot, parentBlock, parentBlockHash, parentBlockRoot, entry, getParentExecutionRequests} = request;
 
   if (bid.slot !== slot) {
     throw Error(`Bid slot=${bid.slot} does not match requested slot=${slot}`);
@@ -55,6 +56,16 @@ export async function validateBuilderApiExecutionPayloadBid(
     );
   }
 
+  if (byteArrayEquals(bid.blockHash, bid.parentBlockHash)) {
+    throw Error(`Bid block hash ${toRootHex(bid.blockHash)} must not equal its parent block hash`);
+  }
+
+  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
+  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
+  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
+    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
+  }
+
   const totalPayment = getBuilderBidTotalGwei(bid, entry.maxExecutionPayment);
   if (totalPayment < entry.minBid) {
     throw Error(
@@ -99,12 +110,6 @@ export async function validateBuilderApiExecutionPayloadBid(
     );
   }
 
-  const blobKzgCommitmentsLen = bid.blobKzgCommitments.length;
-  const maxBlobsPerBlock = chain.config.getMaxBlobsPerBlock(computeEpochAtSlot(bid.slot));
-  if (blobKzgCommitmentsLen > maxBlobsPerBlock) {
-    throw Error(`Bid has too many KZG commitments len=${blobKzgCommitmentsLen} limit=${maxBlobsPerBlock}`);
-  }
-
   // The coverage check only applies to the staked collateral payment, a pure execution
   // layer payment bid has nothing to cover on-chain
   if (bid.value > 0 && !state.canBuilderCoverBid(bid.builderIndex, bid.value)) {
@@ -118,6 +123,20 @@ export async function validateBuilderApiExecutionPayloadBid(
     throw Error(`Invalid bid prevRandao=${toHex(bid.prevRandao)} expected=${toHex(randaoMix)}`);
   }
 
+  // The parent's payload does not try to exit the builder
+  if (byteArrayEquals(bid.parentBlockHash, state.latestExecutionPayloadBid.blockHash)) {
+    const requests = await getParentExecutionRequests();
+    if (
+      requests.builderExits.some(
+        (request) =>
+          byteArrayEquals(request.pubkey, builder.pubkey) &&
+          byteArrayEquals(request.sourceAddress, builder.executionAddress)
+      )
+    ) {
+      throw Error(`Bid builderIndex=${bid.builderIndex} may exit in the parent payload`);
+    }
+  }
+
   // The builder must honor the proposer preferences it learned over gossip
   const bidEpoch = computeEpochAtSlot(bid.slot);
   const dependentRootHex = (() => {

package/src/execution/engine/mock.ts

@@ -296,7 +296,10 @@ export class ExecutionEngineMockBackend implements JsonRpcBackend {
 
     // 5. Client software MUST update its forkchoice state if payloads referenced by forkchoiceState.headBlockHash and
     //    forkchoiceState.finalizedBlockHash are VALID.
-    if (!this.validBlocks.has(finalizedBlockHash)) {
+    //
+    // A zero finalizedBlockHash means "no finalized execution block" (e.g. the finalized checkpoint has no execution
+    // payload, pre-merge/pre-TTD). Real EL clients accept this, so the mock must not treat it as an unknown block.
+    if (finalizedBlockHash !== ZERO_HASH_HEX && !this.validBlocks.has(finalizedBlockHash)) {
       throw Error(`Unknown finalizedBlockHash ${finalizedBlockHash}`);
     }
     this.headBlockHash = headBlockHash;

package/src/metrics/metrics/lodestar.ts

@@ -2,6 +2,7 @@
 import {NotReorgedReason} from "@lodestar/fork-choice";
 import {ArchiveStoreTask} from "../../chain/archiveStore/archiveStore.js";
 import {FrequencyStateArchiveStep} from "../../chain/archiveStore/strategies/frequencyStateArchiveStrategy.js";
+import type {LateCanonicalBlockReason} from "../../chain/archiveStore/utils/archiveBlocks.js";
 import {BlockInputSource} from "../../chain/blocks/blockInput/index.js";
 import {PayloadErrorCode} from "../../chain/blocks/importExecutionPayload.js";
 import {
@@ -20,6 +21,7 @@ import {ReprocessStatus} from "../../chain/reprocess.js";
 import {RejectReason} from "../../chain/seenCache/seenAttestationData.js";
 import {CacheItemType} from "../../chain/stateCache/types.js";
 import {OpSource} from "../../chain/validatorMonitor.js";
+import type {FlatFileStoreOperation} from "../../db/flatFileStore/metrics.js";
 import {ExecutionPayloadStatus} from "../../execution/index.js";
 import {GossipType} from "../../network/index.js";
 import {CannotAcceptWorkReason, ReprocessRejectReason} from "../../network/processor/index.js";
@@ -83,15 +85,15 @@ export function createLodestarMetrics(
       }),
       jobTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_time_seconds",
-        help: "Time to process gossip validation queue job in seconds",
+        help: "Time to process gossip validation queue job in seconds (accepted messages only)",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       jobWaitTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_job_wait_time_seconds",
         help: "Time from job added to the queue to starting the job in seconds",
         labelNames: ["topic"],
-        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2, 5, 10],
+        buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 2],
       }),
       concurrency: register.gauge<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_concurrency",
@@ -104,9 +106,10 @@ export function createLodestarMetrics(
         help: "Age of the first item of each key in the indexed queues in seconds",
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
-      queueTime: register.histogram({
+      queueTime: register.histogram<{topic: GossipType}>({
         name: "lodestar_gossip_validation_queue_time_seconds",
         help: "Total time an item stays in queue until it is processed in seconds",
+        labelNames: ["topic"],
         buckets: [0.01, 0.02, 0.05, 0.1, 0.2, 0.5, 1, 5],
       }),
     },
@@ -920,6 +923,12 @@ export function createLodestarMetrics(
         help: "Count of errors, by error type, while processing blocks",
         labelNames: ["error"],
       }),
+
+      preStateSource: register.counter<{source: "parentState" | "fallbackPreState" | "preState"}>({
+        name: "lodestar_gossip_block_validation_pre_state_source_total",
+        help: "Source of the pre-state used for gossip block validation",
+        labelNames: ["source"],
+      }),
     },
     gossipBlob: {
       recvToValidation: register.histogram({
@@ -1040,6 +1049,11 @@ export function createLodestarMetrics(
         name: "lodestar_import_block_set_head_after_cutoff_total",
         help: "Total times an imported block is set as head after ATTESTATION_DUE_BPS of the slot",
       }),
+      lateCanonicalBlock: register.counter<{reason: LateCanonicalBlockReason}>({
+        name: "lodestar_import_block_late_canonical_total",
+        help: "Total finalized-canonical blocks this node imported after the attestation cutoff; reason distinguishes this node's processing lag (slow_import) from late reception (late_receive)",
+        labelNames: ["reason"],
+      }),
       bySource: register.gauge<{source: BlockInputSource}>({
         name: "lodestar_import_block_by_source_total",
         help: "Total number of imported blocks by source",
@@ -1345,10 +1359,10 @@ export function createLodestarMetrics(
           help: "Total number of InsertOutcome as a result of adding an execution payload bid from gossip to the pool",
           labelNames: ["insertOutcome"],
         }),
-        apiInsertOutcome: register.counter<{insertOutcome: InsertOutcome}>({
-          name: "lodestar_oppool_execution_payload_bid_pool_api_insert_outcome_total",
-          help: "Total number of InsertOutcome as a result of adding an execution payload bid from api to the pool",
-          labelNames: ["insertOutcome"],
+        apiValidationTime: register.histogram({
+          name: "lodestar_api_execution_payload_bid_validation_time_seconds",
+          help: "Time elapsed for signed execution payload bid validation - api path",
+          buckets: [0.005, 0.01, 0.025, 0.05, 0.1, 0.5],
         }),
       },
     },
@@ -2030,6 +2044,11 @@ export function createLodestarMetrics(
     },
 
     builderApi: {
+      statusChecks: register.counter<{status: "success" | "error"}>({
+        name: "lodestar_builder_api_status_checks_total",
+        help: "Total count of status checks sent to external builders ahead of a proposal",
+        labelNames: ["status"],
+      }),
       bidRequests: register.counter({
         name: "lodestar_builder_api_bid_requests_total",
         help: "Total count of execution payload bid requests sent to external builders",
@@ -2081,7 +2100,7 @@ export function createLodestarMetrics(
       }),
       dbSizeTotal: register.gauge({
         name: "lodestar_db_size_bytes_total",
-        help: "Approximate number of bytes of file system space used by db",
+        help: "Approximate number of bytes of file system space used by LevelDB",
       }),
       dbApproximateSizeTime: register.histogram({
         name: "lodestar_db_approximate_size_time_seconds",
@@ -2090,6 +2109,61 @@ export function createLodestarMetrics(
       }),
     },
 
+    flatFileStore: {
+      operationDuration: {
+        read: register.histogram({
+          name: "lodestar_flat_file_store_read_duration_seconds",
+          help: "Duration of flat file store reads in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        write: register.histogram({
+          name: "lodestar_flat_file_store_write_duration_seconds",
+          help: "Duration of flat file store writes in seconds",
+          buckets: [0.005, 0.01, 0.05, 0.1, 0.5, 1],
+        }),
+        delete: register.histogram({
+          name: "lodestar_flat_file_store_delete_duration_seconds",
+          help: "Duration of flat file store deletions in seconds",
+          buckets: [0.0001, 0.0005, 0.001, 0.005, 0.01, 0.05],
+        }),
+        prune: register.histogram({
+          name: "lodestar_flat_file_store_prune_duration_seconds",
+          help: "Duration of flat file store pruning in seconds",
+          buckets: [0.0005, 0.01, 0.05, 0.5, 5, 30],
+        }),
+      },
+      operationErrors: register.counter<{operation: FlatFileStoreOperation}>({
+        name: "lodestar_flat_file_store_operation_errors_total",
+        help: "Total count of failed flat file store operations",
+        labelNames: ["operation"],
+      }),
+      readBytes: register.counter({
+        name: "lodestar_flat_file_store_read_bytes_total",
+        help: "Total bytes read from flat file storage",
+      }),
+      writeBytes: register.counter({
+        name: "lodestar_flat_file_store_write_bytes_total",
+        help: "Total bytes written to flat file storage",
+      }),
+      prunedDirectories: register.counter({
+        name: "lodestar_flat_file_store_pruned_directories_total",
+        help: "Total count of slot directories pruned from flat file storage",
+      }),
+      slotIndexSize: register.gauge({
+        name: "lodestar_flat_file_store_indexed_slots",
+        help: "Count of indexed flat file slot directories, including empty directories awaiting pruning",
+      }),
+      startupDuration: register.histogram({
+        name: "lodestar_flat_file_store_startup_duration_seconds",
+        help: "Duration of flat file store slot index reconstruction in seconds",
+        buckets: [0.1, 0.5, 1, 5, 10, 30, 60, 300],
+      }),
+      startupErrors: register.counter({
+        name: "lodestar_flat_file_store_startup_errors_total",
+        help: "Total count of flat file store startup failures",
+      }),
+    },
+
     pruneHistory: {
       pruneCount: register.gauge({
         name: "lodestar_prune_history_prune_count_total",

package/src/monitoring/clientStats.ts

@@ -107,7 +107,7 @@ function createBeaconNodeStats(): ClientStats {
       metricName: "lodestar_db_size_bytes_total",
       jsonType: JsonType.Number,
       defaultValue: 0,
-      description: "Amount of bytes consumed on disk by the beacon node's database",
+      description: "Amount of bytes consumed on disk by the beacon node's LevelDB database",
     }),
     networkLibp2pBytesTotalReceive: new MetricProperty({
       jsonKey: "network_libp2p_bytes_total_receive",

package/src/network/gossip/encoding.ts

@@ -123,9 +123,6 @@ export class DataTransformSnappy implements DataTransform {
     if (uncompressedDataLength > maxSize) {
       throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${maxSize}`);
     }
-    if (uncompressedDataLength > sszType.maxSize) {
-      throw Error(`ssz_snappy decoded data length ${uncompressedDataLength} > ${sszType.maxSize}`);
-    }
 
     // Only after sanity length checks, we can decompress the data
     // Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
@@ -146,9 +143,6 @@ export class DataTransformSnappy implements DataTransform {
     if (data.length > maxSize) {
       throw Error(`ssz_snappy encoded data length ${data.length} > ${maxSize}`);
     }
-    if (data.length > sszType.maxSize) {
-      throw Error(`ssz_snappy encoded data length ${data.length} > ${sszType.maxSize}`);
-    }
 
     // Using Buffer.alloc() instead of Buffer.allocUnsafe() to mitigate high GC pressure observed in some environments
     const compressedData = Buffer.alloc(snappyWasm.max_compress_len(data.length));

package/src/network/gossip/topic.ts

@@ -4,17 +4,11 @@ import {
   ATTESTATION_SUBNET_COUNT,
   ForkName,
   ForkSeq,
-  MAX_ATTESTER_SLASHING_SIZE,
-  MAX_DATA_COLUMN_SIDECAR_SIZE,
-  MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE,
-  MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE,
-  MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE,
   SYNC_COMMITTEE_SUBNET_COUNT,
   isForkPostAltair,
   isForkPostElectra,
   isForkPostFulu,
   isForkPostGloas,
-  isForkPostHeze,
 } from "@lodestar/params";
 import {Attestation, SingleAttestation, ssz, sszTypesFor} from "@lodestar/types";
 import {GossipAction, GossipActionError, GossipErrorCode} from "../../chain/errors/gossipValidation.js";
@@ -139,27 +133,11 @@ export function getGossipSSZType(topic: GossipTopic) {
 }
 
 /**
- * Return the maximum uncompressed SSZ byte length accepted for a gossip object.
+ * Return the maximum uncompressed SSZ byte length accepted for a gossip object, the SSZ type max size
+ * or MAX_PAYLOAD_SIZE, whichever is smaller.
  */
 export function getGossipSSZMaxSize(topic: GossipTopic, maxPayloadSize: number, sszType?: CompositeTypeAny): number {
-  const {fork} = topic.boundary;
-  // Gloas progressive containers have broad theoretical SSZ max sizes; use the preset p2p bounds instead.
-  switch (topic.type) {
-    case GossipType.beacon_block:
-      return maxPayloadSize;
-    case GossipType.beacon_aggregate_and_proof:
-      return isForkPostGloas(fork) ? MAX_SIGNED_AGGREGATE_AND_PROOF_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.attester_slashing:
-      return isForkPostGloas(fork) ? MAX_ATTESTER_SLASHING_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.data_column_sidecar:
-      return isForkPostGloas(fork) ? MAX_DATA_COLUMN_SIDECAR_SIZE : (sszType ?? getGossipSSZType(topic)).maxSize;
-    case GossipType.execution_payload:
-      return maxPayloadSize;
-    case GossipType.execution_payload_bid:
-      return isForkPostHeze(fork) ? MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE_HEZE : MAX_SIGNED_EXECUTION_PAYLOAD_BID_SIZE;
-    default:
-      return (sszType ?? getGossipSSZType(topic)).maxSize;
-  }
+  return Math.min((sszType ?? getGossipSSZType(topic)).maxSize, maxPayloadSize);
 }
 
 /**

package/src/network/network.ts

@@ -481,7 +481,7 @@ export class Network implements INetwork {
   }
 
   async publishLightClientFinalityUpdate(update: LightClientFinalityUpdate): Promise<number> {
-    const epoch = computeEpochAtSlot(update.signatureSlot);
+    const epoch = computeEpochAtSlot(update.attestedHeader.beacon.slot);
     const boundary = this.config.getForkBoundaryAtEpoch(epoch);
 
     return this.publishGossip<GossipType.light_client_finality_update>(
@@ -491,7 +491,7 @@ export class Network implements INetwork {
   }
 
   async publishLightClientOptimisticUpdate(update: LightClientOptimisticUpdate): Promise<number> {
-    const epoch = computeEpochAtSlot(update.signatureSlot);
+    const epoch = computeEpochAtSlot(update.attestedHeader.beacon.slot);
     const boundary = this.config.getForkBoundaryAtEpoch(epoch);
 
     return this.publishGossip<GossipType.light_client_optimistic_update>(
@@ -516,7 +516,8 @@ export class Network implements INetwork {
 
     return this.publishGossip<GossipType.execution_payload_bid>(
       {type: GossipType.execution_payload_bid, boundary},
-      signedBid
+      signedBid,
+      {floodPublish: true}
     );
   }
 
@@ -542,17 +543,19 @@ export class Network implements INetwork {
 
   private async publishGossip<K extends GossipType>(
     topic: GossipTopicMap[K],
-    object: GossipTypeMap[K]
+    object: GossipTypeMap[K],
+    opts?: PublishOpts
   ): Promise<number> {
     const topicStr = stringifyGossipTopic(this.config, topic);
     const sszType = getGossipSSZType(topic);
     const messageData = (sszType.serialize as (object: GossipTypeMap[GossipType]) => Uint8Array)(object);
-    const opts: PublishOpts = {
+    const publishOpts: PublishOpts = {
       ignoreDuplicatePublishError: gossipTopicIgnoreDuplicatePublishError[topic.type],
       // Leave undefined unless the topic opts out, so `--network.allowPublishToZeroPeers` still applies
       allowPublishToZeroTopicPeers: gossipTopicAllowPublishToZeroPeers[topic.type] ? true : undefined,
+      ...opts,
     };
-    const sentPeers = await this.core.publishGossip(topicStr, messageData, opts);
+    const sentPeers = await this.core.publishGossip(topicStr, messageData, publishOpts);
 
     this.logger.verbose("Publish to topic", {topic: topicStr, sentPeers, currentSlot: this.clock.currentSlot});
     return sentPeers;