@lodestar/beacon-node 1.51.0-dev.d422f387ac → 1.51.0-dev.d5620be4a1

package/package.json

@@ -11,7 +11,7 @@
   "bugs": {
     "url": "https://github.com/ChainSafe/lodestar/issues"
   },
-  "version": "1.51.0-dev.d422f387ac",
+  "version": "1.51.0-dev.d5620be4a1",
   "type": "module",
   "exports": {
     ".": {
@@ -126,18 +126,18 @@
     "@libp2p/prometheus-metrics": "^5.0.31",
     "@libp2p/tcp": "^11.0.30",
     "@libp2p/utils": "^7.4.3",
-    "@lodestar/api": "^1.51.0-dev.d422f387ac",
-    "@lodestar/config": "^1.51.0-dev.d422f387ac",
-    "@lodestar/db": "^1.51.0-dev.d422f387ac",
-    "@lodestar/fork-choice": "^1.51.0-dev.d422f387ac",
-    "@lodestar/logger": "^1.51.0-dev.d422f387ac",
-    "@lodestar/native": "^1.51.0-dev.d422f387ac",
-    "@lodestar/params": "^1.51.0-dev.d422f387ac",
-    "@lodestar/reqresp": "^1.51.0-dev.d422f387ac",
-    "@lodestar/state-transition": "^1.51.0-dev.d422f387ac",
-    "@lodestar/types": "^1.51.0-dev.d422f387ac",
-    "@lodestar/utils": "^1.51.0-dev.d422f387ac",
-    "@lodestar/validator": "^1.51.0-dev.d422f387ac",
+    "@lodestar/api": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/config": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/db": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/fork-choice": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/logger": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/native": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/params": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/reqresp": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/state-transition": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/types": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/utils": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/validator": "^1.51.0-dev.d5620be4a1",
     "@multiformats/multiaddr": "^13.0.1",
     "datastore-core": "^12.0.1",
     "datastore-level": "^13.0.1",
@@ -159,8 +159,8 @@
     "@ethereumjs/rlp": "^4.0.1",
     "@libp2p/interface-internal": "^3.2.0",
     "@libp2p/logger": "^6.2.15",
-    "@lodestar/spec-test-util": "^1.51.0-dev.d422f387ac",
-    "@lodestar/test-utils": "^1.51.0-dev.d422f387ac",
+    "@lodestar/spec-test-util": "^1.51.0-dev.d5620be4a1",
+    "@lodestar/test-utils": "^1.51.0-dev.d5620be4a1",
     "@types/js-yaml": "^4.0.5",
     "@types/qs": "^6.9.7",
     "@types/tmp": "^0.2.3",
@@ -174,5 +174,5 @@
     "beacon",
     "blockchain"
   ],
-  "gitHead": "b3fa153108a41d4d29ef8cfcd18185ac06db8f1b"
+  "gitHead": "c76975acffbeab8e33a2f1ae9197767e6eac357f"
 }

package/src/api/impl/beacon/blocks/index.ts

@@ -37,7 +37,7 @@ import {
   sszTypesFor,
 } from "@lodestar/types";
 import {fromHex, prettyGweiToEth, sleep, toHex, toRootHex} from "@lodestar/utils";
-import {BlockInputSource, isBlockInputBlobs, isBlockInputColumns} from "../../../../chain/blocks/blockInput/index.js";
+import {BlockInputSource, isBlockInputColumns} from "../../../../chain/blocks/blockInput/index.js";
 import {PayloadEnvelopeInputSource} from "../../../../chain/blocks/payloadEnvelopeInput/index.js";
 import {ImportBlockOpts} from "../../../../chain/blocks/types.js";
 import {verifyBlocksInEpoch} from "../../../../chain/blocks/verifyBlock.js";
@@ -67,7 +67,6 @@ import {validateApiExecutionPayloadEnvelope} from "../../../../chain/validation/
 import {OpSource} from "../../../../chain/validatorMonitor.js";
 import {
   computePreFuluKzgCommitmentsInclusionProof,
-  getBlobSidecars,
   kzgCommitmentToVersionedHash,
   reconstructBlobs,
 } from "../../../../util/blobs.js";
@@ -110,6 +109,9 @@ export function getBeaconBlockApi(
     const signedBlock = signedBlockContents.signedBlock;
     const slot = signedBlock.message.slot;
     const fork = config.getForkName(slot);
+    if (!isForkPostFulu(fork)) {
+      throw new ApiError(400, `Publishing blocks is not supported for pre-fulu fork=${fork}`);
+    }
     const blockRoot = toRootHex(chain.config.getForkTypes(slot).BeaconBlock.hashTreeRoot(signedBlock.message));
 
     const blockForImport = chain.seenBlockInputCache.getByBlock({
@@ -131,40 +133,26 @@ export function getBeaconBlockApi(
       });
     }
 
-    let blobSidecars: deneb.BlobSidecars, dataColumnSidecars: fulu.DataColumnSidecar[];
-
-    if (isDenebBlockContents(signedBlockContents)) {
-      if (isForkPostGloas(fork)) {
-        // After gloas, data columns are not published with the block but when publishing the execution payload envelope
-        blobSidecars = [];
-        dataColumnSidecars = [];
-      } else if (isForkPostFulu(fork)) {
-        const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
-        // If the block was produced by this node, we will already have computed cells
-        // Otherwise, we will compute them from the blobs in this function
-        const cells =
-          (chain.blockProductionCache.get(blockRoot) as ProduceFullFulu)?.cells ??
-          signedBlockContents.blobs.map((blob) => kzg.computeCells(blob));
-        const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
-          cells: rowCells,
-          proofs: signedBlockContents.kzgProofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
-        }));
-        dataColumnSidecars = getDataColumnSidecarsFromBlock(
-          config,
-          signedBlock as SignedBeaconBlock<ForkPostFulu>,
-          cellsAndProofs
-        ) as fulu.DataColumnSidecar[];
-        timer?.();
-        blobSidecars = [];
-      } else if (isForkPostDeneb(fork)) {
-        blobSidecars = getBlobSidecars(config, signedBlock, signedBlockContents.blobs, signedBlockContents.kzgProofs);
-        dataColumnSidecars = [];
-      } else {
-        throw Error(`Invalid data fork=${fork} for publish`);
-      }
-    } else {
-      blobSidecars = [];
-      dataColumnSidecars = [];
+    // After gloas, data columns are not published with the block but when publishing the execution payload envelope.
+    let dataColumnSidecars: fulu.DataColumnSidecar[] = [];
+
+    if (isDenebBlockContents(signedBlockContents) && isForkPostFulu(fork) && !isForkPostGloas(fork)) {
+      const timer = metrics?.peerDas.dataColumnSidecarComputationTime.startTimer();
+      // If the block was produced by this node, we will already have computed cells
+      // Otherwise, we will compute them from the blobs in this function
+      const cells =
+        (chain.blockProductionCache.get(blockRoot) as ProduceFullFulu)?.cells ??
+        signedBlockContents.blobs.map((blob) => kzg.computeCells(blob));
+      const cellsAndProofs = cells.map((rowCells, rowIndex) => ({
+        cells: rowCells,
+        proofs: signedBlockContents.kzgProofs.slice(rowIndex * NUMBER_OF_COLUMNS, (rowIndex + 1) * NUMBER_OF_COLUMNS),
+      }));
+      dataColumnSidecars = getDataColumnSidecarsFromBlock(
+        config,
+        signedBlock as SignedBeaconBlock<ForkPostFulu>,
+        cellsAndProofs
+      ) as fulu.DataColumnSidecar[];
+      timer?.();
     }
 
     if (isBlockInputColumns(blockForImport)) {
@@ -182,19 +170,6 @@ export function getBeaconBlockApi(
           {throwOnDuplicateAdd: false}
         );
       }
-    } else if (isBlockInputBlobs(blockForImport)) {
-      for (const blobSidecar of blobSidecars) {
-        blockForImport.addBlob(
-          {
-            blockRootHex: blockRoot,
-            blobSidecar,
-            source: BlockInputSource.api,
-            seenTimestampSec,
-          },
-          // Same as above for columns
-          {throwOnDuplicateAdd: false}
-        );
-      }
     }
 
     // check what validations have been requested before broadcasting and publishing the block
@@ -384,7 +359,6 @@ export function getBeaconBlockApi(
       //
       () => network.publishBeaconBlock(signedBlock),
       ...dataColumnSidecars.map((dataColumnSidecar) => () => network.publishDataColumnSidecar(dataColumnSidecar)),
-      ...blobSidecars.map((blobSidecar) => () => network.publishBlobSidecar(blobSidecar)),
       () =>
         // there is no rush to persist block since we published it to gossip anyway
         chain.processBlock(blockForImport, opts).catch((e) => {
@@ -454,20 +428,6 @@ export function getBeaconBlockApi(
           });
         }
       }
-    } else if (isBlockInputBlobs(blockForImport) && chain.emitter.listenerCount(routes.events.EventType.blobSidecar)) {
-      const blobSidecars = blockForImport.getBlobs();
-      const versionedHashes = blockForImport.getVersionedHashes();
-
-      for (const blobSidecar of blobSidecars) {
-        const {index, kzgCommitment} = blobSidecar;
-        chain.emitter.emit(routes.events.EventType.blobSidecar, {
-          blockRoot,
-          slot,
-          index,
-          kzgCommitment: toHex(kzgCommitment),
-          versionedHash: toHex(versionedHashes[index]),
-        });
-      }
     }
   };
 

package/src/api/impl/beacon/pool/index.ts

@@ -123,6 +123,32 @@ export function getBeaconPoolApi({
               logger.debug("Error adding unaggregated attestation to pool", {subnet}, e as Error);
             }
 
+            // Collected only for vote slots that an upcoming block of a connected validator can include
+            if (isForkPostElectra(fork) && chain.proposerTracker.shouldCollect(slot)) {
+              try {
+                const insertOutcome = chain.singleAttestationPool.add(
+                  attestation as SingleAttestation<ForkPostElectra>,
+                  attDataRootHex,
+                  validatorCommitteeIndex,
+                  committeeSize
+                );
+                metrics?.opPool.singleAttestationPool.apiInsertOutcome.inc({insertOutcome});
+              } catch (e) {
+                logger.debug(
+                  "Error adding single attestation to pool",
+                  {
+                    slot,
+                    committeeIndex,
+                    validatorCommitteeIndex,
+                    validatorIndex: (attestation as SingleAttestation<ForkPostElectra>).attesterIndex,
+                    subnet,
+                    attDataRoot: attDataRootHex,
+                  },
+                  e as Error
+                );
+              }
+            }
+
             if (isForkPostElectra(fork)) {
               chain.emitter.emit(
                 routes.events.EventType.singleAttestation,

package/src/api/impl/validator/index.ts

@@ -94,7 +94,7 @@ import {
 import {validateGossipProposerPreferences} from "../../../chain/validation/proposerPreferences.js";
 import {validateSyncCommitteeGossipContributionAndProof} from "../../../chain/validation/syncCommitteeContributionAndProof.js";
 import {ZERO_HASH} from "../../../constants/index.js";
-import {BUILDER_BID_DEADLINE_MS, BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
+import {BuilderApiBid, decodeBuilderUrl} from "../../../execution/builder/apiClient.js";
 import {BuilderStatus, NoBidReceived} from "../../../execution/builder/http.js";
 import {getBuilderBidTotalGwei, validateBuilderApiExecutionPayloadBid} from "../../../execution/builder/validateBid.js";
 import {validateGossipFnRetryUnknownRoot} from "../../../network/processor/gossipHandlers.js";
@@ -123,10 +123,14 @@ import {
  * A cutoff of 1667 BPS (2 seconds for 12s slots) gives enough time and if there are unexpected delays
  * it ensures we publish in time as proposals after the attestation deadline will likely be orphaned
  * due to proposer boost reorg.
- *
- * TODO GLOAS: re-evaluate cutoff timing due to attestation deadline changes in gloas
  */
 const BLOCK_PRODUCTION_RACE_CUTOFF_BPS = 1667;
+/**
+ * Post-gloas cutoff, kept at 1667 BPS (2 seconds for 12s slots) for now.
+ *
+ * TODO GLOAS: revisit, the block propagates faster as it only commits to a bid and the attestation deadline changed
+ */
+export const BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS = 1667;
 /** Rejection message of the bid block branch when there is no viable bid to commit to */
 const NO_BID_AVAILABLE = "No builder bid available";
 
@@ -1005,33 +1009,50 @@ export function getValidatorApi(
         (!circuitBreakerActive &&
           chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex) !== null);
 
-      // Select the p2p bid once builders had time to bid up, matching the deadline advertised
-      // on builder API bid requests, unless the circuit breaker is active
-      const p2pBidPromise: Promise<PooledExecutionPayloadBid | null> = circuitBreakerActive
-        ? Promise.resolve(null)
-        : sleep(Math.max(0, BUILDER_BID_DEADLINE_MS - chain.clock.msFromSlot(slot))).then(() => {
-            const p2pBid = chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
-            // Discard p2p bids below the proposer's configured floor on the total payment.
-            // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
-            if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
-              logger.info("Best p2p bid below configured minimum", {
-                slot,
-                bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
-                minBid: prettyGweiToEth(builderConfig.minBid),
-              });
-              return null;
-            }
-            return p2pBid;
-          });
+      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
+      // not delay the proposal when a builder bid block is available (and vice versa)
+      const cutoffMs = Math.max(
+        0,
+        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_GLOAS_BPS) - chain.clock.msFromSlot(slot)
+      );
+
+      // Give p2p bids the same time to bid up as granted to builder API bid requests via the advertised
+      // deadline, the pool is read once the deadline passed and the bid requests settled. A late request
+      // does not wait past the race cutoff, a pooled bid is still the fastest fallback if the local block fails
+      const bidDeadlinePromise = circuitBreakerActive
+        ? Promise.resolve()
+        : sleep(Math.min(chain.builderApiClient.bidDeadline, cutoffMs));
 
       // Candidates are ranked by their boosted counted total payment, the p2p bid is governed
       // by the top-level factors and each builder API bid by its own entry. Ties prefer the
       // earliest received builder API bid, so the signed block can be routed back to its
       // builder directly.
       const bestBidPromise: Promise<BidCandidate | null> = (async () => {
-        const [builderApiBids, p2pBid] = await Promise.all([builderApiBidsPromise, p2pBidPromise]);
+        // Past the race cutoff a pooled p2p bid is used right away rather than waiting for pending
+        // builder API requests, the local block is selected at the cutoff anyway if it is ready
+        let [builderApiBids] = await Promise.all([
+          Promise.race([builderApiBidsPromise, sleep(cutoffMs).then(() => null)]),
+          bidDeadlinePromise,
+        ]);
         let parentExecutionRequestsPromise: Promise<gloas.ExecutionRequests> | null = null;
 
+        let p2pBid: PooledExecutionPayloadBid | null = circuitBreakerActive
+          ? null
+          : chain.executionPayloadBidPool.getBestBid(slot, bidParentBlockHash, parentBlockRootHex);
+        // Discard p2p bids below the proposer's configured floor on the total payment.
+        // A p2p bid's total is just its value since gossip validation enforces executionPayment=0.
+        if (p2pBid !== null && BigInt(p2pBid.signedBid.message.value) < builderConfig.minBid) {
+          logger.info("Best p2p bid below configured minimum", {
+            slot,
+            bidValue: prettyGweiToEth(p2pBid.signedBid.message.value),
+            minBid: prettyGweiToEth(builderConfig.minBid),
+          });
+          p2pBid = null;
+        }
+        if (builderApiBids === null) {
+          builderApiBids = p2pBid !== null ? [] : await builderApiBidsPromise;
+        }
+
         const candidates = (
           await Promise.all(
             builderApiBids.map(async ({url, entry, signedBid, receivedMs}): Promise<BidCandidate | null> => {
@@ -1118,13 +1139,6 @@ export function getValidatorApi(
         return fn().finally(() => t?.({source}));
       };
 
-      // Calculate cutoff time based on start of the slot, ensures a slow local payload build does
-      // not delay the proposal when a builder bid block is available (and vice versa)
-      const cutoffMs = Math.max(
-        0,
-        config.getSlotComponentDurationMs(fork, BLOCK_PRODUCTION_RACE_CUTOFF_BPS) - chain.clock.msFromSlot(slot)
-      );
-
       // use abort controller to stop waiting for the bid block if the engine block will be selected
       const controller = new AbortController();
 

package/src/chain/GetBlobsTracker.ts

@@ -4,12 +4,8 @@ import {Logger, pruneSetToMax} from "@lodestar/utils";
 import {BLOB_AND_PROOF_V2_RPC_BYTES} from "../execution/engine/types.js";
 import {IExecutionEngine} from "../execution/index.js";
 import {Metrics} from "../metrics/metrics.js";
-import {
-  DataColumnEngineResult,
-  getBlobSidecarsFromExecution,
-  getDataColumnSidecarsFromExecution,
-} from "../util/execution.js";
-import {IBlockInput, isBlockInputBlobs} from "./blocks/blockInput/index.js";
+import {DataColumnEngineResult, getDataColumnSidecarsFromExecution} from "../util/execution.js";
+import {IBlockInput, isBlockInputColumns} from "./blocks/blockInput/index.js";
 import {PayloadEnvelopeInput} from "./blocks/payloadEnvelopeInput/index.js";
 import {ChainEventEmitter} from "./emitter.js";
 
@@ -20,7 +16,7 @@ const MAX_TRACKED_BLOCK_ROOTS = 64;
 
 export type GetBlobsTrackerInit = {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
@@ -31,7 +27,7 @@ export type GetBlobsTrackerInit = {
  */
 export class GetBlobsTracker {
   logger: Logger;
-  executionEngine: IExecutionEngine;
+  executionEngine: Pick<IExecutionEngine, "getBlobs">;
   emitter: ChainEventEmitter;
   metrics: Metrics | null;
   config: ChainForkConfig;
@@ -51,6 +47,10 @@ export class GetBlobsTracker {
   }
 
   triggerGetBlobs(input: IBlockInput | PayloadEnvelopeInput): void {
+    if (!(input instanceof PayloadEnvelopeInput) && !isBlockInputColumns(input)) {
+      return;
+    }
+
     if (this.activeReconstructions.has(input.blockRootHex)) {
       this.pendingTriggers.add(input.blockRootHex);
       return;
@@ -74,37 +74,6 @@ export class GetBlobsTracker {
       }
     }
 
-    // The request is sent right away, before block processing issues newPayload: ELs that drop a
-    // transaction's blobs once the payload is validated can only answer a request that gets there first
-    if (!(input instanceof PayloadEnvelopeInput) && isBlockInputBlobs(input)) {
-      // there is not preallocation for blob sidecars like there is for columns sidecars so no need to
-      // store the index for the preallocated buffers
-      this.activeReconstructions.add(input.blockRootHex);
-      const logCtx = {slot: input.slot, root: input.blockRootHex};
-      this.logger.verbose("Trigger getBlobsV1 for block", logCtx);
-      let requestFailed = false;
-      getBlobSidecarsFromExecution(this.config, this.executionEngine, this.metrics, this.emitter, input)
-        .then(() => {
-          // EL errors and misses are handled inside the helper, blobs still missing means the attempt failed
-          if (!input.hasAllData()) {
-            requestFailed = true;
-            this.recordFailedAttempt(input.blockRootHex);
-          }
-        })
-        .catch((error) => {
-          this.logger.debug("Error during getBlobsV1 for block", logCtx, error as Error);
-          requestFailed = true;
-          this.recordFailedAttempt(input.blockRootHex);
-        })
-        .finally(() => {
-          this.logger.verbose("Completed getBlobsV1 for block", logCtx);
-          this.activeReconstructions.delete(input.blockRootHex);
-          this.retryPendingTrigger(input, requestFailed);
-        });
-
-      return;
-    }
-
     let freeIndex = this.blobsAndProofsBuffers.findIndex(({inUse}) => !inUse);
     if (freeIndex === -1) {
       freeIndex = this.blobsAndProofsBuffers.length;
@@ -126,6 +95,8 @@ export class GetBlobsTracker {
     const logCtx = {slot: input.slot, root: input.blockRootHex};
     this.logger.verbose("Trigger getBlobsV2 for block", logCtx);
     let requestFailed = false;
+    // The request is sent right away, before block processing issues newPayload: ELs that drop a
+    // transaction's blobs once the payload is validated can only answer a request that gets there first
     getDataColumnSidecarsFromExecution(
       this.config,
       this.executionEngine,