@lodestar/beacon-node 1.46.0-rc.0 → 1.47.0-dev.d2f4b96574

package/src/chain/blocks/importExecutionPayload.ts

@@ -250,7 +250,7 @@ export async function importExecutionPayload(
   const head = this.forkChoice.getHead();
   if (!this.opts.disableImportExecutionFcU && blockRootHex === head.blockRoot) {
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     this.executionEngine.notifyForkchoiceUpdate(fork, blockHashHex, safeBlockHash, finalizedBlockHash).catch((e) => {
       if (!isErrorAborted(e) && !isQueueErrorAborted(e)) {
         this.logger.error("Error pushing notifyForkchoiceUpdate()", {blockHashHex, finalizedBlockHash}, e);

package/src/chain/blocks/index.ts

@@ -108,11 +108,9 @@ export async function processBlocks(
       throw segmentExecStatus.execAborted.execError;
     }
 
-    if (opts.skipVerifyBlockSignatures !== true) {
-      for (const blockInput of relevantBlocks) {
-        const block = blockInput.getBlock().message;
-        this.seenBlockProposers.add(block.slot, block.proposerIndex);
-      }
+    for (const blockInput of relevantBlocks) {
+      const block = blockInput.getBlock().message;
+      this.seenBlockProposers.add(block.slot, block.proposerIndex);
     }
 
     const {executionStatuses} = segmentExecStatus;

package/src/chain/blocks/verifyBlock.ts

@@ -6,9 +6,9 @@ import {
   computeEpochAtSlot,
   signedBlockToSignedHeader,
 } from "@lodestar/state-transition";
-import {IndexedAttestation, Slot, deneb, ssz} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {IndexedAttestation, Slot, deneb} from "@lodestar/types";
 import {getBlobKzgCommitments} from "../../util/dataColumns.js";
+import {callInNextEventLoop} from "../../util/eventLoop.js";
 import type {BeaconChain} from "../chain.js";
 import {BlockError, BlockErrorCode} from "../errors/index.js";
 import {BlockProcessOpts} from "../options.js";
@@ -193,34 +193,30 @@ export async function verifyBlocksInEpoch(
       ),
 
       // All signatures at once
-      opts.skipVerifyBlockSignatures !== true
-        ? verifyBlocksSignatures(
-            this.config,
-            this.bls,
-            this.logger,
-            this.metrics,
-            preState0,
-            blocks,
-            indexedAttestationsByBlock,
-            opts
-          )
-        : Promise.resolve({verifySignaturesTime: Date.now()}),
+      verifyBlocksSignatures(
+        this.config,
+        this.bls,
+        this.logger,
+        this.metrics,
+        preState0,
+        blocks,
+        indexedAttestationsByBlock,
+        opts
+      ),
 
       // TODO GLOAS: can verify payload signatures in batch too
       // maybe chain with the above verifyBlocksSignatures()
     ]);
 
-    if (opts.skipVerifyBlockSignatures !== true) {
-      for (const block of blocks) {
-        const {slot, proposerIndex} = block.message;
-        const signedBlockHeader = signedBlockToSignedHeader(this.config, block);
-        const blockRoot = toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(signedBlockHeader.message));
-        this.seenBlockProposers.observeBlockRoot(slot, proposerIndex, blockRoot, signedBlockHeader);
-        // Only produce a slashing while importing the block. A block that is verified before it is published
-        // must not be treated as equivocation evidence since it may never be seen by the network
-        if (opts.verifyOnly !== true && this.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
-          this.processProposerEquivocation(slot, proposerIndex);
-        }
+    for (const blockInput of blockInputs) {
+      const block = blockInput.getBlock();
+      const {slot, proposerIndex} = block.message;
+      const signedBlockHeader = signedBlockToSignedHeader(this.config, block);
+      this.seenBlockProposers.observeBlockRoot(slot, proposerIndex, blockInput.blockRootHex, signedBlockHeader);
+      // Only produce a slashing while importing the block. A block that is verified before it is published
+      // must not be treated as equivocation evidence since it may never be seen by the network
+      if (opts.verifyOnly !== true && this.seenBlockProposers.isEquivocating(slot, proposerIndex)) {
+        callInNextEventLoop(() => this.processProposerEquivocation(slot, proposerIndex));
       }
     }
 

package/src/chain/chain.ts

@@ -1174,11 +1174,15 @@ export class BeaconChain implements IBeaconChain {
 
   /** Produce a proposer slashing from two conflicting signed block headers observed for the same slot and proposer */
   private async produceProposerSlashing(blockSlot: Slot, proposerIndex: ValidatorIndex): Promise<void> {
-    if (
-      this.opts.disableProposerSlashings === true ||
-      this.opPool.hasSeenProposerSlashing(proposerIndex) ||
-      this.producingProposerSlashing.has(proposerIndex)
-    ) {
+    if (this.opts.disableProposerSlashings === true || this.producingProposerSlashing.has(proposerIndex)) {
+      return;
+    }
+
+    if (this.opPool.hasSeenProposerSlashing(proposerIndex)) {
+      this.logger.debug("Not producing proposer slashing, already in the op pool", {
+        slot: blockSlot,
+        proposerIndex,
+      });
       return;
     }
 
@@ -1213,7 +1217,12 @@ export class BeaconChain implements IBeaconChain {
       this.emitter.emit(routes.events.EventType.proposerSlashing, proposerSlashing);
       this.emitter.emit(ChainEvent.publishProposerSlashing, proposerSlashing);
       this.metrics?.opPool.proposerSlashingsProduced.inc();
-      this.logger.info("Produced proposer slashing from observed equivocation", {slot: blockSlot, proposerIndex});
+      this.logger.info("Produced proposer slashing from observed equivocation", {
+        slot: blockSlot,
+        proposerIndex,
+        header1Root: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(header1.message)),
+        header2Root: toRootHex(ssz.phase0.BeaconBlockHeader.hashTreeRoot(header2.message)),
+      });
     } finally {
       this.producingProposerSlashing.delete(proposerIndex);
     }

package/src/chain/options.ts

@@ -86,8 +86,6 @@ export type BlockProcessOpts = {
   verifyOnly?: boolean;
   /** Used to specify to skip execution payload validation */
   skipVerifyExecutionPayload?: boolean;
-  /** Used to specify to skip block signatures validation */
-  skipVerifyBlockSignatures?: boolean;
 };
 
 export type PoolOpts = {

package/src/chain/prepareNextSlot.ts

@@ -224,7 +224,7 @@ export class PrepareNextSlotScheduler {
           this.metrics?.blockPayload.payloadAdvancePrepTime.observe(preparationTime);
 
           const safeBlockHash = getSafeExecutionBlockHash(this.chain.forkChoice, this.logger);
-          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice, this.logger);
+          const finalizedBlockHash = getFinalizedExecutionBlockHash(this.chain.forkChoice);
 
           // awaiting here instead of throwing an async call because there is no other task
           // left for scheduler and this gives nice semantics to catch and log errors in the

package/src/chain/produceBlock/produceBlockBody.ts

@@ -271,7 +271,7 @@ export async function produceBlockBody<T extends BlockType>(
     // full and blinded no longer makes sense in gloas, it might be a good idea to move
     // this into a completely separate function and have pre/post gloas more separated
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     // TODO GLOAS: post-Gloas, proposer feeRecipient is also carried (signed) in
     // ProposerPreferencesPool. Consider using this unified cache instead
     // see https://github.com/ChainSafe/lodestar/issues/9379
@@ -417,7 +417,7 @@ export async function produceBlockBody<T extends BlockType>(
     }
 
     const safeBlockHash = getSafeExecutionBlockHash(this.forkChoice, this.logger);
-    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice, this.logger);
+    const finalizedBlockHash = getFinalizedExecutionBlockHash(this.forkChoice);
     const feeRecipient = requestedFeeRecipient ?? this.beaconProposerCache.getOrDefault(proposerIndex);
     const feeRecipientType = requestedFeeRecipient
       ? "requested"

package/src/chain/seenCache/seenBlockProposers.ts

@@ -3,7 +3,7 @@ import {Epoch, RootHex, Slot, ValidatorIndex, phase0} from "@lodestar/types";
 import {MapDef} from "@lodestar/utils";
 
 /** Two distinct block roots signed by the same proposer for the same slot are sufficient to establish an equivocation */
-const MAX_BLOCK_ROOTS_PER_PROPOSAL = 2;
+const MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL = 2;
 
 /**
  * Keeps a cache to filter block proposals from the same validator in the same slot.
@@ -36,7 +36,8 @@ export class SeenBlockProposers {
 
   isEquivocating(blockSlot: Slot, proposerIndex: ValidatorIndex): boolean {
     return (
-      (this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.size ?? 0) >= MAX_BLOCK_ROOTS_PER_PROPOSAL
+      (this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex)?.size ?? 0) >=
+      MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL
     );
   }
 
@@ -53,7 +54,10 @@ export class SeenBlockProposers {
     proposerIndex: ValidatorIndex
   ): [phase0.SignedBeaconBlockHeader, phase0.SignedBeaconBlockHeader] | null {
     const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.get(blockSlot)?.get(proposerIndex);
-    if (signedBlockHeaderByRoot === undefined || signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL) {
+    if (
+      signedBlockHeaderByRoot === undefined ||
+      signedBlockHeaderByRoot.size < MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL
+    ) {
       return null;
     }
     const [signedHeader1, signedHeader2] = signedBlockHeaderByRoot.values();
@@ -72,7 +76,10 @@ export class SeenBlockProposers {
     }
 
     const signedBlockHeaderByRoot = this.signedBlockHeadersBySlot.getOrDefault(blockSlot).getOrDefault(proposerIndex);
-    if (signedBlockHeaderByRoot.size < MAX_BLOCK_ROOTS_PER_PROPOSAL && !signedBlockHeaderByRoot.has(blockRoot)) {
+    if (
+      signedBlockHeaderByRoot.size < MIN_EQUIVOCATION_BLOCK_ROOTS_PER_PROPOSAL &&
+      !signedBlockHeaderByRoot.has(blockRoot)
+    ) {
       signedBlockHeaderByRoot.set(blockRoot, signedBlockHeader);
     }
   }

package/src/network/core/networkCore.ts

@@ -70,6 +70,15 @@ export type BaseNetworkInit = {
   initialCustodyGroupCount: number;
 };
 
+/** Counts of what is still keeping this thread's event loop alive, e.g. `TCPWrap=3,Timeout=1` */
+function formatActiveResources(): string {
+  const counts = new Map<string, number>();
+  for (const resource of process.getActiveResourcesInfo()) {
+    counts.set(resource, (counts.get(resource) ?? 0) + 1);
+  }
+  return Array.from(counts, ([name, count]) => `${name}=${count}`).join(",");
+}
+
 /**
  * This class is meant to work both:
  * - In a libp2p worker
@@ -289,7 +298,10 @@ export class NetworkCore implements INetworkCore {
     this.attnetsService.close();
     this.syncnetsService.close();
     await this.libp2p.stop();
-    this.logger.debug("network lib2p closed");
+    // Diagnostic for the shutdown hang, this thread can spin in `Environment::CleanupHandles()` on
+    // a handle that never closes and `Worker.terminate()` then never resolves. Diffing this list
+    // between a clean and a stuck shutdown should narrow down which handle it is
+    this.logger.debug("network lib2p closed", {activeResources: formatActiveResources()});
 
     this.closed = true;
   }

package/src/network/core/networkCoreWorkerHandler.ts

@@ -10,6 +10,7 @@ import {BeaconConfig, chainConfigToJson} from "@lodestar/config";
 import type {LoggerNode} from "@lodestar/logger/node";
 import {ResponseIncoming, ResponseOutgoing} from "@lodestar/reqresp";
 import {Status} from "@lodestar/types";
+import {withTimeout} from "@lodestar/utils";
 import {Metrics} from "../../metrics/index.js";
 import {AsyncIterableBridgeCaller, AsyncIterableBridgeHandler} from "../../util/asyncIterableToEvents.js";
 import {PeerIdStr, peerIdFromString} from "../../util/peerId.js";
@@ -58,6 +59,10 @@ type WorkerNetworkCoreModules = WorkerNetworkCoreInitModules & {
 
 const NETWORK_WORKER_EXIT_TIMEOUT_MS = 1000;
 const NETWORK_WORKER_EXIT_RETRY_COUNT = 3;
+/** `getApi().close()` is an unbounded RPC into the worker and runs before `BeaconNode.close()`
+ * archives chain state to disk, so it is bounded here. Set well above the ~2-4s a close takes on
+ * mainnet, tripping it leaves libp2p half torn down */
+const NETWORK_CORE_CLOSE_TIMEOUT_MS = 5000;
 
 /**
  * NetworkCore implementation using a Worker thread
@@ -165,15 +170,25 @@ export class WorkerNetworkCore implements INetworkCore {
 
   async close(): Promise<void> {
     this.modules.logger.debug("closing network core running in network worker");
-    await this.getApi().close();
+    try {
+      await withTimeout(async () => this.getApi().close(), NETWORK_CORE_CLOSE_TIMEOUT_MS);
+    } catch (e) {
+      this.modules.logger.debug("Error closing network core", {}, e as Error);
+    }
     this.modules.logger.debug("terminating network worker");
-    await terminateWorkerThread({
+    const terminated = await terminateWorkerThread({
       worker: this.getApi(),
       retryCount: NETWORK_WORKER_EXIT_RETRY_COUNT,
       retryMs: NETWORK_WORKER_EXIT_TIMEOUT_MS,
       logger: this.modules.logger,
     });
-    this.modules.logger.debug("terminated network worker");
+    if (terminated) {
+      this.modules.logger.debug("terminated network worker");
+    } else {
+      // Nothing more to do, `process.exit()` joins the thread regardless so the process still waits
+      // on it. What matters is returning, so the caller can archive state and close the db
+      this.modules.logger.debug("Network worker did not terminate in time, continuing shutdown without it");
+    }
   }
 
   async test(): Promise<void> {

package/src/sync/range/batch.ts

@@ -593,7 +593,7 @@ export class Batch {
     if (allComplete) {
       const attempt: Attempt = {
         peers: this.getSuccessfulPeers(),
-        hash: hashBlocks(blocks, this.config),
+        hash: hashBlocks(blocks, payloadEnvelopes),
       };
       this.state = {status: BatchStatus.AwaitingProcessing, blocks, payloadEnvelopes: newPayloadEnvelopes, attempt};
     } else {

package/src/sync/range/utils/hashBlocks.ts

@@ -1,27 +1,62 @@
-import {ChainForkConfig} from "@lodestar/config";
-import {RootHex, SignedBeaconBlock} from "@lodestar/types";
-import {toRootHex} from "@lodestar/utils";
+import {digest} from "@chainsafe/as-sha256";
+import {RootHex, Slot, ssz} from "@lodestar/types";
+import {fromHex, toRootHex} from "@lodestar/utils";
 import {IBlockInput} from "../../../chain/blocks/blockInput/types.js";
+import {PayloadEnvelopeInput} from "../../../chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js";
+
+const ROOT_SIZE = 32;
+const SIGNATURE_SIZE = 96;
+/** Every block and every payload envelope contributes a fixed-size (root, signature) entry */
+const ENTRY_SIZE = ROOT_SIZE + SIGNATURE_SIZE;
+/** uint32 LE counts of block and envelope entries, so the variable-length sections are self-describing */
+const COUNT_SIZE = 4;
+const HEADER_SIZE = COUNT_SIZE * 2;
 
 /**
- * String to uniquely identify block segments. Used for peer scoring and to compare if batches are equivalent.
+ * Root uniquely identifying a batch attempt (its blocks AND payloads). Used for peer scoring and to
+ * compare if two attempts are equivalent.
+ *
+ * Signatures are part of the id for BOTH blocks and payload envelopes, not just their message roots:
+ * a peer serving a correct message with a garbage signature would otherwise produce the same id as
+ * the honest attempt and escape scoring in `SyncChain.advanceChain`. Signatures are not verified at
+ * download time, only during processChainSegment (block signatures in verifyBlocksSignatures,
+ * envelope signatures in importExecutionPayload).
+ *
+ * Entries are written as raw bytes into a single exactly-sized buffer and digested once, so an
+ * `Attempt` retains 32 bytes rather than the ~16 KB a full 32-slot gloas batch would need if the
+ * roots and signatures were concatenated as hex. sha256 (not a 64-bit hash) because a peer that
+ * could force a collision with the winning attempt would escape peer scoring.
  */
-export function hashBlocks(blocks: IBlockInput[], config: ChainForkConfig): RootHex {
-  switch (blocks.length) {
-    case 0:
-      return "0x";
-    case 1: {
-      const block0 = blocks[0].getBlock();
-      return toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0));
-    }
-    default: {
-      const block0 = blocks[0].getBlock();
-      const blockN = blocks.at(-1)?.getBlock() as SignedBeaconBlock;
-      return (
-        // TODO(fulu): should we be doing checks for presence to make sure these do not blow up?
-        toRootHex(config.getForkTypes(block0.message.slot).SignedBeaconBlock.hashTreeRoot(block0)) +
-        toRootHex(config.getForkTypes(blockN.message.slot).SignedBeaconBlock.hashTreeRoot(blockN))
-      );
-    }
+export function hashBlocks(blocks: IBlockInput[], payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null): RootHex {
+  // Envelopes without a payload carry no attributable data, so they are skipped entirely. A `null`
+  // map and a map whose entries all lack a payload are therefore equivalent, which is intended.
+  const envelopes =
+    payloadEnvelopes && payloadEnvelopes.size > 0
+      ? Array.from(payloadEnvelopes.entries())
+          .filter(([, envelope]) => envelope.hasPayloadEnvelope())
+          .sort(([slotA], [slotB]) => slotA - slotB)
+      : [];
+
+  const buf = Buffer.allocUnsafe(HEADER_SIZE + (blocks.length + envelopes.length) * ENTRY_SIZE);
+  buf.writeUInt32LE(blocks.length, 0);
+  buf.writeUInt32LE(envelopes.length, COUNT_SIZE);
+  let offset = HEADER_SIZE;
+
+  for (const block of blocks) {
+    buf.set(fromHex(block.blockRootHex), offset);
+    offset += ROOT_SIZE;
+    buf.set(block.getBlock().signature, offset);
+    offset += SIGNATURE_SIZE;
   }
+
+  for (const [, envelope] of envelopes) {
+    const signedEnvelope = envelope.getPayloadEnvelope();
+    // envelope's root is cached thanks to cachePermanentRootStruct, so this avoids re-hashing
+    buf.set(ssz.gloas.ExecutionPayloadEnvelope.hashTreeRoot(signedEnvelope.message), offset);
+    offset += ROOT_SIZE;
+    buf.set(signedEnvelope.signature, offset);
+    offset += SIGNATURE_SIZE;
+  }
+
+  return toRootHex(digest(buf));
 }

package/src/util/workerEvents.ts

@@ -111,6 +111,12 @@ export function wireEventsOnMainThread<EventData>(
   }
 }
 
+/**
+ * Terminate a worker thread, bounded to `retryCount * retryMs`.
+ *
+ * @returns `false` if it could not be terminated, the worker thread is then still running and the
+ * caller has to decide how to proceed without it.
+ */
 export async function terminateWorkerThread({
   worker,
   retryMs,
@@ -121,7 +127,7 @@ export async function terminateWorkerThread({
   retryMs: number;
   retryCount: number;
   logger?: Logger;
-}): Promise<void> {
+}): Promise<boolean> {
   const terminated = new Promise((resolve) => {
     Thread.events(worker).subscribe((event) => {
       if (event.type === "termination") {
@@ -131,13 +137,19 @@ export async function terminateWorkerThread({
   });
 
   for (let i = 0; i < retryCount; i++) {
-    await Thread.terminate(worker);
-    const result = await Promise.race([terminated, sleep(retryMs).then(() => false)]);
+    // `Worker.terminate()` resolves only once the worker thread exits, and the thread can spin
+    // forever in `Environment::CleanupHandles()` when a libuv handle on its loop never closes, so
+    // it has to be raced too, otherwise `retryCount * retryMs` never applies.
+    const result = await Promise.race([
+      Thread.terminate(worker).then(() => terminated),
+      sleep(retryMs).then(() => false),
+    ]);
 
-    if (result) return;
+    if (result) return true;
 
-    logger?.warn("Worker thread failed to terminate, retrying...");
+    logger?.debug("Worker thread failed to terminate, retrying...");
   }
 
-  throw new Error(`Worker thread failed to terminate in ${retryCount * retryMs}ms.`);
+  logger?.debug(`Worker thread failed to terminate in ${retryCount * retryMs}ms`);
+  return false;
 }