@lodestar/beacon-node 1.37.0-dev.f0ce024c1a → 1.37.0-rc.0

package/src/network/processor/gossipHandlers.ts

@@ -516,6 +516,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         throw new GossipActionError(GossipAction.REJECT, {code: "PRE_DENEB_BLOCK"});
       }
       const blockInput = await validateBeaconBlob(blobSidecar, topic.subnet, peerIdStr, seenTimestampSec);
+      chain.serializedCache.set(blobSidecar, serializedData);
       if (!blockInput.hasBlockAndAllData()) {
         const cutoffTimeMs = getCutoffTimeMs(chain, blobSlot, BLOCK_AVAILABILITY_CUTOFF_MS);
         chain.logger.debug("Received gossip blob, waiting for full data availability", {
@@ -562,6 +563,7 @@ function getSequentialHandlers(modules: ValidatorFnsModules, options: GossipHand
         peerIdStr,
         seenTimestampSec
       );
+      chain.serializedCache.set(dataColumnSidecar, serializedData);
       const blockInputMeta = blockInput.getLogMeta();
       const {receivedColumns} = blockInputMeta;
       // it's not helpful to track every single column received

package/src/network/reqresp/utils/collect.ts

@@ -1,6 +1,6 @@
 import {Type} from "@chainsafe/ssz";
 import {RequestError, RequestErrorCode, ResponseIncoming} from "@lodestar/reqresp";
-import {WithBytes} from "@lodestar/types";
+import {SerializedCache} from "../../../util/serializedCache.ts";
 import {ResponseTypeGetter} from "../types.js";
 
 /**
@@ -32,13 +32,16 @@ export async function collectExactOneTyped<T>(
 export async function collectMaxResponseTyped<T>(
   source: AsyncIterable<ResponseIncoming>,
   maxResponses: number,
-  typeFn: ResponseTypeGetter<T>
+  typeFn: ResponseTypeGetter<T>,
+  serializedCache?: SerializedCache
 ): Promise<T[]> {
   // else: zero or more responses
   const responses: T[] = [];
   for await (const chunk of source) {
     const type = typeFn(chunk.fork, chunk.protocolVersion);
     const response = sszDeserializeResponse(type, chunk.data);
+    // optionally cache the serialized response if the cache is available
+    serializedCache?.set(response as object, chunk.data);
     responses.push(response);
 
     if (maxResponses !== undefined && responses.length >= maxResponses) {
@@ -58,14 +61,17 @@ export async function collectMaxResponseTyped<T>(
 export async function collectMaxResponseTypedWithBytes<T>(
   source: AsyncIterable<ResponseIncoming>,
   maxResponses: number,
-  typeFn: ResponseTypeGetter<T>
-): Promise<WithBytes<T>[]> {
+  typeFn: ResponseTypeGetter<T>,
+  serializedCache?: SerializedCache
+): Promise<T[]> {
   // else: zero or more responses
-  const responses: WithBytes<T>[] = [];
+  const responses: T[] = [];
   for await (const chunk of source) {
     const type = typeFn(chunk.fork, chunk.protocolVersion);
     const data = sszDeserializeResponse(type, chunk.data);
-    responses.push({data, bytes: chunk.data});
+    responses.push(data);
+    // optionally cache the serialized response if the cache is available
+    serializedCache?.set(data as object, chunk.data);
 
     if (maxResponses !== undefined && responses.length >= maxResponses) {
       break;

package/src/network/reqresp/utils/collectSequentialBlocksInRange.ts

@@ -1,6 +1,7 @@
 import {ResponseIncoming} from "@lodestar/reqresp";
-import {SignedBeaconBlock, WithBytes, phase0} from "@lodestar/types";
+import {SignedBeaconBlock, phase0} from "@lodestar/types";
 import {LodestarError} from "@lodestar/utils";
+import {SerializedCache} from "../../../util/serializedCache.ts";
 import {ReqRespMethod, responseSszTypeByMethod} from "../types.js";
 import {sszDeserializeResponse} from "./collect.js";
 
@@ -10,9 +11,10 @@ import {sszDeserializeResponse} from "./collect.js";
  */
 export async function collectSequentialBlocksInRange(
   blockStream: AsyncIterable<ResponseIncoming>,
-  {count, startSlot}: Pick<phase0.BeaconBlocksByRangeRequest, "count" | "startSlot">
-): Promise<WithBytes<SignedBeaconBlock>[]> {
-  const blocks: WithBytes<SignedBeaconBlock>[] = [];
+  {count, startSlot}: Pick<phase0.BeaconBlocksByRangeRequest, "count" | "startSlot">,
+  serializedCache?: SerializedCache
+): Promise<SignedBeaconBlock[]> {
+  const blocks: SignedBeaconBlock[] = [];
 
   for await (const chunk of blockStream) {
     const blockType = responseSszTypeByMethod[ReqRespMethod.BeaconBlocksByRange](chunk.fork, chunk.protocolVersion);
@@ -30,11 +32,13 @@ export async function collectSequentialBlocksInRange(
     }
 
     const prevBlock = blocks.at(-1);
-    if (prevBlock && prevBlock.data.message.slot >= blockSlot) {
+    if (prevBlock && prevBlock.message.slot >= blockSlot) {
       throw new BlocksByRangeError({code: BlocksByRangeErrorCode.BAD_SEQUENCE});
     }
 
-    blocks.push({data: block, bytes: chunk.data});
+    blocks.push(block);
+    // optionally cache the serialized response if the cache is available
+    serializedCache?.set(block, chunk.data);
     if (blocks.length >= count) {
       break; // Done, collected all blocks
     }

package/src/sync/backfill/backfill.ts

@@ -749,27 +749,33 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter}
     const anchorBlock = res[0];
 
     // GENESIS_SLOT doesn't has valid signature
-    if (anchorBlock.data.message.slot === GENESIS_SLOT) return;
+    if (anchorBlock.message.slot === GENESIS_SLOT) return;
     await verifyBlockProposerSignature(this.chain.bls, this.chain.getHeadState(), [anchorBlock]);
 
     // We can write to the disk if this is ahead of prevFinalizedCheckpointBlock otherwise
     // we will need to go make checks on the top of sync loop before writing as it might
     // override prevFinalizedCheckpointBlock
-    if (this.prevFinalizedCheckpointBlock.slot < anchorBlock.data.message.slot)
-      await this.db.blockArchive.putBinary(anchorBlock.data.message.slot, anchorBlock.bytes);
+    if (this.prevFinalizedCheckpointBlock.slot < anchorBlock.message.slot) {
+      const serialized = this.chain.serializedCache.get(anchorBlock);
+      if (serialized) {
+        await this.db.blockArchive.putBinary(anchorBlock.message.slot, serialized);
+      } else {
+        await this.db.blockArchive.put(anchorBlock.message.slot, anchorBlock);
+      }
+    }
 
     this.syncAnchor = {
-      anchorBlock: anchorBlock.data,
+      anchorBlock: anchorBlock,
       anchorBlockRoot,
-      anchorSlot: anchorBlock.data.message.slot,
-      lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.data.message.slot, block: anchorBlock.data},
+      anchorSlot: anchorBlock.message.slot,
+      lastBackSyncedBlock: {root: anchorBlockRoot, slot: anchorBlock.message.slot, block: anchorBlock},
     };
 
     this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.blockbyroot});
 
     this.logger.verbose("Fetched new anchorBlock", {
       root: toRootHex(anchorBlockRoot),
-      slot: anchorBlock.data.message.slot,
+      slot: anchorBlock.message.slot,
     });
 
     return;
@@ -825,15 +831,33 @@ export class BackfillSync extends (EventEmitter as {new (): BackfillSyncEmitter}
         nextAnchor.slot > this.prevFinalizedCheckpointBlock.slot
           ? verifiedBlocks
           : verifiedBlocks.slice(0, verifiedBlocks.length - 1);
-      await this.db.blockArchive.batchPutBinary(
-        blocksToPut.map((block) => ({
-          key: block.data.message.slot,
-          value: block.bytes,
-          slot: block.data.message.slot,
-          blockRoot: this.config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message),
-          parentRoot: block.data.message.parentRoot,
-        }))
-      );
+
+      const binaryPuts = [];
+      const nonBinaryPuts = [];
+
+      for (const block of blocksToPut) {
+        const serialized = this.chain.serializedCache.get(block);
+        const item = {
+          key: block.message.slot,
+          slot: block.message.slot,
+          blockRoot: this.config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message),
+          parentRoot: block.message.parentRoot,
+        };
+
+        if (serialized) {
+          binaryPuts.push({...item, value: serialized});
+        } else {
+          nonBinaryPuts.push({...item, value: block});
+        }
+      }
+
+      if (binaryPuts.length > 0) {
+        await this.db.blockArchive.batchPutBinary(binaryPuts);
+      }
+      if (nonBinaryPuts.length > 0) {
+        await this.db.blockArchive.batchPut(nonBinaryPuts);
+      }
+
       this.metrics?.backfillSync.totalBlocks.inc({method: BackfillSyncMethod.rangesync}, verifiedBlocks.length);
     }
 

package/src/sync/backfill/verify.ts

@@ -1,7 +1,7 @@
 import {BeaconConfig} from "@lodestar/config";
 import {GENESIS_SLOT} from "@lodestar/params";
 import {CachedBeaconStateAllForks, ISignatureSet, getBlockProposerSignatureSet} from "@lodestar/state-transition";
-import {Root, SignedBeaconBlock, Slot, WithBytes, ssz} from "@lodestar/types";
+import {Root, SignedBeaconBlock, Slot, ssz} from "@lodestar/types";
 import {IBlsVerifier} from "../../chain/bls/index.js";
 import {BackfillSyncError, BackfillSyncErrorCode} from "./errors.js";
 
@@ -14,19 +14,19 @@ export type BackfillBlock = BackfillBlockHeader & {block: SignedBeaconBlock};
 
 export function verifyBlockSequence(
   config: BeaconConfig,
-  blocks: WithBytes<SignedBeaconBlock>[],
+  blocks: SignedBeaconBlock[],
   anchorRoot: Root
 ): {
   nextAnchor: BackfillBlock | null;
-  verifiedBlocks: WithBytes<SignedBeaconBlock>[];
+  verifiedBlocks: SignedBeaconBlock[];
   error?: BackfillSyncErrorCode.NOT_LINEAR;
 } {
   let nextRoot: Root = anchorRoot;
   let nextAnchor: BackfillBlock | null = null;
 
-  const verifiedBlocks: WithBytes<SignedBeaconBlock>[] = [];
+  const verifiedBlocks: SignedBeaconBlock[] = [];
   for (const block of blocks.reverse()) {
-    const blockRoot = config.getForkTypes(block.data.message.slot).BeaconBlock.hashTreeRoot(block.data.message);
+    const blockRoot = config.getForkTypes(block.message.slot).BeaconBlock.hashTreeRoot(block.message);
     if (!ssz.Root.equals(blockRoot, nextRoot)) {
       if (ssz.Root.equals(nextRoot, anchorRoot)) {
         throw new BackfillSyncError({code: BackfillSyncErrorCode.NOT_ANCHORED});
@@ -34,8 +34,8 @@ export function verifyBlockSequence(
       return {nextAnchor, verifiedBlocks, error: BackfillSyncErrorCode.NOT_LINEAR};
     }
     verifiedBlocks.push(block);
-    nextAnchor = {block: block.data, slot: block.data.message.slot, root: nextRoot};
-    nextRoot = block.data.message.parentRoot;
+    nextAnchor = {block: block, slot: block.message.slot, root: nextRoot};
+    nextRoot = block.message.parentRoot;
   }
   return {nextAnchor, verifiedBlocks};
 }
@@ -43,12 +43,12 @@ export function verifyBlockSequence(
 export async function verifyBlockProposerSignature(
   bls: IBlsVerifier,
   state: CachedBeaconStateAllForks,
-  blocks: WithBytes<SignedBeaconBlock>[]
+  blocks: SignedBeaconBlock[]
 ): Promise<void> {
-  if (blocks.length === 1 && blocks[0].data.message.slot === GENESIS_SLOT) return;
+  if (blocks.length === 1 && blocks[0].message.slot === GENESIS_SLOT) return;
   const signatures = blocks.reduce((sigs: ISignatureSet[], block) => {
     // genesis block doesn't have valid signature
-    if (block.data.message.slot !== GENESIS_SLOT) sigs.push(getBlockProposerSignatureSet(state, block.data));
+    if (block.message.slot !== GENESIS_SLOT) sigs.push(getBlockProposerSignatureSet(state, block));
     return sigs;
   }, []);
 

package/src/sync/utils/downloadByRange.ts

@@ -254,7 +254,7 @@ export async function requestByRange({
   if (blocksRequest) {
     requests.push(
       network.sendBeaconBlocksByRange(peerIdStr, blocksRequest).then((blockResponse) => {
-        blocks = blockResponse.map(({data}) => data);
+        blocks = blockResponse;
       })
     );
   }

package/src/sync/utils/downloadByRoot.ts

@@ -314,7 +314,7 @@ export async function fetchAndValidateBlock({
   blockRoot,
 }: Omit<FetchByRootAndValidateBlockProps, "chain">): Promise<SignedBeaconBlock> {
   const response = await network.sendBeaconBlocksByRoot(peerIdStr, [blockRoot]);
-  const block = response.at(0)?.data;
+  const block = response.at(0);
   if (!block) {
     throw new DownloadByRootError({
       code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE,

package/lib/network/gossip/snappy_bun.d.ts

@@ -1,3 +0,0 @@
-import { snappy } from "@lodestar/bun";
-export declare const compress: typeof snappy.compress, uncompress: typeof snappy.uncompress;
-//# sourceMappingURL=snappy_bun.d.ts.map

package/lib/network/gossip/snappy_bun.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"snappy_bun.d.ts","sourceRoot":"","sources":["../../../src/network/gossip/snappy_bun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,eAAe,CAAC;AACrC,eAAO,MAAO,QAAQ,0BAAE,UAAU,0BAAU,CAAC"}

package/lib/network/gossip/snappy_bun.js

@@ -1,3 +0,0 @@
-import { snappy } from "@lodestar/bun";
-export const { compress, uncompress } = snappy;
-//# sourceMappingURL=snappy_bun.js.map

package/lib/network/gossip/snappy_bun.js.map

@@ -1 +0,0 @@
-{"version":3,"file":"snappy_bun.js","sourceRoot":"","sources":["../../../src/network/gossip/snappy_bun.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,MAAM,EAAC,MAAM,eAAe,CAAC;AACrC,MAAM,CAAC,MAAM,EAAC,QAAQ,EAAE,UAAU,EAAC,GAAG,MAAM,CAAC"}

package/src/network/gossip/snappy_bun.ts

@@ -1,2 +0,0 @@
-import {snappy} from "@lodestar/bun";
-export const {compress, uncompress} = snappy;