@lodestar/beacon-node 1.34.0-dev.8d743ffafc → 1.34.0-dev.91df6658d5

package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js

@@ -1,9 +1,6 @@
-import { NUMBER_OF_COLUMNS } from "@lodestar/params";
 import { RespStatus, ResponseError } from "@lodestar/reqresp";
 import { computeEpochAtSlot } from "@lodestar/state-transition";
-import { ssz } from "@lodestar/types";
 import { fromHex, toHex } from "@lodestar/utils";
-import { COLUMN_SIZE_IN_WRAPPER_INDEX, CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX, DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX, NUM_COLUMNS_IN_WRAPPER_INDEX, } from "../../../db/repositories/dataColumnSidecars.js";
 export async function* onDataColumnSidecarsByRoot(requestBody, chain, db) {
     // SPEC: minimum_request_epoch = max(finalized_epoch, current_epoch - MIN_EPOCHS_FOR_DATA_COLUMN_SIDECARS_REQUESTS, FULU_FORK_EPOCH)
     const finalizedEpoch = chain.forkChoice.getFinalizedCheckpoint().epoch;
@@ -21,27 +18,16 @@ export async function* onDataColumnSidecarsByRoot(requestBody, chain, db) {
         if (!block || computeEpochAtSlot(block.slot) < minimumRequestEpoch) {
             continue;
         }
-        const dataColumnSidecarsBytesWrapped = await db.dataColumnSidecars.getBinary(fromHex(block.blockRoot));
-        if (!dataColumnSidecarsBytesWrapped) {
-            // Handle the same to onBeaconBlocksByRange
-            throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root ${block.blockRoot} slot ${block.slot}`);
+        const dataColumns = await db.dataColumnSidecar.getManyBinary(fromHex(block.blockRoot), columns);
+        if (!dataColumns) {
+            throw new ResponseError(RespStatus.SERVER_ERROR, `No item for root=${block.blockRoot}, slot=${block.slot}`);
         }
-        const retrivedColumnsLen = ssz.Uint8.deserialize(dataColumnSidecarsBytesWrapped.slice(NUM_COLUMNS_IN_WRAPPER_INDEX, COLUMN_SIZE_IN_WRAPPER_INDEX));
-        const retrievedColumnsSizeBytes = dataColumnSidecarsBytesWrapped.slice(COLUMN_SIZE_IN_WRAPPER_INDEX, CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX);
-        const columnsSize = ssz.UintNum64.deserialize(retrievedColumnsSizeBytes);
-        const dataColumnSidecarsBytes = dataColumnSidecarsBytesWrapped.slice(DATA_COLUMN_SIDECARS_IN_WRAPPER_INDEX + 4 * retrivedColumnsLen);
-        const dataColumnsIndex = dataColumnSidecarsBytesWrapped.slice(CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX, CUSTODY_COLUMNS_IN_IN_WRAPPER_INDEX + NUMBER_OF_COLUMNS);
-        for (const index of columns) {
-            const dataIndex = (dataColumnsIndex[index] ?? 0) - 1;
-            if (dataIndex < 0) {
-                throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${index} not custodied`);
-            }
-            const dataColumnSidecarBytes = dataColumnSidecarsBytes.slice(dataIndex * columnsSize, (dataIndex + 1) * columnsSize);
-            if (dataColumnSidecarBytes.length !== columnsSize) {
-                throw Error(`Inconsistent state, dataColumnSidecar blockRoot=${blockRootHex} index=${index} dataColumnSidecarBytes=${dataColumnSidecarBytes.length} expected=${columnsSize}`);
+        for (const [index, dataColumnBytes] of dataColumns.entries()) {
+            if (!dataColumnBytes) {
+                throw new ResponseError(RespStatus.SERVER_ERROR, `dataColumnSidecar index=${columns[index]} not custodied`);
             }
             yield {
-                data: dataColumnSidecarBytes,
+                data: dataColumnBytes,
                 boundary: chain.config.getForkBoundaryAtEpoch(computeEpochAtSlot(block.slot)),
             };
         }

package/lib/network/reqresp/handlers/dataColumnSidecarsByRoot.js.map

@@ -1 +1 @@
-{"version":3,"file":"dataColumnSidecarsByRoot.js","sourceRoot":"","sources":["../../../../src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,iBAAiB,EAAC,MAAM,kBAAkB,CAAC;AACnD,OAAO,EAAC,UAAU,EAAE,aAAa,EAAmB,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAC9D,OAAO,EAAO,GAAG,EAAC,MAAM,iBAAiB,CAAC;AAC1C,OAAO,EAAC,OAAO,EAAE,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAG/C,OAAO,EACL,4BAA4B,EAC5B,mCAAmC,EACnC,qCAAqC,EACrC,4BAA4B,GAC7B,MAAM,gDAAgD,CAAC;AAExD,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,0BAA0B,CAC/C,WAAiD,EACjD,KAAmB,EACnB,EAAa;IAEb,oIAAoI;IACpI,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,sBAAsB,EAAE,CAAC,KAAK,CAAC;IACvE,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC;IAC9C,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAClC,cAAc,EACd,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,4CAA4C,EACxE,KAAK,CAAC,MAAM,CAAC,eAAe,CAC7B,CAAC;IAEF,KAAK,MAAM,2BAA2B,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,EAAC,SAAS,EAAE,OAAO,EAAC,GAAG,2BAA2B,CAAC;QACzD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAEzD,2CAA2C;QAC3C,8EAA8E;QAC9E,mHAAmH;QACnH,4FAA4F;QAC5F,oJAAoJ;QACpJ,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;YACnE,SAAS;QACX,CAAC;QAED,MAAM,8BAA8B,GAAG,MAAM,EAAE,CAAC,kBAAkB,CAAC,SAAS,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,CAAC,CAAC;QACvG,IAAI,CAAC,8BAA8B,EAAE,CAAC;YACpC,2CAA2C;YAC3C,MAAM,IAAI,aAAa,CAAC,UAAU,CAAC,YAAY,EAAE,oBAAoB,KAAK,CAAC,SAAS,SAAS,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC7G,CAAC;QAED,MAAM,kBAAkB,GAAG,GAAG,CAAC,KAAK,CAAC,WAAW,CAC9C,8BAA8B,CAAC,KAAK,CAAC,4BAA4B,EAAE,4BAA4B,CAAC,CACjG,CAAC;QACF,MAAM,yBAAyB,GAAG,8BAA8B,CAAC,KAAK,CACpE,4BAA4B,EAC5B,mCAAmC,CACpC,CAAC;QACF,MAAM,WAAW,GAAG,GAAG,CAAC,SAAS,CAAC,WAAW,CAAC,yBAAyB,CAAC,CAAC;QACzE,MAAM,uBAAuB,GAAG,8BAA8B,CAAC,KAAK,CAClE,qCAAqC,GAAG,CAAC,GAAG,kBAAkB,CAC/D,CAAC;QAEF,MAAM,gBAAgB,GAAG,8BAA8B,CAAC,KAAK,CAC3D,mCAAmC,EACnC,mCAAmC,GAAG,iBAAiB,CACxD,CAAC;QAEF,KAAK,MAAM,KAAK,IAAI,OAAO,EAAE,CAAC;YAC5B,MAAM,SAAS,GAAG,CAAC,gBAAgB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC,GAAG,CAAC,CAAC;YACrD,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;gBAClB,MAAM,IAAI,aAAa,CAAC,UAAU,CAAC,YAAY,EAAE,2BAA2B,KAAK,gBAAgB,CAAC,CAAC;YACrG,CAAC;YAED,MAAM,sBAAsB,GAAG,uBAAuB,CAAC,KAAK,CAC1D,SAAS,GAAG,WAAW,EACvB,CAAC,SAAS,GAAG,CAAC,CAAC,GAAG,WAAW,CAC9B,CAAC;YACF,IAAI,sBAAsB,CAAC,MAAM,KAAK,WAAW,EAAE,CAAC;gBAClD,MAAM,KAAK,CACT,mDAAmD,YAAY,UAAU,KAAK,2BAA2B,sBAAsB,CAAC,MAAM,aAAa,WAAW,EAAE,CACjK,CAAC;YACJ,CAAC;YAED,MAAM;gBACJ,IAAI,EAAE,sBAAsB;gBAC5B,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;aAC9E,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}
+{"version":3,"file":"dataColumnSidecarsByRoot.js","sourceRoot":"","sources":["../../../../src/network/reqresp/handlers/dataColumnSidecarsByRoot.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,UAAU,EAAE,aAAa,EAAmB,MAAM,mBAAmB,CAAC;AAC9E,OAAO,EAAC,kBAAkB,EAAC,MAAM,4BAA4B,CAAC;AAE9D,OAAO,EAAC,OAAO,EAAE,KAAK,EAAC,MAAM,iBAAiB,CAAC;AAI/C,MAAM,CAAC,KAAK,SAAS,CAAC,CAAC,0BAA0B,CAC/C,WAAiD,EACjD,KAAmB,EACnB,EAAa;IAEb,oIAAoI;IACpI,MAAM,cAAc,GAAG,KAAK,CAAC,UAAU,CAAC,sBAAsB,EAAE,CAAC,KAAK,CAAC;IACvE,MAAM,YAAY,GAAG,KAAK,CAAC,KAAK,CAAC,YAAY,CAAC;IAC9C,MAAM,mBAAmB,GAAG,IAAI,CAAC,GAAG,CAClC,cAAc,EACd,YAAY,GAAG,KAAK,CAAC,MAAM,CAAC,4CAA4C,EACxE,KAAK,CAAC,MAAM,CAAC,eAAe,CAC7B,CAAC;IAEF,KAAK,MAAM,2BAA2B,IAAI,WAAW,EAAE,CAAC;QACtD,MAAM,EAAC,SAAS,EAAE,OAAO,EAAC,GAAG,2BAA2B,CAAC;QACzD,MAAM,YAAY,GAAG,KAAK,CAAC,SAAS,CAAC,CAAC;QACtC,MAAM,KAAK,GAAG,KAAK,CAAC,UAAU,CAAC,WAAW,CAAC,YAAY,CAAC,CAAC;QAEzD,2CAA2C;QAC3C,8EAA8E;QAC9E,mHAAmH;QACnH,4FAA4F;QAC5F,oJAAoJ;QACpJ,IAAI,CAAC,KAAK,IAAI,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,GAAG,mBAAmB,EAAE,CAAC;YACnE,SAAS;QACX,CAAC;QAED,MAAM,WAAW,GAAG,MAAM,EAAE,CAAC,iBAAiB,CAAC,aAAa,CAAC,OAAO,CAAC,KAAK,CAAC,SAAS,CAAC,EAAE,OAAO,CAAC,CAAC;QAChG,IAAI,CAAC,WAAW,EAAE,CAAC;YACjB,MAAM,IAAI,aAAa,CAAC,UAAU,CAAC,YAAY,EAAE,oBAAoB,KAAK,CAAC,SAAS,UAAU,KAAK,CAAC,IAAI,EAAE,CAAC,CAAC;QAC9G,CAAC;QAED,KAAK,MAAM,CAAC,KAAK,EAAE,eAAe,CAAC,IAAI,WAAW,CAAC,OAAO,EAAE,EAAE,CAAC;YAC7D,IAAI,CAAC,eAAe,EAAE,CAAC;gBACrB,MAAM,IAAI,aAAa,CAAC,UAAU,CAAC,YAAY,EAAE,2BAA2B,OAAO,CAAC,KAAK,CAAC,gBAAgB,CAAC,CAAC;YAC9G,CAAC;YAED,MAAM;gBACJ,IAAI,EAAE,eAAe;gBACrB,QAAQ,EAAE,KAAK,CAAC,MAAM,CAAC,sBAAsB,CAAC,kBAAkB,CAAC,KAAK,CAAC,IAAI,CAAC,CAAC;aAC9E,CAAC;QACJ,CAAC;IACH,CAAC;AACH,CAAC"}

package/lib/sync/unknownBlock.d.ts

@@ -1,8 +1,12 @@
 import { ChainForkConfig } from "@lodestar/config";
 import { Logger } from "@lodestar/utils";
+import { BlockInput, NullBlockInput } from "../chain/blocks/types.js";
 import { IBeaconChain } from "../chain/index.js";
 import { Metrics } from "../metrics/index.js";
 import { INetwork } from "../network/index.js";
+import { PeerSyncMeta } from "../network/peers/peersData.js";
+import { CustodyConfig } from "../util/dataColumns.js";
+import { PeerIdStr } from "../util/peerId.js";
 import { SyncOptions } from "./options.js";
 export declare class UnknownBlockSync {
     private readonly config;
@@ -21,6 +25,7 @@ export declare class UnknownBlockSync {
     private subscribedToNetworkEvents;
     private engineGetBlobsCache;
     private blockInputsRetryTrackerCache;
+    private peerBalancer;
     constructor(config: ChainForkConfig, network: INetwork, chain: IBeaconChain, logger: Logger, metrics: Metrics | null, opts?: SyncOptions | undefined);
     subscribeToNetwork(): void;
     unsubscribeFromNetwork(): void;
@@ -47,6 +52,8 @@ export declare class UnknownBlockSync {
      */
     private addUnknownParent;
     private addUnknownBlock;
+    private onPeerConnected;
+    private onPeerDisconnected;
     /**
      * Gather tip parent blocks with unknown parent and do a search for all of them
      */
@@ -64,7 +71,7 @@ export declare class UnknownBlockSync {
      *   - from deneb, fetch all missing blobs
      *   - from peerDAS, fetch sampled colmns
      * TODO: this means we only have block root, and nothing else. Consider to reflect this in the function name
-     * Will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers if connectPeers.length > MAX_ATTEMPTS_PER_BLOCK.
+     * prefulu, will attempt a max of `MAX_ATTEMPTS_PER_BLOCK` on different peers, postfulu we may attempt more as defined in `getMaxDownloadAttempts()` function
      * Also verifies the received block root + returns the peer that provided the block for future downscoring.
      */
     private fetchUnknownBlockRoot;
@@ -72,8 +79,7 @@ export declare class UnknownBlockSync {
      * We have partial block input:
      * - we have block but not have all blobs (deneb) or needed columns (fulu)
      * - we don't have block and have some blobs (deneb) or some columns (fulu)
-     * Fetches missing blobs for the blockinput, in future can also pull block is thats also missing
-     * along with the blobs (i.e. only some blobs are available)
+     * Fetches missing block/data columns/block for the blockinput. This function returns either preData or availableData BlockInput.
      */
     private fetchUnavailableBlockInput;
     /**
@@ -84,5 +90,40 @@ export declare class UnknownBlockSync {
      */
     private removeAndDownscoreAllDescendants;
     private removeAllDescendants;
+    private getMaxDownloadAttempts;
+}
+/**
+ * Class to track active byRoots requests and balance them across eligible peers.
+ */
+export declare class UnknownBlockPeerBalancer {
+    readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
+    readonly activeRequests: Map<PeerIdStr, number>;
+    private readonly custodyConfig;
+    constructor(custodyConfig: CustodyConfig);
+    /** Trigger on each peer re-status */
+    onPeerConnected(peerId: PeerIdStr, syncMeta: PeerSyncMeta): void;
+    onPeerDisconnected(peerId: PeerIdStr): void;
+    /**
+     * called from fetchUnknownBlockRoot() where we only have block root and nothing else
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     * pendingColumns is empty for prefulu, or the 1st time we we download a block by root
+     */
+    bestPeerForPendingColumns(pendingColumns: Set<number> | null, excludedPeers: Set<PeerIdStr>): PeerSyncMeta | null;
+    /**
+     * called from fetchUnavailableBlockInput() where we have either BlockInput or NullBlockInput
+     * excludedPeers are the peers that we requested already so we don't want to try again
+     */
+    bestPeerForBlockInput(unavailableBlockInput: BlockInput | NullBlockInput, excludedPeers: Set<PeerIdStr>): PeerSyncMeta | null;
+    /**
+     * Consumers don't need to call this method directly, it is called internally by bestPeer*() methods
+     * make this public for testing
+     */
+    onRequest(peerId: PeerIdStr): void;
+    /**
+     * Consumers should call this method when a request is completed for a peer.
+     */
+    onRequestCompleted(peerId: PeerIdStr): void;
+    getTotalActiveRequests(): number;
+    private filterPeers;
 }
 //# sourceMappingURL=unknownBlock.d.ts.map