@lodestar/beacon-node 1.36.0-dev.d42eb95bf0 → 1.36.0-dev.d8afb6dc39

package/lib/sync/utils/downloadByRange.d.ts

@@ -6,7 +6,6 @@ import { SeenBlockInput } from "../../chain/seenCache/seenGossipBlockInput.js";
 import { INetwork } from "../../network/index.js";
 import { PeerIdStr } from "../../util/peerId.js";
 import { WarnResult } from "../../util/wrapError.js";
-import { DownloadByRootErrorCode } from "./downloadByRoot.js";
 export type DownloadByRangeRequests = {
     blocksRequest?: phase0.BeaconBlocksByRangeRequest;
     blobsRequest?: deneb.BlobSidecarsByRangeRequest;
@@ -76,7 +75,7 @@ export declare function validateResponses({ config, batchBlocks, blocksRequest,
  * - must allow for skip slots
  * - check is a chain of blocks where via parentRoot matches hashTreeRoot of block before
  */
-export declare function validateBlockByRangeResponse(config: ChainForkConfig, blocksRequest: phase0.BeaconBlocksByRangeRequest, blocks: SignedBeaconBlock[]): ValidatedBlock[];
+export declare function validateBlockByRangeResponse(config: ChainForkConfig, blocksRequest: phase0.BeaconBlocksByRangeRequest, blocks: SignedBeaconBlock[]): WarnResult<ValidatedBlock[], DownloadByRangeError>;
 /**
  * Should not be called directly. Only exported for unit testing purposes.
  * This is used only in Deneb and Electra
@@ -84,8 +83,47 @@ export declare function validateBlockByRangeResponse(config: ChainForkConfig, bl
 export declare function validateBlobsByRangeResponse(dataRequestBlocks: ValidatedBlock[], blobSidecars: deneb.BlobSidecars): Promise<ValidatedBlobSidecars[]>;
 /**
  * Should not be called directly. Only exported for unit testing purposes
+ *
+ * Spec states:
+ * 1) must be within range [start_slot, start_slot + count]
+ * 2) should respond with all columns in the range or and 3:ResourceUnavailable (and potentially get down-scored)
+ * 3) must response with at least the sidecars of the first blob-carrying block that exists in the range
+ * 4) must include all sidecars from each block from which there are blobs
+ * 5) where they exists, sidecars must be sent in (slot, index) order
+ * 6) clients may limit the number of sidecars in a response
+ * 7) clients may stop responding mid-response if their view of fork-choice changes
+ *
+ * We will interpret the spec as follows
+ * - Errors when validating: 1, 3, 5
+ * - Warnings when validating: 2, 4, 6, 7
+ *
+ * For "warning" cases, where we get a partial response but sidecars are validated and correct with respect to the
+ * blocks, then they will be kept.  This loosening of the spec is to help ensure sync goes smoothly and we can find
+ * the data needed in difficult network situations.
+ *
+ * Assume for the following two examples we request indices 5, 10, 15 for a range of slots 32-63
+ *
+ * For slots where we receive no sidecars, example slot 45, but blobs exist we will stop validating subsequent
+ * slots, 45-63.  The next round of requests will get structured to pull the from the slot that had columns
+ * missing to the end of the range for all columns indices that were requested for the current partially failed
+ * request (slots 45-63 and indices 5, 10, 15).
+ *
+ * For slots where only some of the requested sidecars are received we will proceed with validation. For simplicity sake
+ * we will assume that if we only get some indices back for a (or several) slot(s) that the indices we get will be
+ * consistent. IE if a peer returns only index 5, they will most likely return that same index for subsequent slot
+ * (index 5 for slots 34, 35, 36, etc). They will not likely return 5 on slot 34, 10 on slot 35, 15 on slot 36, etc.
+ * This assumption makes the code simpler. For both cases the request for the next round will be structured correctly
+ * to pull any missing column indices for whatever range remains.  The simplification just leads to re-verification
+ * of the columns but the number of columns downloaded will be the same regardless of if they are validated twice.
+ *
+ * validateColumnsByRangeResponse makes some assumptions about the data being passed in
+ * blocks are:
+ * - slotwise in order
+ * - form a chain
+ * - non-sparse response (any missing block is a skipped slot not a bad response)
+ * - last block is last slot received
  */
-export declare function validateColumnsByRangeResponse(request: fulu.DataColumnSidecarsByRangeRequest, dataRequestBlocks: ValidatedBlock[], columnSidecars: fulu.DataColumnSidecars): Promise<WarnResult<ValidatedColumnSidecars[], DownloadByRangeError>>;
+export declare function validateColumnsByRangeResponse(config: ChainForkConfig, request: fulu.DataColumnSidecarsByRangeRequest, blocks: ValidatedBlock[], columnSidecars: fulu.DataColumnSidecars): Promise<WarnResult<ValidatedColumnSidecars[], DownloadByRangeError>>;
 /**
  * Given a data request, return only the blocks and roots that correspond to the data request (sorted). Assumes that
  * cached have slots that are all before the current batch of downloaded blocks
@@ -95,7 +133,7 @@ export declare function getBlocksForDataValidation(dataRequest: {
     count: number;
 }, cached: IBlockInput[] | undefined, current: ValidatedBlock[] | undefined): ValidatedBlock[];
 export declare enum DownloadByRangeErrorCode {
-    MISSING_BLOCKS = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCKS",
+    MISSING_BLOCKS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCK_RESPONSE",
     MISSING_BLOBS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOBS_RESPONSE",
     MISSING_COLUMNS_RESPONSE = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS_RESPONSE",
     /** Error at the reqresp layer */
@@ -110,26 +148,29 @@ export declare enum DownloadByRangeErrorCode {
     MISSING_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS",
     OVER_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_OVER_COLUMNS",
     EXTRA_COLUMNS = "DOWNLOAD_BY_RANGE_ERROR_EXTRA_COLUMNS",
+    NO_COLUMNS_FOR_BLOCK = "DOWNLOAD_BY_RANGE_ERROR_NO_COLUMNS_FOR_BLOCK",
+    DUPLICATE_COLUMN = "DOWNLOAD_BY_RANGE_ERROR_DUPLICATE_COLUMN",
+    OUT_OF_ORDER_COLUMNS = "DOWNLOAD_BY_RANGE_OUT_OF_ORDER_COLUMNS",
     /** Cached block input type mismatches new data */
+    MISMATCH_BLOCK_FORK = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_FORK",
     MISMATCH_BLOCK_INPUT_TYPE = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_INPUT_TYPE"
 }
 export type DownloadByRangeErrorType = {
-    code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE;
-    expectedCount: number;
-} | {
-    code: DownloadByRangeErrorCode.MISSING_BLOCKS | DownloadByRangeErrorCode.MISSING_BLOBS_RESPONSE | DownloadByRangeErrorCode.MISSING_COLUMNS_RESPONSE;
+    code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE | DownloadByRangeErrorCode.MISSING_BLOBS_RESPONSE | DownloadByRangeErrorCode.MISSING_COLUMNS_RESPONSE;
     blockStartSlot?: number;
     blockCount?: number;
     blobStartSlot?: number;
     blobCount?: number;
     columnStartSlot?: number;
     columnCount?: number;
-} | {
-    code: DownloadByRootErrorCode.MISSING_BLOCK_RESPONSE;
-    expectedCount: number;
 } | {
     code: DownloadByRangeErrorCode.OUT_OF_RANGE_BLOCKS;
     slot: number;
+} | {
+    code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK;
+    slot: number;
+    dataFork: string;
+    blockFork: string;
 } | {
     code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOCKS;
 } | {
@@ -155,7 +196,7 @@ export type DownloadByRangeErrorType = {
     expected: number;
     actual: number;
 } | {
-    code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS;
+    code: DownloadByRangeErrorCode.OUT_OF_ORDER_BLOBS | DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS;
     slot: number;
 } | {
     code: DownloadByRangeErrorCode.EXTRA_BLOBS;
@@ -171,7 +212,11 @@ export type DownloadByRangeErrorType = {
     blockRoot: string;
     missingIndices: string;
 } | {
-    code: DownloadByRangeErrorCode.EXTRA_COLUMNS;
+    code: DownloadByRangeErrorCode.DUPLICATE_COLUMN;
+    slot: Slot;
+    index: number;
+} | {
+    code: DownloadByRangeErrorCode.EXTRA_COLUMNS | DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK;
     slot: Slot;
     blockRoot: string;
     invalidIndices: string;

package/lib/sync/utils/downloadByRange.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"downloadByRange.d.ts","sourceRoot":"","sources":["../../../src/sync/utils/downloadByRange.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AAEjD,OAAO,EAAC,iBAAiB,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAC,aAAa,EAAE,MAAM,EAAsD,MAAM,iBAAiB,CAAC;AAC3G,OAAO,EAEL,MAAM,EACN,WAAW,EAGZ,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAC,cAAc,EAAC,MAAM,+CAA+C,CAAC;AAG7E,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AACnD,OAAO,EAAC,uBAAuB,EAAC,MAAM,qBAAqB,CAAC;AAE5D,MAAM,MAAM,uBAAuB,GAAG;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAClD,YAAY,CAAC,EAAE,KAAK,CAAC,0BAA0B,CAAC;IAChD,cAAc,CAAC,EAAE,IAAI,CAAC,gCAAgC,CAAC;CACxD,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC7B,YAAY,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC;IAClC,cAAc,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,uBAAuB,GAAG;IACnE,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;IACtB,OAAO,EAAE,QAAQ,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,KAAK,EAAE,cAAc,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,kBAAkB,CAAC;IAC9B,WAAW,EAAE,WAAW,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,UAAU,CAAC;IACtB,KAAK,EAAE,iBAAiB,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,SAAS,EAAE,UAAU,CAAC;IACtB,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,UAAU,CAAC;IACtB,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,qBAAqB,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAChD,uBAAuB,CAAC,EAAE,uBAAuB,EAAE,CAAC;CACrD,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,SAAS,EACT,SAAS,EACT,WAAW,GACZ,EAAE,0BAA0B,GAAG,WAAW,EAAE,CAqG5C;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,OAAO,EACP,SAAS,EACT,WAAW,EACX,aAAa,EACb,YAAY,EACZ,cAAc,GACf,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CA4B7G;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,EACnC,OAAO,EACP,SAAS,EACT,aAAa,EACb,YAAY,EACZ,cAAc,GACf,EAAE,uBAAuB,GAAG;IAC3B,OAAO,EAAE,QAAQ,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;CACtB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAsCpC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EACN,WAAW,EACX,aAAa,EACb,YAAY,EACZ,cAAc,EACd,MAAM,EACN,YAAY,EACZ,cAAc,GACf,EAAE,uBAAuB,GACxB,wBAAwB,GAAG;IACzB,MAAM,EAAE,eAAe,CAAC;IACxB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CA6ElE;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,eAAe,EACvB,aAAa,EAAE,MAAM,CAAC,0BAA0B,EAChD,MAAM,EAAE,iBAAiB,EAAE,GAC1B,cAAc,EAAE,CAgFlB;AAED;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,iBAAiB,EAAE,cAAc,EAAE,EACnC,YAAY,EAAE,KAAK,CAAC,YAAY,GAC/B,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA4DlC;AAED;;GAEG;AACH,wBAAsB,8BAA8B,CAClD,OAAO,EAAE,IAAI,CAAC,gCAAgC,EAC9C,iBAAiB,EAAE,cAAc,EAAE,EACnC,cAAc,EAAE,IAAI,CAAC,kBAAkB,GACtC,OAAO,CAAC,UAAU,CAAC,uBAAuB,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAkGtE;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,WAAW,EAAE;IAAC,SAAS,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,EAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,EACjC,OAAO,EAAE,cAAc,EAAE,GAAG,SAAS,GACpC,cAAc,EAAE,CA6BlB;AA0BD,oBAAY,wBAAwB;IAClC,cAAc,2CAA2C;IACzD,sBAAsB,mDAAmD;IACzE,wBAAwB,qDAAqD;IAE7E,iCAAiC;IACjC,cAAc,2CAA2C;IAIzD,oBAAoB,iDAAiD;IACrE,YAAY,yCAAyC;IACrD,mBAAmB,0CAA0C;IAC7D,mBAAmB,0CAA0C;IAE7D,aAAa,0CAA0C;IACvD,kBAAkB,+CAA+C;IACjE,WAAW,wCAAwC;IAEnD,eAAe,4CAA4C;IAC3D,YAAY,yCAAyC;IACrD,aAAa,0CAA0C;IAEvD,kDAAkD;IAClD,yBAAyB,sDAAsD;CAChF;AAED,MAAM,MAAM,wBAAwB,GAChC;IACE,IAAI,EAAE,uBAAuB,CAAC,sBAAsB,CAAC;IACrD,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EACA,wBAAwB,CAAC,cAAc,GACvC,wBAAwB,CAAC,sBAAsB,GAC/C,wBAAwB,CAAC,wBAAwB,CAAC;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACD;IACE,IAAI,EAAE,uBAAuB,CAAC,sBAAsB,CAAC;IACrD,aAAa,EAAE,MAAM,CAAC;CACvB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;IACnD,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;CACpD,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,cAAc,CAAC;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,oBAAoB,CAAC;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,kBAAkB,CAAC;IAClD,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,WAAW,CAAC;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,eAAe,CAAC;IAC/C,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAC;IAC7C,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,yBAAyB,CAAC;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,qBAAa,oBAAqB,SAAQ,aAAa,CAAC,wBAAwB,CAAC;CAAG"}
+{"version":3,"file":"downloadByRange.d.ts","sourceRoot":"","sources":["../../../src/sync/utils/downloadByRange.ts"],"names":[],"mappings":"AAAA,OAAO,EAAC,eAAe,EAAC,MAAM,kBAAkB,CAAC;AASjD,OAAO,EAAC,iBAAiB,EAAE,IAAI,EAAE,KAAK,EAAE,IAAI,EAAE,MAAM,EAAC,MAAM,iBAAiB,CAAC;AAC7E,OAAO,EAAC,aAAa,EAAE,MAAM,EAAyC,MAAM,iBAAiB,CAAC;AAC9F,OAAO,EAEL,MAAM,EACN,WAAW,EAGZ,MAAM,wCAAwC,CAAC;AAChD,OAAO,EAAC,cAAc,EAAC,MAAM,+CAA+C,CAAC;AAG7E,OAAO,EAAC,QAAQ,EAAC,MAAM,wBAAwB,CAAC;AAChD,OAAO,EAAC,SAAS,EAAC,MAAM,sBAAsB,CAAC;AAC/C,OAAO,EAAC,UAAU,EAAC,MAAM,yBAAyB,CAAC;AAEnD,MAAM,MAAM,uBAAuB,GAAG;IACpC,aAAa,CAAC,EAAE,MAAM,CAAC,0BAA0B,CAAC;IAClD,YAAY,CAAC,EAAE,KAAK,CAAC,0BAA0B,CAAC;IAChD,cAAc,CAAC,EAAE,IAAI,CAAC,gCAAgC,CAAC;CACxD,CAAC;AAEF,MAAM,MAAM,wBAAwB,GAAG;IACrC,MAAM,CAAC,EAAE,iBAAiB,EAAE,CAAC;IAC7B,YAAY,CAAC,EAAE,KAAK,CAAC,YAAY,CAAC;IAClC,cAAc,CAAC,EAAE,IAAI,CAAC,kBAAkB,CAAC;CAC1C,CAAC;AAEF,MAAM,MAAM,4BAA4B,GAAG,uBAAuB,GAAG;IACnE,MAAM,EAAE,eAAe,CAAC;IACxB,KAAK,EAAE,cAAc,CAAC;IACtB,OAAO,EAAE,QAAQ,CAAC;IAClB,MAAM,EAAE,MAAM,CAAC;IACf,SAAS,EAAE,MAAM,CAAC;IAClB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;CAC7B,CAAC;AAEF,MAAM,MAAM,0BAA0B,GAAG;IACvC,KAAK,EAAE,cAAc,CAAC;IACtB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,kBAAkB,CAAC;IAC9B,WAAW,EAAE,WAAW,EAAE,CAAC;CAC5B,CAAC;AAEF,MAAM,MAAM,cAAc,GAAG;IAC3B,SAAS,EAAE,UAAU,CAAC;IACtB,KAAK,EAAE,iBAAiB,CAAC;CAC1B,CAAC;AAEF,MAAM,MAAM,qBAAqB,GAAG;IAClC,SAAS,EAAE,UAAU,CAAC;IACtB,YAAY,EAAE,KAAK,CAAC,YAAY,CAAC;CAClC,CAAC;AAEF,MAAM,MAAM,uBAAuB,GAAG;IACpC,SAAS,EAAE,UAAU,CAAC;IACtB,cAAc,EAAE,IAAI,CAAC,kBAAkB,CAAC;CACzC,CAAC;AAEF,MAAM,MAAM,kBAAkB,GAAG;IAC/B,eAAe,CAAC,EAAE,cAAc,EAAE,CAAC;IACnC,qBAAqB,CAAC,EAAE,qBAAqB,EAAE,CAAC;IAChD,uBAAuB,CAAC,EAAE,uBAAuB,EAAE,CAAC;CACrD,CAAC;AAEF;;GAEG;AACH,wBAAgB,qBAAqB,CAAC,EACpC,KAAK,EACL,SAAS,EACT,SAAS,EACT,WAAW,GACZ,EAAE,0BAA0B,GAAG,WAAW,EAAE,CAiH5C;AAED,wBAAsB,eAAe,CAAC,EACpC,MAAM,EACN,OAAO,EACP,SAAS,EACT,WAAW,EACX,aAAa,EACb,YAAY,EACZ,cAAc,GACf,EAAE,IAAI,CAAC,4BAA4B,EAAE,OAAO,CAAC,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CA4B7G;AAED;;GAEG;AACH,wBAAsB,cAAc,CAAC,EACnC,OAAO,EACP,SAAS,EACT,aAAa,EACb,YAAY,EACZ,cAAc,GACf,EAAE,uBAAuB,GAAG;IAC3B,OAAO,EAAE,QAAQ,CAAC;IAClB,SAAS,EAAE,SAAS,CAAC;CACtB,GAAG,OAAO,CAAC,wBAAwB,CAAC,CAsCpC;AAED;;GAEG;AACH,wBAAsB,iBAAiB,CAAC,EACtC,MAAM,EACN,WAAW,EACX,aAAa,EACb,YAAY,EACZ,cAAc,EACd,MAAM,EACN,YAAY,EACZ,cAAc,GACf,EAAE,uBAAuB,GACxB,wBAAwB,GAAG;IACzB,MAAM,EAAE,eAAe,CAAC;IACxB,WAAW,CAAC,EAAE,WAAW,EAAE,CAAC;CAC7B,GAAG,OAAO,CAAC,UAAU,CAAC,kBAAkB,EAAE,oBAAoB,CAAC,CAAC,CAqFlE;AAED;;;;;;;;GAQG;AACH,wBAAgB,4BAA4B,CAC1C,MAAM,EAAE,eAAe,EACvB,aAAa,EAAE,MAAM,CAAC,0BAA0B,EAChD,MAAM,EAAE,iBAAiB,EAAE,GAC1B,UAAU,CAAC,cAAc,EAAE,EAAE,oBAAoB,CAAC,CA6FpD;AAED;;;GAGG;AACH,wBAAsB,4BAA4B,CAChD,iBAAiB,EAAE,cAAc,EAAE,EACnC,YAAY,EAAE,KAAK,CAAC,YAAY,GAC/B,OAAO,CAAC,qBAAqB,EAAE,CAAC,CA4DlC;AAED;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;GAyCG;AACH,wBAAsB,8BAA8B,CAClD,MAAM,EAAE,eAAe,EACvB,OAAO,EAAE,IAAI,CAAC,gCAAgC,EAC9C,MAAM,EAAE,cAAc,EAAE,EACxB,cAAc,EAAE,IAAI,CAAC,kBAAkB,GACtC,OAAO,CAAC,UAAU,CAAC,uBAAuB,EAAE,EAAE,oBAAoB,CAAC,CAAC,CAwKtE;AAED;;;GAGG;AACH,wBAAgB,0BAA0B,CACxC,WAAW,EAAE;IAAC,SAAS,EAAE,IAAI,CAAC;IAAC,KAAK,EAAE,MAAM,CAAA;CAAC,EAC7C,MAAM,EAAE,WAAW,EAAE,GAAG,SAAS,EACjC,OAAO,EAAE,cAAc,EAAE,GAAG,SAAS,GACpC,cAAc,EAAE,CA6BlB;AA0BD,oBAAY,wBAAwB;IAClC,uBAAuB,mDAAmD;IAC1E,sBAAsB,mDAAmD;IACzE,wBAAwB,qDAAqD;IAE7E,iCAAiC;IACjC,cAAc,2CAA2C;IAIzD,oBAAoB,iDAAiD;IACrE,YAAY,yCAAyC;IACrD,mBAAmB,0CAA0C;IAC7D,mBAAmB,0CAA0C;IAE7D,aAAa,0CAA0C;IACvD,kBAAkB,+CAA+C;IACjE,WAAW,wCAAwC;IAEnD,eAAe,4CAA4C;IAC3D,YAAY,yCAAyC;IACrD,aAAa,0CAA0C;IACvD,oBAAoB,iDAAiD;IACrE,gBAAgB,6CAA6C;IAC7D,oBAAoB,2CAA2C;IAE/D,kDAAkD;IAClD,mBAAmB,gDAAgD;IACnE,yBAAyB,sDAAsD;CAChF;AAED,MAAM,MAAM,wBAAwB,GAChC;IACE,IAAI,EACA,wBAAwB,CAAC,uBAAuB,GAChD,wBAAwB,CAAC,sBAAsB,GAC/C,wBAAwB,CAAC,wBAAwB,CAAC;IACtD,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;CACtB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;IACnD,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;IACnD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;CACnB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,mBAAmB,CAAC;CACpD,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,cAAc,CAAC;IAC9C,cAAc,CAAC,EAAE,MAAM,CAAC;IACxB,UAAU,CAAC,EAAE,MAAM,CAAC;IACpB,aAAa,CAAC,EAAE,MAAM,CAAC;IACvB,SAAS,CAAC,EAAE,MAAM,CAAC;IACnB,eAAe,CAAC,EAAE,MAAM,CAAC;IACzB,WAAW,CAAC,EAAE,MAAM,CAAC;IACrB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,oBAAoB,CAAC;IACpD,IAAI,EAAE,MAAM,CAAC;IACb,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAC5C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,aAAa,CAAC;IAC7C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,kBAAkB,GAAG,wBAAwB,CAAC,oBAAoB,CAAC;IAClG,IAAI,EAAE,MAAM,CAAC;CACd,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,WAAW,CAAC;IAC3C,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,YAAY,CAAC;IAC5C,GAAG,EAAE,MAAM,CAAC;IACZ,MAAM,EAAE,MAAM,CAAC;CAChB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,eAAe,CAAC;IAC/C,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,gBAAgB,CAAC;IAChD,IAAI,EAAE,IAAI,CAAC;IACX,KAAK,EAAE,MAAM,CAAC;CACf,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,aAAa,GAAG,wBAAwB,CAAC,oBAAoB,CAAC;IAC7F,IAAI,EAAE,IAAI,CAAC;IACX,SAAS,EAAE,MAAM,CAAC;IAClB,cAAc,EAAE,MAAM,CAAC;CACxB,GACD;IACE,IAAI,EAAE,wBAAwB,CAAC,yBAAyB,CAAC;IACzD,IAAI,EAAE,MAAM,CAAC;IACb,SAAS,EAAE,MAAM,CAAC;IAClB,QAAQ,EAAE,MAAM,CAAC;IACjB,MAAM,EAAE,MAAM,CAAC;CAChB,CAAC;AAEN,qBAAa,oBAAqB,SAAQ,aAAa,CAAC,wBAAwB,CAAC;CAAG"}

package/lib/sync/utils/downloadByRange.js

@@ -1,4 +1,5 @@
-import { LodestarError, fromHex, prettyBytes, prettyPrintIndices, toRootHex } from "@lodestar/utils";
+import { isForkPostFulu, isForkPostGloas, } from "@lodestar/params";
+import { LodestarError, fromHex, prettyPrintIndices, toRootHex } from "@lodestar/utils";
 import { BlockInputSource, DAType, isBlockInputBlobs, isBlockInputColumns, } from "../../chain/blocks/blockInput/index.js";
 import { validateBlockBlobSidecars } from "../../chain/validation/blobSidecar.js";
 import { validateBlockDataColumnSidecars } from "../../chain/validation/dataColumnSidecar.js";
@@ -37,7 +38,11 @@ export function cacheByRangeResponses({ cache, peerIdStr, responses, batchBlocks
         }
     }
     for (const { blockRoot, blobSidecars } of responses.validatedBlobSidecars ?? []) {
-        const existing = updatedBatchBlocks.get(blobSidecars[0].signedBlockHeader.message.slot);
+        const dataSlot = blobSidecars.at(0)?.signedBlockHeader.message.slot;
+        if (dataSlot === undefined) {
+            throw new Error(`Coding Error: empty blobSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`);
+        }
+        const existing = updatedBatchBlocks.get(dataSlot);
         const blockRootHex = toRootHex(blockRoot);
         if (!existing) {
             throw new Error("Coding error: blockInput must exist when adding blobs");
@@ -46,7 +51,7 @@ export function cacheByRangeResponses({ cache, peerIdStr, responses, batchBlocks
             throw new DownloadByRangeError({
                 code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
                 slot: existing.slot,
-                blockRoot: prettyBytes(existing.blockRootHex),
+                blockRoot: existing.blockRootHex,
                 expected: DAType.Blobs,
                 actual: existing.type,
             });
@@ -63,16 +68,20 @@ export function cacheByRangeResponses({ cache, peerIdStr, responses, batchBlocks
         }
     }
     for (const { blockRoot, columnSidecars } of responses.validatedColumnSidecars ?? []) {
-        const existing = updatedBatchBlocks.get(columnSidecars[0].signedBlockHeader.message.slot);
+        const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
+        if (dataSlot === undefined) {
+            throw new Error(`Coding Error: empty columnSidecars returned for blockRoot=${toRootHex(blockRoot)} from validation functions`);
+        }
+        const existing = updatedBatchBlocks.get(dataSlot);
         const blockRootHex = toRootHex(blockRoot);
         if (!existing) {
-            throw new Error("Coding error: blockInput must exist when adding blobs");
+            throw new Error("Coding error: blockInput must exist when adding columns");
         }
         if (!isBlockInputColumns(existing)) {
             throw new DownloadByRangeError({
                 code: DownloadByRangeErrorCode.MISMATCH_BLOCK_INPUT_TYPE,
                 slot: existing.slot,
-                blockRoot: prettyBytes(existing.blockRootHex),
+                blockRoot: existing.blockRootHex,
                 expected: DAType.Columns,
                 actual: existing.type,
             });
@@ -157,23 +166,27 @@ export async function validateResponses({ config, batchBlocks, blocksRequest, bl
     // If no blocksRequest is provided, batchBlocks must have been provided from cache
     if ((blobsRequest || columnsRequest) && !(blocks || batchBlocks)) {
         throw new DownloadByRangeError({
-            code: DownloadByRangeErrorCode.MISSING_BLOCKS,
+            code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
             ...requestsLogMeta({ blobsRequest, columnsRequest }),
         }, "No blocks to validate data requests against");
     }
     const validatedResponses = {};
     let warnings = null;
     if (blocksRequest) {
-        validatedResponses.validatedBlocks = validateBlockByRangeResponse(config, blocksRequest, blocks ?? []);
+        const result = validateBlockByRangeResponse(config, blocksRequest, blocks ?? []);
+        if (result.warnings?.length) {
+            warnings = result.warnings;
+        }
+        validatedResponses.validatedBlocks = result.result;
     }
     const dataRequest = blobsRequest ?? columnsRequest;
     if (!dataRequest) {
-        return { result: validatedResponses, warnings: null };
+        return { result: validatedResponses, warnings };
     }
-    const dataRequestBlocks = getBlocksForDataValidation(dataRequest, batchBlocks, blocksRequest ? validatedResponses.validatedBlocks : undefined);
-    if (!dataRequestBlocks.length) {
+    const blocksForDataValidation = getBlocksForDataValidation(dataRequest, batchBlocks, validatedResponses.validatedBlocks?.length ? validatedResponses.validatedBlocks : undefined);
+    if (!blocksForDataValidation.length) {
         throw new DownloadByRangeError({
-            code: DownloadByRangeErrorCode.MISSING_BLOCKS,
+            code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
             ...requestsLogMeta({ blobsRequest, columnsRequest }),
         }, "No blocks in data request slot range to validate data response against");
     }
@@ -184,7 +197,7 @@ export async function validateResponses({ config, batchBlocks, blocksRequest, bl
                 ...requestsLogMeta({ blobsRequest, columnsRequest }),
             }, "No blobSidecars to validate against blobsRequest");
         }
-        validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(dataRequestBlocks, blobSidecars);
+        validatedResponses.validatedBlobSidecars = await validateBlobsByRangeResponse(blocksForDataValidation, blobSidecars);
     }
     if (columnsRequest) {
         if (!columnSidecars) {
@@ -193,7 +206,7 @@ export async function validateResponses({ config, batchBlocks, blocksRequest, bl
                 ...requestsLogMeta({ blobsRequest, columnsRequest }),
             }, "No columnSidecars to check columnRequest against");
         }
-        const validatedColumnSidecarsResult = await validateColumnsByRangeResponse(columnsRequest, dataRequestBlocks, columnSidecars);
+        const validatedColumnSidecarsResult = await validateColumnsByRangeResponse(config, columnsRequest, blocksForDataValidation, columnSidecars);
         validatedResponses.validatedColumnSidecars = validatedColumnSidecarsResult.result;
         warnings = validatedColumnSidecarsResult.warnings;
     }
@@ -210,17 +223,27 @@ export async function validateResponses({ config, batchBlocks, blocksRequest, bl
  */
 export function validateBlockByRangeResponse(config, blocksRequest, blocks) {
     const { startSlot, count } = blocksRequest;
-    // TODO(fulu): This was added by @twoeths in #8150 but it breaks for epochs with 0 blocks during chain
-    //    liveness issues. See comment https://github.com/ChainSafe/lodestar/issues/8147#issuecomment-3246434697
-    // if (!blocks.length) {
-    //   throw new DownloadByRangeError(
-    //     {
-    //       code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
-    //       expectedCount: blocksRequest.count,
-    //     },
-    //     "Zero blocks in response"
-    //   );
-    // }
+    // An error was thrown here by @twoeths in #8150 but it breaks for epochs with 0 blocks during chain
+    // liveness issues. See comment https://github.com/ChainSafe/lodestar/issues/8147#issuecomment-3246434697
+    // There are instances where clients return no blocks though.  Need to monitor this via the warns to see
+    // if what the correct behavior should be
+    if (!blocks.length) {
+        throw new DownloadByRangeError({
+            code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
+            ...requestsLogMeta({ blocksRequest }),
+        });
+        // TODO: this was causing deadlock again. need to come back and fix this so that its possible to process through
+        //       an empty epoch for periods with poor liveness
+        // return {
+        //   result: [],
+        //   warnings: [
+        //     new DownloadByRangeError({
+        //       code: DownloadByRangeErrorCode.MISSING_BLOCKS_RESPONSE,
+        //       ...requestsLogMeta({blocksRequest}),
+        //     }),
+        //   ],
+        // };
+    }
     if (blocks.length > count) {
         throw new DownloadByRangeError({
             code: DownloadByRangeErrorCode.EXTRA_BLOCKS,
@@ -260,13 +283,16 @@ export function validateBlockByRangeResponse(config, blocksRequest, blocks) {
                 throw new DownloadByRangeError({
                     code: DownloadByRangeErrorCode.PARENT_ROOT_MISMATCH,
                     slot: blocks[i].message.slot,
-                    expected: prettyBytes(blockRoot),
-                    actual: prettyBytes(parentRoot),
+                    expected: toRootHex(blockRoot),
+                    actual: toRootHex(parentRoot),
                 }, `Block parent root does not match the previous block's root in BeaconBlocksByRange response`);
             }
         }
     }
-    return response;
+    return {
+        result: response,
+        warnings: null,
+    };
 }
 /**
  * Should not be called directly. Only exported for unit testing purposes.
@@ -313,82 +339,171 @@ export async function validateBlobsByRangeResponse(dataRequestBlocks, blobSideca
 }
 /**
  * Should not be called directly. Only exported for unit testing purposes
+ *
+ * Spec states:
+ * 1) must be within range [start_slot, start_slot + count]
+ * 2) should respond with all columns in the range or and 3:ResourceUnavailable (and potentially get down-scored)
+ * 3) must response with at least the sidecars of the first blob-carrying block that exists in the range
+ * 4) must include all sidecars from each block from which there are blobs
+ * 5) where they exists, sidecars must be sent in (slot, index) order
+ * 6) clients may limit the number of sidecars in a response
+ * 7) clients may stop responding mid-response if their view of fork-choice changes
+ *
+ * We will interpret the spec as follows
+ * - Errors when validating: 1, 3, 5
+ * - Warnings when validating: 2, 4, 6, 7
+ *
+ * For "warning" cases, where we get a partial response but sidecars are validated and correct with respect to the
+ * blocks, then they will be kept.  This loosening of the spec is to help ensure sync goes smoothly and we can find
+ * the data needed in difficult network situations.
+ *
+ * Assume for the following two examples we request indices 5, 10, 15 for a range of slots 32-63
+ *
+ * For slots where we receive no sidecars, example slot 45, but blobs exist we will stop validating subsequent
+ * slots, 45-63.  The next round of requests will get structured to pull the from the slot that had columns
+ * missing to the end of the range for all columns indices that were requested for the current partially failed
+ * request (slots 45-63 and indices 5, 10, 15).
+ *
+ * For slots where only some of the requested sidecars are received we will proceed with validation. For simplicity sake
+ * we will assume that if we only get some indices back for a (or several) slot(s) that the indices we get will be
+ * consistent. IE if a peer returns only index 5, they will most likely return that same index for subsequent slot
+ * (index 5 for slots 34, 35, 36, etc). They will not likely return 5 on slot 34, 10 on slot 35, 15 on slot 36, etc.
+ * This assumption makes the code simpler. For both cases the request for the next round will be structured correctly
+ * to pull any missing column indices for whatever range remains.  The simplification just leads to re-verification
+ * of the columns but the number of columns downloaded will be the same regardless of if they are validated twice.
+ *
+ * validateColumnsByRangeResponse makes some assumptions about the data being passed in
+ * blocks are:
+ * - slotwise in order
+ * - form a chain
+ * - non-sparse response (any missing block is a skipped slot not a bad response)
+ * - last block is last slot received
  */
-export async function validateColumnsByRangeResponse(request, dataRequestBlocks, columnSidecars) {
-    // Expected column count considering currently-validated batch blocks
-    // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
-    const expectedColumnCount = dataRequestBlocks.reduce((acc, { block }) => {
-        return block.message.body.blobKzgCommitments.length > 0
-            ? request.columns.length + acc
-            : acc;
-    }, 0);
-    const nextSlot = dataRequestBlocks.length
-        ? dataRequestBlocks.at(-1).block.message.slot + 1
-        : request.startSlot;
-    const possiblyMissingBlocks = nextSlot - request.startSlot + request.count;
-    // Allow for extra columns if some blocks are missing from the end of a batch
-    // Eg: If we requested 10 blocks but only 8 were returned, allow for up to 2 * columns.length extra columns
-    const maxColumnCount = expectedColumnCount + possiblyMissingBlocks * request.columns.length;
-    if (columnSidecars.length > maxColumnCount) {
-        // this never happens on devnet, so throw error for now
-        throw new DownloadByRangeError({
-            code: DownloadByRangeErrorCode.OVER_COLUMNS,
-            max: maxColumnCount,
-            actual: columnSidecars.length,
-        }, "Extra data columns received in DataColumnSidecarsByRange response");
-    }
+export async function validateColumnsByRangeResponse(config, request, blocks, columnSidecars) {
     const warnings = [];
-    // no need to check for columnSidecars.length  vs expectedColumnCount here, will be checked per-block below
-    const requestedColumns = new Set(request.columns);
-    const validateSidecarsPromises = [];
-    for (let blockIndex = 0, columnSidecarIndex = 0; blockIndex < dataRequestBlocks.length; blockIndex++) {
-        const { block, blockRoot } = dataRequestBlocks[blockIndex];
-        const slot = block.message.slot;
-        const blockRootHex = toRootHex(blockRoot);
-        // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
-        const blockKzgCommitments = block.message.body
-            .blobKzgCommitments;
-        const expectedColumns = blockKzgCommitments.length ? request.columns.length : 0;
-        if (expectedColumns === 0) {
+    const seenColumns = new Map();
+    let currentSlot = -1;
+    let currentIndex = -1;
+    // Check for duplicates and order
+    for (const columnSidecar of columnSidecars) {
+        const slot = columnSidecar.signedBlockHeader.message.slot;
+        let seenSlotColumns = seenColumns.get(slot);
+        if (!seenSlotColumns) {
+            seenSlotColumns = new Map();
+            seenColumns.set(slot, seenSlotColumns);
+        }
+        if (seenSlotColumns.has(columnSidecar.index)) {
+            warnings.push(new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.DUPLICATE_COLUMN,
+                slot,
+                index: columnSidecar.index,
+            }));
             continue;
         }
-        const blockColumnSidecars = [];
-        while (columnSidecarIndex < columnSidecars.length) {
-            const columnSidecar = columnSidecars[columnSidecarIndex];
-            if (columnSidecar.signedBlockHeader.message.slot !== block.message.slot) {
-                // We've reached columns for the next block
-                break;
+        if (currentSlot > slot) {
+            warnings.push(new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
+                slot,
+            }, "ColumnSidecars received out of slot order"));
+        }
+        if (currentSlot === slot && currentIndex > columnSidecar.index) {
+            warnings.push(new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.OUT_OF_ORDER_COLUMNS,
+                slot,
+            }, "Column indices out of order within a slot"));
+        }
+        seenSlotColumns.set(columnSidecar.index, columnSidecar);
+        if (currentSlot !== slot) {
+            // a new slot has started, reset index
+            currentIndex = -1;
+        }
+        else {
+            currentIndex = columnSidecar.index;
+        }
+        currentSlot = slot;
+    }
+    const validationPromises = [];
+    for (const { blockRoot, block } of blocks) {
+        const slot = block.message.slot;
+        const rootHex = toRootHex(blockRoot);
+        const forkName = config.getForkName(slot);
+        const columnSidecarsMap = seenColumns.get(slot) ?? new Map();
+        const columnSidecars = Array.from(columnSidecarsMap.values()).sort((a, b) => a.index - b.index);
+        let blobCount;
+        if (!isForkPostFulu(forkName)) {
+            const dataSlot = columnSidecars.at(0)?.signedBlockHeader.message.slot;
+            throw new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.MISMATCH_BLOCK_FORK,
+                slot,
+                blockFork: forkName,
+                dataFork: dataSlot ? config.getForkName(dataSlot) : "unknown",
+            });
+        }
+        if (isForkPostGloas(forkName)) {
+            // TODO GLOAS: Post-gloas's blobKzgCommitments is not in beacon block body. Need to source it from somewhere else.
+            // if block without columns is passed default to zero and throw below
+            blobCount = 0;
+        }
+        else {
+            blobCount = block.message.body.blobKzgCommitments.length;
+        }
+        if (columnSidecars.length === 0) {
+            if (!blobCount) {
+                // no columns in the slot
+                continue;
             }
-            blockColumnSidecars.push(columnSidecar);
-            columnSidecarIndex++;
+            /**
+             * If no columns are found for a block and there are commitments on the block then stop checking and just
+             * return early. Even if there were columns returned for subsequent slots that doesn't matter because
+             * we will be re-requesting them again anyway.  Leftovers just get ignored
+             */
+            warnings.push(new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.MISSING_COLUMNS,
+                slot,
+                blockRoot: rootHex,
+                missingIndices: prettyPrintIndices(request.columns),
+            }));
+            break;
         }
-        const returnedColumns = new Set(blockColumnSidecars.map((c) => c.index));
-        const missingIndices = request.columns.filter((i) => !returnedColumns.has(i));
+        const returnedColumns = Array.from(columnSidecarsMap.keys()).sort();
+        if (!blobCount) {
+            // columns for a block that does not have blobs
+            // TODO(fulu): should this be a hard error with no data retained from peer or just a warning
+            throw new DownloadByRangeError({
+                code: DownloadByRangeErrorCode.NO_COLUMNS_FOR_BLOCK,
+                slot,
+                blockRoot: rootHex,
+                invalidIndices: prettyPrintIndices(returnedColumns),
+            }, "Block has no blob commitments but data column sidecars were provided");
+        }
+        const missingIndices = request.columns.filter((i) => !columnSidecarsMap.has(i));
         if (missingIndices.length > 0) {
             warnings.push(new DownloadByRangeError({
                 code: DownloadByRangeErrorCode.MISSING_COLUMNS,
                 slot,
-                blockRoot: blockRootHex,
+                blockRoot: rootHex,
                 missingIndices: prettyPrintIndices(missingIndices),
             }, "Missing data columns in DataColumnSidecarsByRange response"));
         }
-        const extraIndices = [...returnedColumns].filter((i) => !requestedColumns.has(i));
+        const extraIndices = returnedColumns.filter((i) => !request.columns.includes(i));
         if (extraIndices.length > 0) {
             warnings.push(new DownloadByRangeError({
                 code: DownloadByRangeErrorCode.EXTRA_COLUMNS,
                 slot,
-                blockRoot: blockRootHex,
+                blockRoot: rootHex,
                 invalidIndices: prettyPrintIndices(extraIndices),
             }, "Data column in not in requested columns in DataColumnSidecarsByRange response"));
         }
-        validateSidecarsPromises.push(validateBlockDataColumnSidecars(slot, blockRoot, blockKzgCommitments.length, blockColumnSidecars).then(() => ({
+        validationPromises.push(validateBlockDataColumnSidecars(slot, blockRoot, blobCount, columnSidecars).then(() => ({
             blockRoot,
-            columnSidecars: blockColumnSidecars,
+            columnSidecars,
         })));
     }
-    // Await all sidecar validations in parallel
-    const result = await Promise.all(validateSidecarsPromises);
-    return { result, warnings: warnings.length ? warnings : null };
+    const validatedColumns = await Promise.all(validationPromises);
+    return {
+        result: validatedColumns,
+        warnings: warnings.length ? warnings : null,
+    };
 }
 /**
  * Given a data request, return only the blocks and roots that correspond to the data request (sorted). Assumes that
@@ -438,7 +553,7 @@ function requestsLogMeta({ blocksRequest, blobsRequest, columnsRequest }) {
 }
 export var DownloadByRangeErrorCode;
 (function (DownloadByRangeErrorCode) {
-    DownloadByRangeErrorCode["MISSING_BLOCKS"] = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCKS";
+    DownloadByRangeErrorCode["MISSING_BLOCKS_RESPONSE"] = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOCK_RESPONSE";
     DownloadByRangeErrorCode["MISSING_BLOBS_RESPONSE"] = "DOWNLOAD_BY_RANGE_ERROR_MISSING_BLOBS_RESPONSE";
     DownloadByRangeErrorCode["MISSING_COLUMNS_RESPONSE"] = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS_RESPONSE";
     /** Error at the reqresp layer */
@@ -454,7 +569,11 @@ export var DownloadByRangeErrorCode;
     DownloadByRangeErrorCode["MISSING_COLUMNS"] = "DOWNLOAD_BY_RANGE_ERROR_MISSING_COLUMNS";
     DownloadByRangeErrorCode["OVER_COLUMNS"] = "DOWNLOAD_BY_RANGE_ERROR_OVER_COLUMNS";
     DownloadByRangeErrorCode["EXTRA_COLUMNS"] = "DOWNLOAD_BY_RANGE_ERROR_EXTRA_COLUMNS";
+    DownloadByRangeErrorCode["NO_COLUMNS_FOR_BLOCK"] = "DOWNLOAD_BY_RANGE_ERROR_NO_COLUMNS_FOR_BLOCK";
+    DownloadByRangeErrorCode["DUPLICATE_COLUMN"] = "DOWNLOAD_BY_RANGE_ERROR_DUPLICATE_COLUMN";
+    DownloadByRangeErrorCode["OUT_OF_ORDER_COLUMNS"] = "DOWNLOAD_BY_RANGE_OUT_OF_ORDER_COLUMNS";
     /** Cached block input type mismatches new data */
+    DownloadByRangeErrorCode["MISMATCH_BLOCK_FORK"] = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_FORK";
     DownloadByRangeErrorCode["MISMATCH_BLOCK_INPUT_TYPE"] = "DOWNLOAD_BY_RANGE_ERROR_MISMATCH_BLOCK_INPUT_TYPE";
 })(DownloadByRangeErrorCode || (DownloadByRangeErrorCode = {}));
 export class DownloadByRangeError extends LodestarError {