@lodestar/beacon-node 1.43.0-dev.87cbe69c66 → 1.43.0-dev.899087b3b2

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (89) hide show
  1. package/lib/chain/blocks/blockInput/blockInput.d.ts +3 -0
  2. package/lib/chain/blocks/blockInput/blockInput.d.ts.map +1 -1
  3. package/lib/chain/blocks/blockInput/blockInput.js +4 -1
  4. package/lib/chain/blocks/blockInput/blockInput.js.map +1 -1
  5. package/lib/chain/blocks/importBlock.d.ts.map +1 -1
  6. package/lib/chain/blocks/importBlock.js +0 -16
  7. package/lib/chain/blocks/importBlock.js.map +1 -1
  8. package/lib/chain/blocks/importExecutionPayload.d.ts +4 -0
  9. package/lib/chain/blocks/importExecutionPayload.d.ts.map +1 -1
  10. package/lib/chain/blocks/importExecutionPayload.js +12 -1
  11. package/lib/chain/blocks/importExecutionPayload.js.map +1 -1
  12. package/lib/chain/blocks/index.d.ts.map +1 -1
  13. package/lib/chain/blocks/index.js +6 -7
  14. package/lib/chain/blocks/index.js.map +1 -1
  15. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts +11 -1
  16. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.d.ts.map +1 -1
  17. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js +23 -1
  18. package/lib/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.js.map +1 -1
  19. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts +16 -0
  20. package/lib/chain/blocks/payloadEnvelopeInput/types.d.ts.map +1 -1
  21. package/lib/chain/blocks/verifyExecutionPayloadEnvelope.d.ts.map +1 -1
  22. package/lib/chain/blocks/verifyExecutionPayloadEnvelope.js +5 -4
  23. package/lib/chain/blocks/verifyExecutionPayloadEnvelope.js.map +1 -1
  24. package/lib/chain/chain.d.ts.map +1 -1
  25. package/lib/chain/chain.js +25 -10
  26. package/lib/chain/chain.js.map +1 -1
  27. package/lib/chain/initState.d.ts.map +1 -1
  28. package/lib/chain/initState.js +6 -1
  29. package/lib/chain/initState.js.map +1 -1
  30. package/lib/chain/prepareNextSlot.js +1 -1
  31. package/lib/chain/prepareNextSlot.js.map +1 -1
  32. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts +14 -10
  33. package/lib/chain/seenCache/seenPayloadEnvelopeInput.d.ts.map +1 -1
  34. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js +51 -17
  35. package/lib/chain/seenCache/seenPayloadEnvelopeInput.js.map +1 -1
  36. package/lib/chain/seenCache/seenProposerPreferences.d.ts +1 -1
  37. package/lib/chain/seenCache/seenProposerPreferences.d.ts.map +1 -1
  38. package/lib/chain/seenCache/seenProposerPreferences.js +6 -11
  39. package/lib/chain/seenCache/seenProposerPreferences.js.map +1 -1
  40. package/lib/network/processor/gossipHandlers.d.ts.map +1 -1
  41. package/lib/network/processor/gossipHandlers.js +14 -0
  42. package/lib/network/processor/gossipHandlers.js.map +1 -1
  43. package/lib/sync/constants.d.ts +3 -1
  44. package/lib/sync/constants.d.ts.map +1 -1
  45. package/lib/sync/constants.js +3 -4
  46. package/lib/sync/constants.js.map +1 -1
  47. package/lib/sync/range/batch.d.ts +12 -3
  48. package/lib/sync/range/batch.d.ts.map +1 -1
  49. package/lib/sync/range/batch.js +163 -33
  50. package/lib/sync/range/batch.js.map +1 -1
  51. package/lib/sync/range/chain.d.ts +13 -2
  52. package/lib/sync/range/chain.d.ts.map +1 -1
  53. package/lib/sync/range/chain.js +38 -4
  54. package/lib/sync/range/chain.js.map +1 -1
  55. package/lib/sync/range/range.d.ts.map +1 -1
  56. package/lib/sync/range/range.js +14 -3
  57. package/lib/sync/range/range.js.map +1 -1
  58. package/lib/sync/sync.d.ts.map +1 -1
  59. package/lib/sync/sync.js +13 -0
  60. package/lib/sync/sync.js.map +1 -1
  61. package/lib/sync/unknownBlock.d.ts +7 -0
  62. package/lib/sync/unknownBlock.d.ts.map +1 -1
  63. package/lib/sync/unknownBlock.js +138 -10
  64. package/lib/sync/unknownBlock.js.map +1 -1
  65. package/lib/sync/utils/downloadByRange.d.ts +29 -8
  66. package/lib/sync/utils/downloadByRange.d.ts.map +1 -1
  67. package/lib/sync/utils/downloadByRange.js +87 -42
  68. package/lib/sync/utils/downloadByRange.js.map +1 -1
  69. package/package.json +16 -16
  70. package/src/chain/blocks/blockInput/blockInput.ts +4 -1
  71. package/src/chain/blocks/importBlock.ts +0 -18
  72. package/src/chain/blocks/importExecutionPayload.ts +18 -6
  73. package/src/chain/blocks/index.ts +5 -7
  74. package/src/chain/blocks/payloadEnvelopeInput/payloadEnvelopeInput.ts +31 -2
  75. package/src/chain/blocks/payloadEnvelopeInput/types.ts +17 -0
  76. package/src/chain/blocks/verifyExecutionPayloadEnvelope.ts +7 -4
  77. package/src/chain/chain.ts +29 -9
  78. package/src/chain/initState.ts +9 -1
  79. package/src/chain/prepareNextSlot.ts +1 -1
  80. package/src/chain/seenCache/seenPayloadEnvelopeInput.ts +66 -20
  81. package/src/chain/seenCache/seenProposerPreferences.ts +7 -12
  82. package/src/network/processor/gossipHandlers.ts +18 -0
  83. package/src/sync/constants.ts +4 -4
  84. package/src/sync/range/batch.ts +187 -38
  85. package/src/sync/range/chain.ts +52 -5
  86. package/src/sync/range/range.ts +16 -3
  87. package/src/sync/sync.ts +13 -1
  88. package/src/sync/unknownBlock.ts +170 -10
  89. package/src/sync/utils/downloadByRange.ts +148 -44
@@ -63,6 +63,27 @@ enum FetchResult {
63
63
  FailureMaxAttempts = "failure_max_attempts",
64
64
  }
65
65
 
66
+ class UnknownBlockRateLimitedError extends Error {
67
+ constructor(message: string) {
68
+ super(message);
69
+ this.name = "UnknownBlockRateLimitedError";
70
+ }
71
+ }
72
+
73
+ function getRateLimitedUntilMs(e: unknown): number | null {
74
+ if (!(e instanceof RequestError)) {
75
+ return null;
76
+ }
77
+
78
+ switch (e.type.code) {
79
+ case RequestErrorCode.RESP_RATE_LIMITED:
80
+ case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
81
+ return e.type.rateLimitedUntilMs ?? null;
82
+ default:
83
+ return null;
84
+ }
85
+ }
86
+
66
87
  /**
67
88
  * BlockInputSync is a class that handles ReqResp to find blocks and data related to a specific blockRoot. The
68
89
  * blockRoot may have been found via object gossip, or the API. Gossip objects that can trigger a search are block,
@@ -106,6 +127,7 @@ export class BlockInputSync {
106
127
  private readonly maxPendingBlocks;
107
128
  private subscribedToNetworkEvents = false;
108
129
  private peerBalancer: UnknownBlockPeerBalancer;
130
+ private rateLimitBackoffTimeout: NodeJS.Timeout | undefined;
109
131
 
110
132
  constructor(
111
133
  private readonly config: ChainForkConfig,
@@ -155,6 +177,7 @@ export class BlockInputSync {
155
177
 
156
178
  unsubscribeFromNetwork(): void {
157
179
  this.logger.verbose("BlockInputSync disabled.");
180
+ this.clearRateLimitBackoffTimer();
158
181
  this.chain.emitter.off(ChainEvent.unknownBlockRoot, this.onUnknownBlockRoot);
159
182
  this.chain.emitter.off(ChainEvent.unknownEnvelopeBlockRoot, this.onUnknownEnvelopeBlockRoot);
160
183
  this.chain.emitter.off(ChainEvent.incompleteBlockInput, this.onIncompleteBlockInput);
@@ -402,6 +425,7 @@ export class BlockInputSync {
402
425
  private onPeerDisconnected = (data: NetworkEventData[NetworkEvent.peerDisconnected]): void => {
403
426
  const peerId = data.peer;
404
427
  this.peerBalancer.onPeerDisconnected(peerId);
428
+ this.scheduleRateLimitBackoffRetry();
405
429
  };
406
430
 
407
431
  /**
@@ -519,7 +543,7 @@ export class BlockInputSync {
519
543
  */
520
544
  private triggerUnknownBlockSearch = (): void => {
521
545
  // Cheap early stop to prevent calling the network.getConnectedPeers()
522
- if (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0) {
546
+ if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
523
547
  return;
524
548
  }
525
549
 
@@ -607,6 +631,36 @@ export class BlockInputSync {
607
631
  }
608
632
  };
609
633
 
634
+ private scheduleRateLimitBackoffRetry(): void {
635
+ this.clearRateLimitBackoffTimer();
636
+
637
+ if (!this.subscribedToNetworkEvents || (this.pendingBlocks.size === 0 && this.pendingPayloads.size === 0)) {
638
+ return;
639
+ }
640
+
641
+ const now = Date.now();
642
+ const retryAt = this.peerBalancer.getNextRateLimitRetryAt();
643
+ if (retryAt === null) {
644
+ return;
645
+ }
646
+
647
+ this.rateLimitBackoffTimeout = setTimeout(
648
+ () => {
649
+ this.rateLimitBackoffTimeout = undefined;
650
+ this.triggerUnknownBlockSearch();
651
+ this.scheduleRateLimitBackoffRetry();
652
+ },
653
+ Math.max(0, retryAt - now)
654
+ );
655
+ }
656
+
657
+ private clearRateLimitBackoffTimer(): void {
658
+ if (this.rateLimitBackoffTimeout !== undefined) {
659
+ clearTimeout(this.rateLimitBackoffTimeout);
660
+ this.rateLimitBackoffTimeout = undefined;
661
+ }
662
+ }
663
+
610
664
  private async downloadBlock(block: BlockInputSyncCacheItem): Promise<void> {
611
665
  if (block.status !== PendingBlockInputStatus.pending) {
612
666
  return;
@@ -678,6 +732,16 @@ export class BlockInputSync {
678
732
  this.onUnknownBlockRoot({rootHex: pending.blockInput.parentRootHex, source: BlockInputSource.byRoot});
679
733
  }
680
734
  } else {
735
+ if (res.err instanceof UnknownBlockRateLimitedError) {
736
+ const pendingBlock = this.pendingBlocks.get(rootHex);
737
+ if (pendingBlock) {
738
+ pendingBlock.status = PendingBlockInputStatus.pending;
739
+ }
740
+ this.logger.debug("Deferring unknown block download due to peer rate limit", logCtx, res.err);
741
+ this.scheduleRateLimitBackoffRetry();
742
+ return;
743
+ }
744
+
681
745
  this.metrics?.blockInputSync.downloadedBlocksError.inc();
682
746
  this.logger.debug("Ignoring unknown block root after many failed downloads", logCtx, res.err);
683
747
  this.removeAndDownScoreAllDescendants(block);
@@ -994,12 +1058,19 @@ export class BlockInputSync {
994
1058
  let envelope = payloadInput?.hasPayloadEnvelope() ? payloadInput.getPayloadEnvelope() : undefined;
995
1059
 
996
1060
  let i = 0;
1061
+ let deferredByRateLimit = false;
997
1062
  while (i++ < this.getMaxDownloadAttempts()) {
998
1063
  const pendingColumns = payloadInput?.hasAllData()
999
1064
  ? new Set<number>()
1000
1065
  : new Set(payloadInput?.getMissingSampledColumnMeta().missing ?? []);
1001
1066
  const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
1002
1067
  if (peerMeta === null) {
1068
+ if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
1069
+ throw new UnknownBlockRateLimitedError(
1070
+ `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
1071
+ );
1072
+ }
1073
+
1003
1074
  throw Error(
1004
1075
  `Error fetching payload by root slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`
1005
1076
  );
@@ -1076,7 +1147,12 @@ export class BlockInputSync {
1076
1147
  e as Error
1077
1148
  );
1078
1149
 
1079
- if (e instanceof RequestError) {
1150
+ const rateLimitedUntilMs = getRateLimitedUntilMs(e);
1151
+ if (rateLimitedUntilMs !== null) {
1152
+ deferredByRateLimit = true;
1153
+ this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
1154
+ this.scheduleRateLimitBackoffRetry();
1155
+ } else if (e instanceof RequestError) {
1080
1156
  switch (e.type.code) {
1081
1157
  case RequestErrorCode.REQUEST_RATE_LIMITED:
1082
1158
  case RequestErrorCode.REQUEST_TIMEOUT:
@@ -1093,6 +1169,12 @@ export class BlockInputSync {
1093
1169
  }
1094
1170
  }
1095
1171
 
1172
+ if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
1173
+ throw new UnknownBlockRateLimitedError(
1174
+ `Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts: peers are rate-limited`
1175
+ );
1176
+ }
1177
+
1096
1178
  throw Error(`Error fetching payload with slot=${slot} root=${rootHex} after ${i - 1} attempts.`);
1097
1179
  }
1098
1180
 
@@ -1176,6 +1258,7 @@ export class BlockInputSync {
1176
1258
  }
1177
1259
 
1178
1260
  let i = 0;
1261
+ let deferredByRateLimit = false;
1179
1262
  while (i++ < this.getMaxDownloadAttempts()) {
1180
1263
  const pendingColumns =
1181
1264
  isPendingBlockInput(cacheItem) && isBlockInputColumns(cacheItem.blockInput)
@@ -1183,6 +1266,12 @@ export class BlockInputSync {
1183
1266
  : defaultPendingColumns;
1184
1267
  const peerMeta = this.peerBalancer.bestPeerForPendingColumns(pendingColumns, excludedPeers);
1185
1268
  if (peerMeta === null) {
1269
+ if (this.peerBalancer.getNextRateLimitRetryAt(pendingColumns, excludedPeers) !== null) {
1270
+ throw new UnknownBlockRateLimitedError(
1271
+ `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: peers with needed columns are rate-limited`
1272
+ );
1273
+ }
1274
+
1186
1275
  // no more peer with needed columns to try, throw error
1187
1276
  const message = `Error fetching UnknownBlockRoot slot=${slot} root=${rootHex} after ${i}: cannot find peer with needed columns=${prettyPrintIndices(Array.from(pendingColumns))}`;
1188
1277
  this.metrics?.blockInputSync.fetchTimeSec.observe(
@@ -1238,14 +1327,23 @@ export class BlockInputSync {
1238
1327
  } else if (e instanceof RequestError) {
1239
1328
  // should look into req_resp metrics in this case
1240
1329
  downloadByRootMetrics?.error.inc({code: "req_resp", client: peerClient});
1241
- switch (e.type.code) {
1242
- case RequestErrorCode.REQUEST_RATE_LIMITED:
1243
- case RequestErrorCode.REQUEST_TIMEOUT:
1244
- // do not exclude peer for these errors
1245
- break;
1246
- default:
1247
- excludedPeers.add(peerId);
1248
- break;
1330
+ const rateLimitedUntilMs = getRateLimitedUntilMs(e);
1331
+ if (rateLimitedUntilMs !== null) {
1332
+ deferredByRateLimit = true;
1333
+ this.peerBalancer.onRateLimited(peerId, rateLimitedUntilMs);
1334
+ this.scheduleRateLimitBackoffRetry();
1335
+ } else {
1336
+ switch (e.type.code) {
1337
+ case RequestErrorCode.REQUEST_RATE_LIMITED:
1338
+ case RequestErrorCode.RESP_RATE_LIMITED:
1339
+ case RequestErrorCode.REQUEST_SELF_RATE_LIMITED:
1340
+ case RequestErrorCode.REQUEST_TIMEOUT:
1341
+ // do not exclude peer for these errors
1342
+ break;
1343
+ default:
1344
+ excludedPeers.add(peerId);
1345
+ break;
1346
+ }
1249
1347
  }
1250
1348
  } else {
1251
1349
  // investigate if this happens
@@ -1273,6 +1371,10 @@ export class BlockInputSync {
1273
1371
 
1274
1372
  const message = `Error fetching BlockInput with slot=${slot} root=${rootHex} after ${i - 1} attempts.`;
1275
1373
 
1374
+ if (deferredByRateLimit && this.peerBalancer.getNextRateLimitRetryAt() !== null) {
1375
+ throw new UnknownBlockRateLimitedError(`${message} Peers are rate-limited.`);
1376
+ }
1377
+
1276
1378
  if (!isPendingBlockInput(cacheItem)) {
1277
1379
  throw Error(`${message} No block and no data was found.`);
1278
1380
  }
@@ -1404,10 +1506,12 @@ export class BlockInputSync {
1404
1506
  export class UnknownBlockPeerBalancer {
1405
1507
  readonly peersMeta: Map<PeerIdStr, PeerSyncMeta>;
1406
1508
  readonly activeRequests: Map<PeerIdStr, number>;
1509
+ readonly rateLimitedUntilByPeer: Map<PeerIdStr, number>;
1407
1510
 
1408
1511
  constructor() {
1409
1512
  this.peersMeta = new Map();
1410
1513
  this.activeRequests = new Map();
1514
+ this.rateLimitedUntilByPeer = new Map();
1411
1515
  }
1412
1516
 
1413
1517
  /** Trigger on each peer re-status */
@@ -1422,6 +1526,41 @@ export class UnknownBlockPeerBalancer {
1422
1526
  onPeerDisconnected(peerId: PeerIdStr): void {
1423
1527
  this.peersMeta.delete(peerId);
1424
1528
  this.activeRequests.delete(peerId);
1529
+ this.rateLimitedUntilByPeer.delete(peerId);
1530
+ }
1531
+
1532
+ onRateLimited(peerId: PeerIdStr, rateLimitedUntilMs: number): void {
1533
+ this.rateLimitedUntilByPeer.set(peerId, rateLimitedUntilMs);
1534
+ }
1535
+
1536
+ getNextRateLimitRetryAt(pendingColumns?: Set<number>, excludedPeers?: Set<PeerIdStr>): number | null {
1537
+ const now = Date.now();
1538
+ let retryAt: number | null = null;
1539
+
1540
+ for (const [peerId, rateLimitedUntil] of this.rateLimitedUntilByPeer.entries()) {
1541
+ if (rateLimitedUntil <= now) {
1542
+ this.rateLimitedUntilByPeer.delete(peerId);
1543
+ continue;
1544
+ }
1545
+
1546
+ if (excludedPeers?.has(peerId)) {
1547
+ continue;
1548
+ }
1549
+
1550
+ const syncMeta = this.peersMeta.get(peerId);
1551
+ if (syncMeta === undefined) {
1552
+ this.rateLimitedUntilByPeer.delete(peerId);
1553
+ continue;
1554
+ }
1555
+
1556
+ if (pendingColumns !== undefined && !this.peerHasPendingColumns(syncMeta, pendingColumns)) {
1557
+ continue;
1558
+ }
1559
+
1560
+ retryAt = Math.min(retryAt ?? rateLimitedUntil, rateLimitedUntil);
1561
+ }
1562
+
1563
+ return retryAt;
1425
1564
  }
1426
1565
 
1427
1566
  /**
@@ -1470,6 +1609,7 @@ export class UnknownBlockPeerBalancer {
1470
1609
  }
1471
1610
 
1472
1611
  private filterPeers(pendingDataColumns: Set<number>, excludedPeers: Set<PeerIdStr>): PeerIdStr[] {
1612
+ const now = Date.now();
1473
1613
  let maxColumnCount = 0;
1474
1614
  const considerPeers: {peerId: PeerIdStr; columnCount: number}[] = [];
1475
1615
  for (const [peerId, syncMeta] of this.peersMeta.entries()) {
@@ -1478,12 +1618,24 @@ export class UnknownBlockPeerBalancer {
1478
1618
  continue;
1479
1619
  }
1480
1620
 
1621
+ const rateLimitedUntil = this.rateLimitedUntilByPeer.get(peerId);
1622
+ if (rateLimitedUntil !== undefined) {
1623
+ if (now < rateLimitedUntil) {
1624
+ continue;
1625
+ }
1626
+ this.rateLimitedUntilByPeer.delete(peerId);
1627
+ }
1628
+
1481
1629
  const activeRequests = this.activeRequests.get(peerId) ?? 0;
1482
1630
  if (activeRequests >= MAX_CONCURRENT_REQUESTS) {
1483
1631
  // should return peer with no more than MAX_CONCURRENT_REQUESTS active requests
1484
1632
  continue;
1485
1633
  }
1486
1634
 
1635
+ if (!this.peerHasPendingColumns(syncMeta, pendingDataColumns)) {
1636
+ continue;
1637
+ }
1638
+
1487
1639
  if (pendingDataColumns.size === 0) {
1488
1640
  considerPeers.push({peerId, columnCount: 0});
1489
1641
  continue;
@@ -1517,4 +1669,12 @@ export class UnknownBlockPeerBalancer {
1517
1669
 
1518
1670
  return eligiblePeers;
1519
1671
  }
1672
+
1673
+ private peerHasPendingColumns(syncMeta: PeerSyncMeta, pendingDataColumns: Set<number>): boolean {
1674
+ if (pendingDataColumns.size === 0) {
1675
+ return true;
1676
+ }
1677
+
1678
+ return syncMeta.custodyColumns.some((column) => pendingDataColumns.has(column));
1679
+ }
1520
1680
  }
@@ -8,13 +8,14 @@ import {
8
8
  isForkPostGloas,
9
9
  } from "@lodestar/params";
10
10
  import {
11
+ ColumnIndex,
11
12
  DataColumnSidecar,
13
+ RootHex,
12
14
  SignedBeaconBlock,
13
15
  Slot,
14
16
  deneb,
15
17
  fulu,
16
18
  gloas,
17
- isGloasBeaconBlock,
18
19
  isGloasDataColumnSidecar,
19
20
  phase0,
20
21
  } from "@lodestar/types";
@@ -46,6 +47,22 @@ export type DownloadByRangeRequests = {
46
47
  blobsRequest?: deneb.BlobSidecarsByRangeRequest;
47
48
  columnsRequest?: fulu.DataColumnSidecarsByRangeRequest;
48
49
  envelopesRequest?: gloas.ExecutionPayloadEnvelopesByRangeRequest;
50
+ /**
51
+ * Post-Gloas only. Fetches the dangling-parent's payload envelope and/or its missing sampled
52
+ * data columns by-root. Set by `Batch` for the first batch of a `SyncChain` after the first
53
+ * block's parentRoot is known.
54
+ */
55
+ parentPayloadRequest?: {
56
+ blockRoot?: Uint8Array;
57
+ columns?: ColumnIndex[];
58
+ envelopeBlockRoot?: Uint8Array;
59
+ };
60
+ };
61
+
62
+ export type ParentPayloadCommitments = {
63
+ blockRoot: Uint8Array;
64
+ blockRootHex: RootHex;
65
+ kzgCommitments: deneb.BlobKzgCommitments;
49
66
  };
50
67
 
51
68
  export type DownloadByRangeResponses = {
@@ -61,6 +78,8 @@ export type DownloadAndCacheByRangeProps = DownloadByRangeRequests & {
61
78
  logger: Logger;
62
79
  peerIdStr: string;
63
80
  batchBlocks?: IBlockInput[];
81
+ /** Required when `parentPayloadRequest` is set; supplies the data needed to validate the parent's columns. */
82
+ parentPayloadCommitments?: ParentPayloadCommitments;
64
83
  peerDasMetrics?: BeaconMetrics["peerDas"] | null;
65
84
  };
66
85
 
@@ -202,23 +221,17 @@ export function cacheByRangeResponses({
202
221
  });
203
222
  }
204
223
 
205
- // Attach envelopes to entries whose envelope was returned by the peer. The returned
206
- // payloadEnvelopes map only contains entries with envelopes ready for importExecutionPayload.
207
- let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null = null;
224
+ let payloadEnvelopes: Map<Slot, PayloadEnvelopeInput> | null =
225
+ existingPayloadEnvelopes !== null ? new Map(existingPayloadEnvelopes) : null;
208
226
  if (downloadedPayloadEnvelopes !== null) {
209
- payloadEnvelopes = new Map(existingPayloadEnvelopes ?? []);
210
-
227
+ payloadEnvelopes ??= new Map();
211
228
  for (const [slot, envelope] of downloadedPayloadEnvelopes) {
212
- const blockInput = updatedBatchBlocks.get(slot);
213
- if (!blockInput?.hasBlock() || !isForkPostGloas(blockInput.forkName)) {
214
- // No block to pair this envelope with; drop silently
215
- continue;
216
- }
217
-
218
- const payloadInput = seenPayloadEnvelopeInputCache.get(blockInput.blockRootHex);
229
+ const envelopeBlockRootHex = toRootHex(envelope.message.beaconBlockRoot);
230
+ const payloadInput = seenPayloadEnvelopeInputCache.get(envelopeBlockRootHex);
219
231
  if (payloadInput === undefined) {
220
232
  // Unreachable given the loop above seeded an entry for every gloas block in the batch.
221
- continue;
233
+ // for the parent block, it's populated at BeaconChain init
234
+ throw new Error(`Missing PayloadEnvelopeInput for block ${envelopeBlockRootHex}`);
222
235
  }
223
236
 
224
237
  if (!payloadInput.hasPayloadEnvelope()) {
@@ -308,6 +321,8 @@ export async function downloadByRange({
308
321
  blobsRequest,
309
322
  columnsRequest,
310
323
  envelopesRequest,
324
+ parentPayloadRequest,
325
+ parentPayloadCommitments,
311
326
  peerDasMetrics,
312
327
  }: DownloadAndCacheByRangeProps): Promise<
313
328
  WarnResult<
@@ -324,6 +339,7 @@ export async function downloadByRange({
324
339
  blobsRequest,
325
340
  columnsRequest,
326
341
  envelopesRequest,
342
+ parentPayloadRequest,
327
343
  });
328
344
  } catch (err) {
329
345
  throw new DownloadByRangeError({
@@ -340,6 +356,8 @@ export async function downloadByRange({
340
356
  blobsRequest,
341
357
  columnsRequest,
342
358
  envelopesRequest,
359
+ parentPayloadRequest,
360
+ parentPayloadCommitments,
343
361
  peerDasMetrics,
344
362
  ...response,
345
363
  });
@@ -355,13 +373,14 @@ export async function requestByRange({
355
373
  blobsRequest,
356
374
  columnsRequest,
357
375
  envelopesRequest,
376
+ parentPayloadRequest,
358
377
  }: DownloadByRangeRequests & {
359
378
  network: INetwork;
360
379
  peerIdStr: PeerIdStr;
361
380
  }): Promise<DownloadByRangeResponses> {
362
381
  let blocks: undefined | SignedBeaconBlock[];
363
382
  let blobSidecars: undefined | deneb.BlobSidecars;
364
- let columnSidecars: undefined | DataColumnSidecar[];
383
+ const columnSidecars: DataColumnSidecar[] = [];
365
384
  const payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[] = [];
366
385
 
367
386
  const requests: Promise<unknown>[] = [];
@@ -370,17 +389,6 @@ export async function requestByRange({
370
389
  requests.push(
371
390
  network.sendBeaconBlocksByRange(peerIdStr, blocksRequest).then((blockResponse) => {
372
391
  blocks = blockResponse;
373
- const firstBlock = blockResponse.at(0);
374
- if (firstBlock && isGloasBeaconBlock(firstBlock.message)) {
375
- return network
376
- .sendExecutionPayloadEnvelopesByRoot(peerIdStr, [
377
- firstBlock.message.body.signedExecutionPayloadBid.message.parentBlockRoot,
378
- ])
379
- .then((envelopeResponse) => {
380
- payloadEnvelopes?.unshift(...envelopeResponse);
381
- });
382
- }
383
- return undefined;
384
392
  })
385
393
  );
386
394
  }
@@ -396,7 +404,7 @@ export async function requestByRange({
396
404
  if (columnsRequest) {
397
405
  requests.push(
398
406
  network.sendDataColumnSidecarsByRange(peerIdStr, columnsRequest).then((columnResponse) => {
399
- columnSidecars = columnResponse;
407
+ columnSidecars.push(...columnResponse);
400
408
  })
401
409
  );
402
410
  }
@@ -404,11 +412,34 @@ export async function requestByRange({
404
412
  if (envelopesRequest) {
405
413
  requests.push(
406
414
  network.sendExecutionPayloadEnvelopesByRange(peerIdStr, envelopesRequest).then((envelopeResponse) => {
407
- payloadEnvelopes?.push(...envelopeResponse);
415
+ payloadEnvelopes.push(...envelopeResponse);
408
416
  })
409
417
  );
410
418
  }
411
419
 
420
+ // Only happens on the 1st batch of a SyncChain — fetches whichever pieces of the
421
+ // dangling-parent's payload are still missing from the seen-cache entry.
422
+ if (parentPayloadRequest?.envelopeBlockRoot) {
423
+ requests.push(
424
+ network
425
+ .sendExecutionPayloadEnvelopesByRoot(peerIdStr, [parentPayloadRequest.envelopeBlockRoot])
426
+ .then((envelopeResponse) => {
427
+ payloadEnvelopes.push(...envelopeResponse);
428
+ })
429
+ );
430
+ }
431
+ if (parentPayloadRequest?.blockRoot && parentPayloadRequest.columns && parentPayloadRequest.columns.length > 0) {
432
+ requests.push(
433
+ network
434
+ .sendDataColumnSidecarsByRoot(peerIdStr, [
435
+ {blockRoot: parentPayloadRequest.blockRoot, columns: parentPayloadRequest.columns},
436
+ ])
437
+ .then((columnResponse) => {
438
+ columnSidecars.push(...columnResponse);
439
+ })
440
+ );
441
+ }
442
+
412
443
  await Promise.all(requests);
413
444
 
414
445
  return {
@@ -429,6 +460,8 @@ export async function validateResponses({
429
460
  blobsRequest,
430
461
  columnsRequest,
431
462
  envelopesRequest,
463
+ parentPayloadRequest,
464
+ parentPayloadCommitments,
432
465
  blocks,
433
466
  blobSidecars,
434
467
  columnSidecars,
@@ -438,6 +471,7 @@ export async function validateResponses({
438
471
  DownloadByRangeResponses & {
439
472
  config: ChainForkConfig;
440
473
  batchBlocks?: IBlockInput[];
474
+ parentPayloadCommitments?: ParentPayloadCommitments;
441
475
  peerDasMetrics?: BeaconMetrics["peerDas"] | null;
442
476
  }): Promise<
443
477
  WarnResult<
@@ -458,6 +492,11 @@ export async function validateResponses({
458
492
  );
459
493
  }
460
494
 
495
+ // `parentPayloadRequest` and `parentPayloadCommitments` must be supplied together
496
+ if ((parentPayloadRequest === undefined) !== (parentPayloadCommitments === undefined)) {
497
+ throw new Error("Coding error: parentPayloadRequest and parentPayloadCommitments must be both set or both unset");
498
+ }
499
+
461
500
  const validatedResponses: ValidatedResponses = {};
462
501
  let warnings: DownloadByRangeError[] | null = null;
463
502
 
@@ -469,21 +508,30 @@ export async function validateResponses({
469
508
  validatedResponses.validatedBlocks = result.result;
470
509
  }
471
510
 
511
+ const needsEnvelopeValidation = !!envelopesRequest || parentPayloadCommitments !== undefined;
472
512
  const dataRequest = blobsRequest ?? columnsRequest;
473
- if (!dataRequest && !envelopesRequest) {
513
+ if (!dataRequest && !needsEnvelopeValidation) {
474
514
  return {result: {responses: validatedResponses, payloadEnvelopes: null}, warnings};
475
515
  }
476
516
 
477
517
  if (!dataRequest) {
478
- // Only envelope validation needed
479
- let validatedPayloadEnvelopes: Map<Slot, gloas.SignedExecutionPayloadEnvelope> | null = null;
480
- if (envelopesRequest) {
481
- validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
482
- validatedResponses.validatedBlocks ?? [],
483
- batchBlocks,
484
- payloadEnvelopes ?? []
518
+ // Only envelope and/or parent-by-root validation needed
519
+ if (parentPayloadCommitments !== undefined) {
520
+ const parentValidated = await validateParentPayloadColumns(
521
+ parentPayloadCommitments,
522
+ columnSidecars ?? [],
523
+ peerDasMetrics
485
524
  );
525
+ if (parentValidated) {
526
+ validatedResponses.validatedColumnSidecars = [parentValidated];
527
+ }
486
528
  }
529
+ const validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
530
+ validatedResponses.validatedBlocks ?? [],
531
+ batchBlocks,
532
+ payloadEnvelopes ?? [],
533
+ parentPayloadCommitments
534
+ );
487
535
  return {result: {responses: validatedResponses, payloadEnvelopes: validatedPayloadEnvelopes}, warnings};
488
536
  }
489
537
 
@@ -542,19 +590,64 @@ export async function validateResponses({
542
590
  warnings = validatedColumnSidecarsResult.warnings;
543
591
  }
544
592
 
545
- // Validate envelopes if an envelopes request was made
593
+ // Parent columns (by-root): KZG-validate against parent's bid commitments and append.
594
+ if (parentPayloadCommitments !== undefined) {
595
+ const parentValidated = await validateParentPayloadColumns(
596
+ parentPayloadCommitments,
597
+ columnSidecars ?? [],
598
+ peerDasMetrics
599
+ );
600
+ if (parentValidated) {
601
+ validatedResponses.validatedColumnSidecars = [
602
+ ...(validatedResponses.validatedColumnSidecars ?? []),
603
+ parentValidated,
604
+ ];
605
+ }
606
+ }
607
+
546
608
  let validatedPayloadEnvelopes: Map<Slot, gloas.SignedExecutionPayloadEnvelope> | null = null;
547
- if (envelopesRequest) {
609
+ if (needsEnvelopeValidation) {
548
610
  validatedPayloadEnvelopes = validateEnvelopesByRangeResponse(
549
611
  validatedResponses.validatedBlocks ?? [],
550
612
  batchBlocks,
551
- payloadEnvelopes ?? []
613
+ payloadEnvelopes ?? [],
614
+ parentPayloadCommitments
552
615
  );
553
616
  }
554
617
 
555
618
  return {result: {responses: validatedResponses, payloadEnvelopes: validatedPayloadEnvelopes}, warnings};
556
619
  }
557
620
 
621
+ async function validateParentPayloadColumns(
622
+ parentPayloadCommitments: ParentPayloadCommitments,
623
+ columnSidecars: DataColumnSidecar[],
624
+ peerDasMetrics?: BeaconMetrics["peerDas"] | null
625
+ ): Promise<ValidatedColumnSidecars | null> {
626
+ const parentColumns: gloas.DataColumnSidecar[] = [];
627
+ for (const cs of columnSidecars) {
628
+ if (isGloasDataColumnSidecar(cs) && byteArrayEquals(cs.beaconBlockRoot, parentPayloadCommitments.blockRoot)) {
629
+ parentColumns.push(cs);
630
+ }
631
+ }
632
+
633
+ if (parentColumns.length === 0) {
634
+ return null;
635
+ }
636
+
637
+ parentColumns.sort((a, b) => a.index - b.index);
638
+ const parentSlot = parentColumns[0].slot;
639
+
640
+ await validateGloasBlockDataColumnSidecars(
641
+ parentSlot,
642
+ parentPayloadCommitments.blockRoot,
643
+ parentPayloadCommitments.kzgCommitments,
644
+ parentColumns,
645
+ peerDasMetrics
646
+ );
647
+
648
+ return {blockRoot: parentPayloadCommitments.blockRoot, columnSidecars: parentColumns};
649
+ }
650
+
558
651
  /**
559
652
  * Should not be called directly. Only exported for unit testing purposes
560
653
  *
@@ -1165,14 +1258,18 @@ export class DownloadByRangeError extends LodestarError<DownloadByRangeErrorType
1165
1258
 
1166
1259
  /**
1167
1260
  * Validates SignedExecutionPayloadEnvelopes received for a range request.
1168
- * For each envelope whose slot appears in the downloaded blocks, verifies that
1169
- * envelope.message.beaconBlockRoot matches the corresponding block's root.
1170
- * Envelopes for slots not in the batch (orphaned payloads) are silently ignored.
1261
+ *
1262
+ * Three categories of envelope slots:
1263
+ * - In-batch slot whose block we have: verify envelope.beaconBlockRoot matches.
1264
+ * - Dangling-parent envelope (only when `parentPayloadCommitments` is set, i.e. the first
1265
+ * batch of a `SyncChain`): keep if `envelope.beaconBlockRoot === parentPayloadCommitments.blockRoot`.
1266
+ * - Other "orphan" envelopes (e.g. unrelated slots): ignored.
1171
1267
  */
1172
1268
  export function validateEnvelopesByRangeResponse(
1173
1269
  validatedBlocks: ValidatedBlock[],
1174
1270
  batchBlocks: IBlockInput[] | undefined,
1175
- payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[]
1271
+ payloadEnvelopes: gloas.SignedExecutionPayloadEnvelope[],
1272
+ parentPayloadCommitments?: ParentPayloadCommitments
1176
1273
  ): Map<Slot, gloas.SignedExecutionPayloadEnvelope> {
1177
1274
  // Build a map of slot -> blockRoot for all blocks in the batch
1178
1275
  const batchBlockRoots = new Map<Slot, Uint8Array>();
@@ -1191,8 +1288,15 @@ export function validateEnvelopesByRangeResponse(
1191
1288
  const slot = payloadEnvelope.message.payload.slotNumber;
1192
1289
  const batchBlockRoot = batchBlockRoots.get(slot);
1193
1290
 
1194
- // Envelopes for slots not in the batch are silently ignored (orphaned payloads or a parent payload)
1195
1291
  if (batchBlockRoot === undefined) {
1292
+ // Keep the requested dangling-parent envelope only when its beaconBlockRoot matches
1293
+ // exactly. All other unrelated envelopes are dropped.
1294
+ if (
1295
+ parentPayloadCommitments !== undefined &&
1296
+ byteArrayEquals(payloadEnvelope.message.beaconBlockRoot, parentPayloadCommitments.blockRoot)
1297
+ ) {
1298
+ payloadEnvelopeMap.set(slot, payloadEnvelope);
1299
+ }
1196
1300
  continue;
1197
1301
  }
1198
1302