@aztec/archiver 0.0.1-commit.29c6b1a3 → 0.0.1-commit.2c0ee1788

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (134) hide show
  1. package/README.md +12 -6
  2. package/dest/archiver.d.ts +26 -15
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +169 -148
  5. package/dest/config.d.ts +5 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +16 -4
  8. package/dest/errors.d.ts +61 -10
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +88 -14
  11. package/dest/factory.d.ts +6 -7
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +41 -34
  14. package/dest/index.d.ts +11 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +10 -2
  17. package/dest/l1/bin/retrieve-calldata.js +36 -33
  18. package/dest/l1/calldata_retriever.d.ts +74 -50
  19. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  20. package/dest/l1/calldata_retriever.js +197 -260
  21. package/dest/l1/data_retrieval.d.ts +26 -17
  22. package/dest/l1/data_retrieval.d.ts.map +1 -1
  23. package/dest/l1/data_retrieval.js +43 -48
  24. package/dest/l1/spire_proposer.d.ts +5 -5
  25. package/dest/l1/spire_proposer.d.ts.map +1 -1
  26. package/dest/l1/spire_proposer.js +9 -17
  27. package/dest/l1/validate_historical_logs.d.ts +23 -0
  28. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  29. package/dest/l1/validate_historical_logs.js +108 -0
  30. package/dest/l1/validate_trace.d.ts +6 -3
  31. package/dest/l1/validate_trace.d.ts.map +1 -1
  32. package/dest/l1/validate_trace.js +13 -9
  33. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  34. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  35. package/dest/modules/contract_data_source_adapter.js +42 -0
  36. package/dest/modules/data_source_base.d.ts +30 -17
  37. package/dest/modules/data_source_base.d.ts.map +1 -1
  38. package/dest/modules/data_source_base.js +98 -125
  39. package/dest/modules/data_store_updater.d.ts +37 -17
  40. package/dest/modules/data_store_updater.d.ts.map +1 -1
  41. package/dest/modules/data_store_updater.js +155 -112
  42. package/dest/modules/instrumentation.d.ts +21 -3
  43. package/dest/modules/instrumentation.d.ts.map +1 -1
  44. package/dest/modules/instrumentation.js +58 -18
  45. package/dest/modules/l1_synchronizer.d.ts +13 -11
  46. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  47. package/dest/modules/l1_synchronizer.js +334 -181
  48. package/dest/modules/validation.d.ts +4 -3
  49. package/dest/modules/validation.d.ts.map +1 -1
  50. package/dest/modules/validation.js +6 -6
  51. package/dest/store/block_store.d.ts +108 -32
  52. package/dest/store/block_store.d.ts.map +1 -1
  53. package/dest/store/block_store.js +477 -141
  54. package/dest/store/contract_class_store.d.ts +17 -4
  55. package/dest/store/contract_class_store.d.ts.map +1 -1
  56. package/dest/store/contract_class_store.js +33 -73
  57. package/dest/store/contract_instance_store.d.ts +28 -1
  58. package/dest/store/contract_instance_store.d.ts.map +1 -1
  59. package/dest/store/contract_instance_store.js +37 -2
  60. package/dest/store/data_stores.d.ts +68 -0
  61. package/dest/store/data_stores.d.ts.map +1 -0
  62. package/dest/store/data_stores.js +50 -0
  63. package/dest/store/function_names_cache.d.ts +17 -0
  64. package/dest/store/function_names_cache.d.ts.map +1 -0
  65. package/dest/store/function_names_cache.js +30 -0
  66. package/dest/store/l2_tips_cache.d.ts +20 -0
  67. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  68. package/dest/store/l2_tips_cache.js +109 -0
  69. package/dest/store/log_store.d.ts +6 -3
  70. package/dest/store/log_store.d.ts.map +1 -1
  71. package/dest/store/log_store.js +150 -55
  72. package/dest/store/message_store.d.ts +5 -1
  73. package/dest/store/message_store.d.ts.map +1 -1
  74. package/dest/store/message_store.js +21 -9
  75. package/dest/test/fake_l1_state.d.ts +25 -1
  76. package/dest/test/fake_l1_state.d.ts.map +1 -1
  77. package/dest/test/fake_l1_state.js +166 -32
  78. package/dest/test/index.d.ts +1 -2
  79. package/dest/test/index.d.ts.map +1 -1
  80. package/dest/test/index.js +3 -2
  81. package/dest/test/mock_archiver.d.ts +1 -1
  82. package/dest/test/mock_archiver.d.ts.map +1 -1
  83. package/dest/test/mock_archiver.js +3 -2
  84. package/dest/test/mock_l1_to_l2_message_source.d.ts +1 -1
  85. package/dest/test/mock_l1_to_l2_message_source.d.ts.map +1 -1
  86. package/dest/test/mock_l1_to_l2_message_source.js +2 -1
  87. package/dest/test/mock_l2_block_source.d.ts +38 -8
  88. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  89. package/dest/test/mock_l2_block_source.js +183 -90
  90. package/dest/test/mock_structs.d.ts +4 -1
  91. package/dest/test/mock_structs.d.ts.map +1 -1
  92. package/dest/test/mock_structs.js +13 -1
  93. package/dest/test/noop_l1_archiver.d.ts +7 -4
  94. package/dest/test/noop_l1_archiver.d.ts.map +1 -1
  95. package/dest/test/noop_l1_archiver.js +14 -8
  96. package/package.json +14 -13
  97. package/src/archiver.ts +215 -170
  98. package/src/config.ts +23 -2
  99. package/src/errors.ts +133 -22
  100. package/src/factory.ts +54 -32
  101. package/src/index.ts +18 -2
  102. package/src/l1/README.md +25 -68
  103. package/src/l1/bin/retrieve-calldata.ts +46 -39
  104. package/src/l1/calldata_retriever.ts +261 -379
  105. package/src/l1/data_retrieval.ts +59 -70
  106. package/src/l1/spire_proposer.ts +7 -15
  107. package/src/l1/validate_historical_logs.ts +140 -0
  108. package/src/l1/validate_trace.ts +24 -6
  109. package/src/modules/contract_data_source_adapter.ts +59 -0
  110. package/src/modules/data_source_base.ts +145 -147
  111. package/src/modules/data_store_updater.ts +187 -141
  112. package/src/modules/instrumentation.ts +71 -19
  113. package/src/modules/l1_synchronizer.ts +419 -219
  114. package/src/modules/validation.ts +10 -9
  115. package/src/store/block_store.ts +589 -179
  116. package/src/store/contract_class_store.ts +39 -107
  117. package/src/store/contract_instance_store.ts +51 -5
  118. package/src/store/data_stores.ts +108 -0
  119. package/src/store/function_names_cache.ts +37 -0
  120. package/src/store/l2_tips_cache.ts +134 -0
  121. package/src/store/log_store.ts +222 -64
  122. package/src/store/message_store.ts +27 -10
  123. package/src/structs/inbox_message.ts +1 -1
  124. package/src/test/fake_l1_state.ts +213 -42
  125. package/src/test/index.ts +3 -1
  126. package/src/test/mock_archiver.ts +3 -2
  127. package/src/test/mock_l1_to_l2_message_source.ts +1 -0
  128. package/src/test/mock_l2_block_source.ts +235 -87
  129. package/src/test/mock_structs.ts +20 -6
  130. package/src/test/noop_l1_archiver.ts +16 -8
  131. package/dest/store/kv_archiver_store.d.ts +0 -340
  132. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  133. package/dest/store/kv_archiver_store.js +0 -447
  134. package/src/store/kv_archiver_store.ts +0 -639
@@ -371,20 +371,27 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
371
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
372
372
  }
373
373
  var _dec, _dec1, _dec2, _dec3, _initProto;
374
+ import { getFinalizedL1Block } from '@aztec/ethereum/queries';
375
+ import { asyncPool } from '@aztec/foundation/async-pool';
374
376
  import { maxBigint } from '@aztec/foundation/bigint';
375
377
  import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
376
- import { Buffer32 } from '@aztec/foundation/buffer';
377
- import { pick } from '@aztec/foundation/collection';
378
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
379
+ import { partition, pick } from '@aztec/foundation/collection';
380
+ import { EthAddress } from '@aztec/foundation/eth-address';
378
381
  import { createLogger } from '@aztec/foundation/log';
382
+ import { retryTimes } from '@aztec/foundation/retry';
379
383
  import { count } from '@aztec/foundation/string';
380
384
  import { Timer, elapsed } from '@aztec/foundation/timer';
381
- import { isDefined } from '@aztec/foundation/types';
385
+ import { isDefined, isErrorClass } from '@aztec/foundation/types';
382
386
  import { L2BlockSourceEvents } from '@aztec/stdlib/block';
383
- import { getEpochAtSlot, getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
387
+ import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
388
+ import { getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
384
389
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
385
390
  import { execInSpan, trackSpan } from '@aztec/telemetry-client';
386
391
  import { InitialCheckpointNumberNotSequentialError } from '../errors.js';
387
- import { retrieveCheckpointsFromRollup, retrieveL1ToL2Message, retrieveL1ToL2Messages, retrievedToPublishedCheckpoint } from '../l1/data_retrieval.js';
392
+ import { getCheckpointBlobDataFromBlobs, retrieveCheckpointCalldataFromRollup, retrieveL1ToL2Message, retrieveL1ToL2Messages, retrievedToPublishedCheckpoint } from '../l1/data_retrieval.js';
393
+ import { getArchiverSynchPoint } from '../store/data_stores.js';
394
+ import { MessageStoreError } from '../store/message_store.js';
388
395
  import { ArchiverDataStoreUpdater } from './data_store_updater.js';
389
396
  import { validateCheckpointAttestations } from './validation.js';
390
397
  _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpochPrune'), _dec2 = trackSpan('Archiver.handleL1ToL2Messages'), _dec3 = trackSpan('Archiver.handleCheckpoints');
@@ -396,8 +403,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
396
403
  debugClient;
397
404
  rollup;
398
405
  inbox;
399
- l1Addresses;
400
- store;
406
+ stores;
401
407
  config;
402
408
  blobClient;
403
409
  epochCache;
@@ -435,13 +441,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
435
441
  l1Timestamp;
436
442
  updater;
437
443
  tracer;
438
- constructor(publicClient, debugClient, rollup, inbox, l1Addresses, store, config, blobClient, epochCache, dateProvider, instrumentation, l1Constants, events, tracer, log = createLogger('archiver:l1-sync')){
444
+ constructor(publicClient, debugClient, rollup, inbox, stores, config, blobClient, epochCache, dateProvider, instrumentation, l1Constants, events, tracer, l2TipsCache, log = createLogger('archiver:l1-sync')){
439
445
  this.publicClient = publicClient;
440
446
  this.debugClient = debugClient;
441
447
  this.rollup = rollup;
442
448
  this.inbox = inbox;
443
- this.l1Addresses = l1Addresses;
444
- this.store = store;
449
+ this.stores = stores;
445
450
  this.config = config;
446
451
  this.blobClient = blobClient;
447
452
  this.epochCache = epochCache;
@@ -451,7 +456,9 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
451
456
  this.events = events;
452
457
  this.log = log;
453
458
  _initProto(this);
454
- this.updater = new ArchiverDataStoreUpdater(this.store);
459
+ this.updater = new ArchiverDataStoreUpdater(this.stores, l2TipsCache, {
460
+ rollupManaLimit: l1Constants.rollupManaLimit
461
+ });
455
462
  this.tracer = tracer;
456
463
  }
457
464
  /** Sets new config */ setConfig(newConfig) {
@@ -463,6 +470,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
463
470
  /** Returns the last L1 timestamp that was synced. */ getL1Timestamp() {
464
471
  return this.l1Timestamp;
465
472
  }
473
+ getSignatureContext() {
474
+ return {
475
+ chainId: this.publicClient.chain.id,
476
+ rollupAddress: EthAddress.fromString(this.rollup.address)
477
+ };
478
+ }
466
479
  /** Checks that the ethereum node we are connected to has a latest timestamp no more than the allowed drift. Throw if not. */ async testEthereumNodeSynced() {
467
480
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
468
481
  if (maxAllowedDelay === 0) {
@@ -477,12 +490,20 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
477
490
  }
478
491
  }
479
492
  async syncFromL1(initialSyncComplete) {
493
+ // In between the various calls to L1, the block number can move meaning some of the following
494
+ // calls will return data for blocks that were not present during earlier calls. To combat this
495
+ // we ensure that all data retrieval methods only retrieve data up to the currentBlockNumber
496
+ // captured at the top of this function.
480
497
  const currentL1Block = await this.publicClient.getBlock({
481
498
  includeTransactions: false
482
499
  });
483
500
  const currentL1BlockNumber = currentL1Block.number;
484
501
  const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
485
502
  const currentL1Timestamp = currentL1Block.timestamp;
503
+ const currentL1BlockData = {
504
+ l1BlockNumber: currentL1BlockNumber,
505
+ l1BlockHash: currentL1BlockHash
506
+ };
486
507
  if (this.l1BlockHash && currentL1BlockHash.equals(this.l1BlockHash)) {
487
508
  this.log.trace(`No new L1 blocks since last sync at L1 block ${this.l1BlockNumber}`);
488
509
  return;
@@ -498,36 +519,16 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
498
519
  maxAllowedDelay
499
520
  });
500
521
  }
501
- // Load sync point for blocks and messages defaulting to start block
502
- const { blocksSynchedTo = this.l1Constants.l1StartBlock, messagesSynchedTo = {
503
- l1BlockNumber: this.l1Constants.l1StartBlock,
504
- l1BlockHash: this.l1Constants.l1StartBlockHash
505
- } } = await this.store.getSynchPoint();
522
+ // Load sync point for blocks defaulting to start block
523
+ const { blocksSynchedTo = this.l1Constants.l1StartBlock } = await getArchiverSynchPoint(this.stores);
506
524
  this.log.debug(`Starting new archiver sync iteration`, {
507
525
  blocksSynchedTo,
508
- messagesSynchedTo,
509
- currentL1BlockNumber,
510
- currentL1BlockHash
526
+ currentL1BlockData
511
527
  });
512
- // ********** Ensuring Consistency of data pulled from L1 **********
513
- /**
514
- * There are a number of calls in this sync operation to L1 for retrieving
515
- * events and transaction data. There are a couple of things we need to bear in mind
516
- * to ensure that data is read exactly once.
517
- *
518
- * The first is the problem of eventually consistent ETH service providers like Infura.
519
- * Each L1 read operation will query data from the last L1 block that it saw emit its kind of data.
520
- * (so pending L1 to L2 messages will read from the last L1 block that emitted a message and so on)
521
- * This will mean the archiver will lag behind L1 and will only advance when there's L2-relevant activity on the chain.
522
- *
523
- * The second is that in between the various calls to L1, the block number can move meaning some
524
- * of the following calls will return data for blocks that were not present during earlier calls.
525
- * To combat this for the time being we simply ensure that all data retrieval methods only retrieve
526
- * data up to the currentBlockNumber captured at the top of this function. We might want to improve on this
527
- * in future but for the time being it should give us the guarantees that we need
528
- */ // ********** Events that are processed per L1 block **********
529
- await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber);
530
- // ********** Events that are processed per checkpoint **********
528
+ // Sync L1 to L2 messages. We retry this a few times since there are error conditions that reset the sync point, requiring a new iteration.
529
+ // Note that we cannot just wait for the l1 synchronizer to loop again, since the synchronizer would report as synced up to the current L1
530
+ // block, when that wouldn't be the case, since L1 to L2 messages would need another iteration.
531
+ await retryTimes(()=>this.handleL1ToL2Messages(currentL1BlockData), 'Handling L1 to L2 messages', 3, 0.1);
531
532
  if (currentL1BlockNumber > blocksSynchedTo) {
532
533
  // First we retrieve new checkpoints and L2 blocks and store them in the DB. This will also update the
533
534
  // pending chain validation status, proven checkpoint number, and synched L1 block number.
@@ -544,7 +545,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
544
545
  // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until
545
546
  // we get a valid checkpoint to advance the syncpoint.
546
547
  if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) {
547
- await this.store.setCheckpointSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
548
+ await this.stores.blocks.setSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
548
549
  }
549
550
  // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg.
550
551
  // We only do this if rollup cant prune on the next submission. Otherwise we will end up
@@ -555,6 +556,8 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
555
556
  }
556
557
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
557
558
  }
559
+ // Update the finalized L2 checkpoint based on L1 finality.
560
+ await this.updateFinalizedCheckpoint();
558
561
  // After syncing has completed, update the current l1 block number and timestamp,
559
562
  // otherwise we risk announcing to the world that we've synced to a given point,
560
563
  // but the corresponding blocks have not been processed (see #12631).
@@ -568,37 +571,64 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
568
571
  l1BlockNumberAtEnd
569
572
  });
570
573
  }
574
+ /** Query L1 for its finalized block and update the finalized checkpoint accordingly. */ async updateFinalizedCheckpoint() {
575
+ try {
576
+ const finalizedL1Block = await getFinalizedL1Block(this.publicClient);
577
+ if (!finalizedL1Block) {
578
+ this.log.trace(`Skipping finalized checkpoint update: L1 has no finalized block yet.`);
579
+ return;
580
+ }
581
+ const finalizedL1BlockNumber = finalizedL1Block.number;
582
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
583
+ blockNumber: finalizedL1BlockNumber
584
+ });
585
+ const localFinalizedCheckpointNumber = await this.stores.blocks.getFinalizedCheckpointNumber();
586
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
587
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
588
+ this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
589
+ finalizedCheckpointNumber,
590
+ finalizedL1BlockNumber
591
+ });
592
+ }
593
+ } catch (err) {
594
+ // The rollup contract may not exist at the finalized L1 block right after deployment.
595
+ if (!err?.message?.includes('returned no data')) {
596
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
597
+ }
598
+ }
599
+ }
571
600
  /** Prune all proposed local blocks that should have been checkpointed by now. */ async pruneUncheckpointedBlocks(currentL1Timestamp) {
572
601
  const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
573
- this.store.getCheckpointedL2BlockNumber(),
574
- this.store.getLatestBlockNumber()
602
+ this.stores.blocks.getCheckpointedL2BlockNumber(),
603
+ this.stores.blocks.getLatestL2BlockNumber()
575
604
  ]);
576
605
  // If there are no uncheckpointed blocks, we got nothing to do
577
606
  if (lastProposedBlockNumber === lastCheckpointedBlockNumber) {
578
607
  this.log.trace(`No uncheckpointed blocks to prune.`);
579
608
  return;
580
609
  }
581
- // What's the slot of the first uncheckpointed block?
610
+ // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
611
+ const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
582
612
  const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
583
- const [firstUncheckpointedBlockHeader] = await this.store.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
613
+ // What's the slot of the first uncheckpointed block?
614
+ const [firstUncheckpointedBlockHeader] = await this.stores.blocks.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
584
615
  const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
585
- // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
586
- const nextL1BlockTimestamp = currentL1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration);
587
- const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
588
- // Prune provisional blocks from slots that have ended without being checkpointed
589
- if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
590
- this.log.warn(`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`, {
591
- firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(),
592
- slotAtNextL1Block
616
+ if (firstUncheckpointedBlockSlot === undefined || firstUncheckpointedBlockSlot >= slotAtNextL1Block) {
617
+ return;
618
+ }
619
+ // Prune provisional blocks from slots that have ended without being checkpointed.
620
+ // This also clears any proposed checkpoint whose blocks are being pruned.
621
+ this.log.warn(`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`, {
622
+ firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(),
623
+ slotAtNextL1Block
624
+ });
625
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
626
+ if (prunedBlocks.length > 0) {
627
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
628
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
629
+ slotNumber: firstUncheckpointedBlockSlot,
630
+ blocks: prunedBlocks
593
631
  });
594
- const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
595
- if (prunedBlocks.length > 0) {
596
- this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
597
- type: L2BlockSourceEvents.L2PruneUncheckpointed,
598
- slotNumber: firstUncheckpointedBlockSlot,
599
- blocks: prunedBlocks
600
- });
601
- }
602
632
  }
603
633
  }
604
634
  /** Queries the rollup contract on whether a prune can be executed on the immediate next L1 block. */ async canPrune(currentL1BlockNumber, currentL1Timestamp) {
@@ -617,7 +647,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
617
647
  }
618
648
  /** Checks if there'd be a reorg for the next checkpoint submission and start pruning now. */ async handleEpochPrune(provenCheckpointNumber, currentL1BlockNumber, currentL1Timestamp) {
619
649
  const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
620
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
650
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
621
651
  const canPrune = localPendingCheckpointNumber > provenCheckpointNumber && rollupCanPrune;
622
652
  if (canPrune) {
623
653
  const timer = new Timer();
@@ -629,12 +659,14 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
629
659
  const pruneFromSlotNumber = header.slotNumber;
630
660
  const pruneFromEpochNumber = getEpochAtSlot(pruneFromSlotNumber, this.l1Constants);
631
661
  const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
632
- const checkpointPromises = Array.from({
662
+ // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
663
+ // promises when the gap between local pending and proven checkpoint numbers is large.
664
+ const BATCH_SIZE = 10;
665
+ const indices = Array.from({
633
666
  length: checkpointsToUnwind
634
- }).fill(0).map((_, i)=>this.store.getCheckpointData(CheckpointNumber(i + pruneFrom)));
635
- const checkpoints = await Promise.all(checkpointPromises);
636
- const blockPromises = await Promise.all(checkpoints.filter(isDefined).map((cp)=>this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber))));
637
- const newBlocks = blockPromises.filter(isDefined).flat();
667
+ }, (_, i)=>CheckpointNumber(i + pruneFrom));
668
+ const checkpoints = (await asyncPool(BATCH_SIZE, indices, (idx)=>this.stores.blocks.getCheckpointData(idx))).filter(isDefined);
669
+ const newBlocks = (await asyncPool(BATCH_SIZE, checkpoints, (cp)=>this.stores.blocks.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)))).filter(isDefined).flat();
638
670
  // Emit an event for listening services to react to the chain prune
639
671
  this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
640
672
  type: L2BlockSourceEvents.L2PruneUnproven,
@@ -643,11 +675,11 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
643
675
  });
644
676
  this.log.debug(`L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`);
645
677
  await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
646
- this.log.warn(`Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` + `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` + `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`);
678
+ this.log.warn(`Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` + `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` + `Updated latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`);
647
679
  this.instrumentation.processPrune(timer.ms());
648
680
  // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
649
681
  // Seems like the next iteration should handle this.
650
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
682
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
651
683
  }
652
684
  return {
653
685
  rollupCanPrune
@@ -668,59 +700,79 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
668
700
  nextEnd
669
701
  ];
670
702
  }
671
- async handleL1ToL2Messages(messagesSyncPoint, currentL1BlockNumber) {
672
- this.log.trace(`Handling L1 to L2 messages from ${messagesSyncPoint.l1BlockNumber} to ${currentL1BlockNumber}.`);
673
- if (currentL1BlockNumber <= messagesSyncPoint.l1BlockNumber) {
674
- return;
703
+ async handleL1ToL2Messages(currentL1Block) {
704
+ // Load the syncpoint, which may have been updated in a previous iteration
705
+ const { messagesSynchedTo = {
706
+ l1BlockNumber: this.l1Constants.l1StartBlock,
707
+ l1BlockHash: this.l1Constants.l1StartBlockHash
708
+ } } = await getArchiverSynchPoint(this.stores);
709
+ // Nothing to do if L1 block number has not moved forward
710
+ const currentL1BlockNumber = currentL1Block.l1BlockNumber;
711
+ if (currentL1BlockNumber <= messagesSynchedTo.l1BlockNumber) {
712
+ return true;
675
713
  }
676
- // Load remote and local inbox states.
677
- const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
678
- const localLastMessage = await this.store.getLastL1ToL2Message();
714
+ // Compare local message store state with the remote. If they match, we just advance the match pointer.
679
715
  const remoteMessagesState = await this.inbox.getState({
680
716
  blockNumber: currentL1BlockNumber
681
717
  });
682
- this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
683
- localMessagesInserted,
684
- localLastMessage,
685
- remoteMessagesState
686
- });
687
- // Compare message count and rolling hash. If they match, no need to retrieve anything.
688
- if (remoteMessagesState.totalMessagesInserted === localMessagesInserted && remoteMessagesState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer32.ZERO)) {
689
- this.log.trace(`No L1 to L2 messages to query between L1 blocks ${messagesSyncPoint.l1BlockNumber} and ${currentL1BlockNumber}.`);
690
- return;
718
+ const localLastMessage = await this.stores.messages.getLastMessage();
719
+ if (await this.localStateMatches(localLastMessage, remoteMessagesState)) {
720
+ this.log.trace(`Local L1 to L2 messages are already in sync with remote at L1 block ${currentL1BlockNumber}`);
721
+ await this.stores.messages.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress);
722
+ return true;
691
723
  }
692
- // Check if our syncpoint is still valid. If not, there was an L1 reorg and we need to re-retrieve messages.
693
- // Note that we need to fetch it from logs and not from inbox state at the syncpoint l1 block number, since it
694
- // could be older than 128 blocks and non-archive nodes cannot resolve it.
695
- if (localLastMessage) {
696
- const remoteLastMessage = await this.retrieveL1ToL2Message(localLastMessage.leaf);
697
- this.log.trace(`Retrieved remote message for local last`, {
698
- remoteLastMessage,
699
- localLastMessage
700
- });
701
- if (!remoteLastMessage || !remoteLastMessage.rollingHash.equals(localLastMessage.rollingHash)) {
702
- this.log.warn(`Rolling back L1 to L2 messages due to hash mismatch or msg not found.`, {
703
- remoteLastMessage,
704
- messagesSyncPoint,
705
- localLastMessage
706
- });
707
- messagesSyncPoint = await this.rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint);
708
- this.log.debug(`Rolled back L1 to L2 messages to L1 block ${messagesSyncPoint.l1BlockNumber}.`, {
709
- messagesSyncPoint
724
+ // If not, then we are out of sync. Most likely there are new messages on the inbox, so we try retrieving them.
725
+ // However, it could also be the case that there was an L1 reorg and our syncpoint is no longer valid.
726
+ // If that's the case, we'd get an exception out of the message store since the rolling hash of the first message
727
+ // we try to insert would not match the one in the db, in which case we rollback to the last common message with L1.
728
+ try {
729
+ await this.retrieveAndStoreMessages(messagesSynchedTo.l1BlockNumber, currentL1BlockNumber);
730
+ } catch (error) {
731
+ if (isErrorClass(error, MessageStoreError)) {
732
+ this.log.warn(`Failed to store L1 to L2 messages retrieved from L1: ${error.message}. Rolling back syncpoint to retry.`, {
733
+ inboxMessage: error.inboxMessage
710
734
  });
735
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
736
+ return false;
711
737
  }
738
+ throw error;
712
739
  }
713
- // Retrieve and save messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
740
+ // Note that, if there are no new messages to insert, but there was an L1 reorg that pruned out last messages,
741
+ // we'd notice by comparing our local state with the remote one again, and seeing they don't match even after
742
+ // our sync attempt. In this case, we also rollback our syncpoint, and trigger a retry.
743
+ const localLastMessageAfterSync = await this.stores.messages.getLastMessage();
744
+ if (!await this.localStateMatches(localLastMessageAfterSync, remoteMessagesState)) {
745
+ this.log.warn(`Local L1 to L2 messages state does not match remote after sync attempt. Rolling back syncpoint to retry.`, {
746
+ localLastMessageAfterSync,
747
+ remoteMessagesState
748
+ });
749
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
750
+ return false;
751
+ }
752
+ // Advance the syncpoint after a successful sync
753
+ await this.stores.messages.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress);
754
+ return true;
755
+ }
756
+ /** Checks if the local rolling hash and message count matches the remote state */ async localStateMatches(localLastMessage, remoteState) {
757
+ const localMessageCount = await this.stores.messages.getTotalL1ToL2MessageCount();
758
+ this.log.trace(`Comparing local and remote inbox state`, {
759
+ localMessageCount,
760
+ localLastMessage,
761
+ remoteState
762
+ });
763
+ return remoteState.totalMessagesInserted === localMessageCount && remoteState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO);
764
+ }
765
+ /** Retrieves L1 to L2 messages from L1 in batches and stores them. */ async retrieveAndStoreMessages(fromL1Block, toL1Block) {
714
766
  let searchStartBlock = 0n;
715
- let searchEndBlock = messagesSyncPoint.l1BlockNumber;
767
+ let searchEndBlock = fromL1Block;
716
768
  let lastMessage;
717
769
  let messageCount = 0;
718
770
  do {
719
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
771
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, toL1Block);
720
772
  this.log.trace(`Retrieving L1 to L2 messages in L1 blocks ${searchStartBlock}-${searchEndBlock}`);
721
773
  const messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
722
774
  const timer = new Timer();
723
- await this.store.addL1ToL2Messages(messages);
775
+ await this.stores.messages.addL1ToL2Messages(messages);
724
776
  const perMsg = timer.ms() / messages.length;
725
777
  this.instrumentation.processNewMessages(messages.length, perMsg);
726
778
  for (const msg of messages){
@@ -731,74 +783,81 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
731
783
  lastMessage = msg;
732
784
  messageCount++;
733
785
  }
734
- }while (searchEndBlock < currentL1BlockNumber)
735
- // Log stats for messages retrieved (if any).
786
+ }while (searchEndBlock < toL1Block)
736
787
  if (messageCount > 0) {
737
788
  this.log.info(`Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for checkpoint ${lastMessage?.checkpointNumber}`, {
738
789
  lastMessage,
739
790
  messageCount
740
791
  });
741
792
  }
742
- // Warn if the resulting rolling hash does not match the remote state we had retrieved.
743
- if (lastMessage && !lastMessage.rollingHash.equals(remoteMessagesState.messagesRollingHash)) {
744
- this.log.warn(`Last message retrieved rolling hash does not match remote state.`, {
745
- lastMessage,
746
- remoteMessagesState
747
- });
748
- }
749
793
  }
750
- async retrieveL1ToL2Message(leaf) {
751
- const currentL1BlockNumber = await this.publicClient.getBlockNumber();
752
- let searchStartBlock = 0n;
753
- let searchEndBlock = this.l1Constants.l1StartBlock - 1n;
754
- do {
755
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
756
- const message = await retrieveL1ToL2Message(this.inbox, leaf, searchStartBlock, searchEndBlock);
757
- if (message) {
758
- return message;
759
- }
760
- }while (searchEndBlock < currentL1BlockNumber)
761
- return undefined;
762
- }
763
- async rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint) {
764
- // Slowly go back through our messages until we find the last common message.
765
- // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
766
- // is a very rare case, so it's fine to query one log at a time.
794
+ /**
795
+ * Rolls back local L1 to L2 messages to the last common message with L1, and updates the syncpoint to the L1 block of that message.
796
+ * If no common message is found, rolls back all messages and sets the syncpoint to the start block.
797
+ */ async rollbackL1ToL2Messages(remoteMessagesState) {
798
+ const { treeInProgress: remoteTreeInProgress, messagesRollingHash: remoteRollingHash } = remoteMessagesState;
799
+ // Slowly go back through our messages until we find the last common message. We could query the logs in
800
+ // batch as an optimization, but the depth of the reorg should not be deep, and this is a very rare case,
801
+ // so it's fine to query one log at a time.
767
802
  let commonMsg;
768
- this.log.verbose(`Searching most recent common L1 to L2 message at or before index ${localLastMessage.index}`);
769
- for await (const msg of this.store.iterateL1ToL2Messages({
770
- reverse: true,
771
- end: localLastMessage.index
803
+ let messagesToDelete = 0;
804
+ this.log.verbose(`Searching most recent common L1 to L2 message`);
805
+ for await (const localMsg of this.stores.messages.iterateL1ToL2Messages({
806
+ reverse: true
772
807
  })){
773
- const remoteMsg = await this.retrieveL1ToL2Message(msg.leaf);
774
808
  const logCtx = {
775
- remoteMsg,
776
- localMsg: msg
809
+ remoteMsg: undefined,
810
+ localMsg,
811
+ remoteMessagesState
777
812
  };
778
- if (remoteMsg && remoteMsg.rollingHash.equals(msg.rollingHash)) {
779
- this.log.verbose(`Found most recent common L1 to L2 message at index ${msg.index} on L1 block ${msg.l1BlockNumber}`, logCtx);
813
+ // First check if the local message rolling hash matches the current rolling hash of the inbox contract,
814
+ // which means we just need to rollback some local messages and we should be back in sync. This means there
815
+ // was an L1 reorg that removed some of the messages we had, but no new messages were added compared.
816
+ if (localMsg.rollingHash.equals(remoteRollingHash)) {
817
+ this.log.info(`Found common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber} matching current remote state`, logCtx);
818
+ commonMsg = localMsg;
819
+ break;
820
+ }
821
+ // If there's no match with the current remote state, check if the message exists on the inbox contract at all
822
+ // by looking at the inbox events. If the L1 reorg *added* new messages in addition to deleting existing ones,
823
+ // then the current remote state's rolling hash will not match anything we have locally, so we need to check existence
824
+ // of individual messages via logs. Note we use logs and not historical queries so we don't have to depend on
825
+ // an archival rpc node, since the message could be from a long time ago if we're catching up with syncing.
826
+ const remoteMsg = await retrieveL1ToL2Message(this.inbox, localMsg);
827
+ logCtx.remoteMsg = remoteMsg;
828
+ if (remoteMsg && remoteMsg.rollingHash.equals(localMsg.rollingHash)) {
829
+ this.log.info(`Found most recent common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber}`, logCtx);
780
830
  commonMsg = remoteMsg;
781
831
  break;
782
832
  } else if (remoteMsg) {
783
- this.log.debug(`Local L1 to L2 message with index ${msg.index} has different rolling hash`, logCtx);
833
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} has different rolling hash`, logCtx);
834
+ messagesToDelete++;
784
835
  } else {
785
- this.log.debug(`Local L1 to L2 message with index ${msg.index} not found on L1`, logCtx);
836
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} not found on L1`, logCtx);
837
+ messagesToDelete++;
786
838
  }
787
839
  }
788
- // Delete everything after the common message we found.
789
- const lastGoodIndex = commonMsg?.index;
790
- this.log.warn(`Deleting all local L1 to L2 messages after index ${lastGoodIndex ?? 'undefined'}`);
791
- await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
840
+ // Delete everything after the common message we found, if anything needs to be deleted.
841
+ // Do not exit early if there are no messages to delete, we still want to update the syncpoint.
842
+ if (messagesToDelete > 0) {
843
+ const lastGoodIndex = commonMsg?.index;
844
+ this.log.warn(`Rolling back all local L1 to L2 messages after index ${lastGoodIndex ?? 'initial'}`);
845
+ await this.stores.messages.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
846
+ }
792
847
  // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
793
848
  // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
794
849
  // after the last common message.
795
850
  const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1Constants.l1StartBlock;
796
851
  const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
797
- messagesSyncPoint = {
852
+ const messagesSyncPoint = {
798
853
  l1BlockNumber: syncPointL1BlockNumber,
799
854
  l1BlockHash: syncPointL1BlockHash
800
855
  };
801
- await this.store.setMessageSynchedL1Block(messagesSyncPoint);
856
+ await this.stores.messages.setMessageSyncState(messagesSyncPoint, remoteTreeInProgress);
857
+ this.log.verbose(`Updated messages syncpoint to L1 block ${syncPointL1BlockNumber}`, {
858
+ ...messagesSyncPoint,
859
+ remoteTreeInProgress
860
+ });
802
861
  return messagesSyncPoint;
803
862
  }
804
863
  async getL1BlockHash(l1BlockNumber) {
@@ -812,8 +871,8 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
812
871
  return Buffer32.fromString(block.hash);
813
872
  }
814
873
  async handleCheckpoints(blocksSynchedTo, currentL1BlockNumber, initialSyncComplete) {
815
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
816
- const initialValidationResult = await this.store.getPendingChainValidationStatus();
874
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
875
+ const initialValidationResult = await this.stores.blocks.getPendingChainValidationStatus();
817
876
  const { provenCheckpointNumber, provenArchive, pendingCheckpointNumber, pendingArchive, archiveOfMyCheckpoint: archiveForLocalPendingCheckpointNumber } = await execInSpan(this.tracer, 'Archiver.getRollupStatus', ()=>this.rollup.status(localPendingCheckpointNumber, {
818
877
  blockNumber: currentL1BlockNumber
819
878
  }));
@@ -836,39 +895,39 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
836
895
  // we need to set it to zero. This is an edge case because we dont have a checkpoint zero (initial checkpoint is one),
837
896
  // so localCheckpointForDestinationProvenCheckpointNumber would not be found below.
838
897
  if (provenCheckpointNumber === 0) {
839
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
898
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
840
899
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
841
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
900
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
842
901
  this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, {
843
902
  provenCheckpointNumber
844
903
  });
845
904
  }
846
905
  }
847
- const localCheckpointForDestinationProvenCheckpointNumber = await this.store.getCheckpointData(provenCheckpointNumber);
906
+ const localCheckpointForDestinationProvenCheckpointNumber = await this.stores.blocks.getCheckpointData(provenCheckpointNumber);
848
907
  // Sanity check. I've hit what seems to be a state where the proven checkpoint is set to a value greater than the latest
849
908
  // synched checkpoint when requesting L2Tips from the archiver. This is the only place where the proven checkpoint is set.
850
- const synched = await this.store.getSynchedCheckpointNumber();
909
+ const synched = await this.stores.blocks.getLatestCheckpointNumber();
851
910
  if (localCheckpointForDestinationProvenCheckpointNumber && synched < localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber) {
852
911
  this.log.error(`Hit local checkpoint greater than last synched checkpoint: ${localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber} > ${synched}`);
853
912
  }
854
913
  this.log.trace(`Local checkpoint for remote proven checkpoint ${provenCheckpointNumber} is ${localCheckpointForDestinationProvenCheckpointNumber?.archive.root.toString() ?? 'undefined'}`);
855
914
  if (localCheckpointForDestinationProvenCheckpointNumber && provenArchive.equals(localCheckpointForDestinationProvenCheckpointNumber.archive.root)) {
856
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
915
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
857
916
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
858
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
917
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
859
918
  this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, {
860
919
  provenCheckpointNumber
861
920
  });
862
921
  const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
863
922
  const provenEpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
864
- const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + localCheckpointForDestinationProvenCheckpointNumber.numBlocks - 1;
923
+ const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + localCheckpointForDestinationProvenCheckpointNumber.blockCount - 1;
865
924
  this.events.emit(L2BlockSourceEvents.L2BlockProven, {
866
925
  type: L2BlockSourceEvents.L2BlockProven,
867
926
  blockNumber: BlockNumber(lastBlockNumberInCheckpoint),
868
927
  slotNumber: provenSlotNumber,
869
928
  epochNumber: provenEpochNumber
870
929
  });
871
- this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
930
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
872
931
  } else {
873
932
  this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
874
933
  }
@@ -878,7 +937,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
878
937
  // If we have 0 checkpoints locally and there are no checkpoints onchain there is nothing to do.
879
938
  const noCheckpoints = localPendingCheckpointNumber === 0 && pendingCheckpointNumber === 0;
880
939
  if (noCheckpoints) {
881
- await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
940
+ await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
882
941
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no checkpoints on chain`);
883
942
  return rollupStatus;
884
943
  }
@@ -886,7 +945,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
886
945
  // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
887
946
  // are any state that could be impacted by it. If we have no checkpoints, there is no impact.
888
947
  if (localPendingCheckpointNumber > 0) {
889
- const localPendingCheckpoint = await this.store.getCheckpointData(localPendingCheckpointNumber);
948
+ const localPendingCheckpoint = await this.stores.blocks.getCheckpointData(localPendingCheckpointNumber);
890
949
  if (localPendingCheckpoint === undefined) {
891
950
  throw new Error(`Missing checkpoint ${localPendingCheckpointNumber}`);
892
951
  }
@@ -900,7 +959,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
900
959
  // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing checkpoints.
901
960
  // We must only set this block number based on actually retrieved logs.
902
961
  // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
903
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
962
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
904
963
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
905
964
  return rollupStatus;
906
965
  }
@@ -917,7 +976,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
917
976
  });
918
977
  let tipAfterUnwind = localPendingCheckpointNumber;
919
978
  while(true){
920
- const candidateCheckpoint = await this.store.getCheckpointData(tipAfterUnwind);
979
+ const candidateCheckpoint = await this.stores.blocks.getCheckpointData(tipAfterUnwind);
921
980
  if (candidateCheckpoint === undefined) {
922
981
  break;
923
982
  }
@@ -933,7 +992,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
933
992
  }
934
993
  const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
935
994
  await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
936
- this.log.warn(`Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` + `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`);
995
+ this.log.warn(`Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` + `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`);
937
996
  }
938
997
  }
939
998
  // Retrieve checkpoints in batches. Each batch is estimated to accommodate up to 'blockBatchSize' L1 blocks,
@@ -945,25 +1004,43 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
945
1004
  do {
946
1005
  [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
947
1006
  this.log.trace(`Retrieving checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
948
- // TODO(md): Retrieve from blob client then from consensus client, then from peers
949
- const retrievedCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointsFromRollup', ()=>retrieveCheckpointsFromRollup(this.rollup, this.publicClient, this.debugClient, this.blobClient, searchStartBlock, searchEndBlock, this.l1Addresses, this.instrumentation, this.log, !initialSyncComplete));
950
- if (retrievedCheckpoints.length === 0) {
1007
+ // First fetch calldata only, no blobs yet, since we may be able to just get that data out of the proposed chain
1008
+ const calldataCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointCalldataFromRollup', ()=>retrieveCheckpointCalldataFromRollup(this.rollup, this.publicClient, this.debugClient, searchStartBlock, searchEndBlock, this.instrumentation, this.log));
1009
+ if (calldataCheckpoints.length === 0) {
951
1010
  // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
952
1011
  // See further details in earlier comments.
953
1012
  this.log.trace(`Retrieved no new checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
954
1013
  continue;
955
1014
  }
956
- this.log.debug(`Retrieved ${retrievedCheckpoints.length} new checkpoints between L1 blocks ${searchStartBlock} and ${searchEndBlock}`, {
957
- lastProcessedCheckpoint: retrievedCheckpoints[retrievedCheckpoints.length - 1].l1,
1015
+ this.log.debug(`Retrieved ${calldataCheckpoints.length} new checkpoint calldata between L1 blocks ${searchStartBlock} and ${searchEndBlock}`, {
1016
+ lastProcessedCheckpoint: calldataCheckpoints[calldataCheckpoints.length - 1].l1,
958
1017
  searchStartBlock,
959
1018
  searchEndBlock
960
1019
  });
961
- const publishedCheckpoints = await Promise.all(retrievedCheckpoints.map((b)=>retrievedToPublishedCheckpoint(b)));
1020
+ // Check if the last checkpoint matches a local pending entry (so we can skip blob fetch).
1021
+ // We only check the last one; if it matches, the blob fetch is skipped for that entry.
1022
+ // TODO(palla/pipelining): We may have more than a single checkpoint to promote
1023
+ const lastCalldataCheckpoint = calldataCheckpoints[calldataCheckpoints.length - 1];
1024
+ const promoteResult = await this.tryBuildPublishedCheckpointFromProposed(lastCalldataCheckpoint);
1025
+ const checkpointToPromote = promoteResult && !('diverged' in promoteResult) ? promoteResult : undefined;
1026
+ const evictProposedFrom = promoteResult && 'diverged' in promoteResult ? promoteResult.fromCheckpointNumber : undefined;
1027
+ // Then fetch blobs in parallel and build the full published checkpoints
1028
+ const toFetchBlobs = checkpointToPromote ? calldataCheckpoints.slice(0, -1) : calldataCheckpoints;
1029
+ const blobFetched = await asyncPool(10, toFetchBlobs, async (checkpoint)=>retrievedToPublishedCheckpoint({
1030
+ ...checkpoint,
1031
+ checkpointBlobData: await getCheckpointBlobDataFromBlobs(this.blobClient, checkpoint.l1.blockHash, checkpoint.blobHashes, checkpoint.checkpointNumber, this.log, !initialSyncComplete, checkpoint.parentBeaconBlockRoot, checkpoint.l1.timestamp)
1032
+ }));
1033
+ // And add the promoted checkpoint to the list of all checkpoints
1034
+ const publishedCheckpoints = checkpointToPromote ? [
1035
+ ...blobFetched,
1036
+ checkpointToPromote
1037
+ ] : blobFetched;
962
1038
  const validCheckpoints = [];
1039
+ // Now loop through all checkpoints and validate their attestations
963
1040
  for (const published of publishedCheckpoints){
964
1041
  const validationResult = this.config.skipValidateCheckpointAttestations ? {
965
1042
  valid: true
966
- } : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.log);
1043
+ } : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.getSignatureContext(), this.log);
967
1044
  // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint
968
1045
  // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one.
969
1046
  // There is an exception though: if a checkpoint is invalidated and replaced with another invalid checkpoint,
@@ -988,7 +1065,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
988
1065
  // Check the inHash of the checkpoint against the l1->l2 messages.
989
1066
  // The messages should've been synced up to the currentL1BlockNumber and must be available for the published
990
1067
  // checkpoints we just retrieved.
991
- const l1ToL2Messages = await this.store.getL1ToL2Messages(published.checkpoint.number);
1068
+ const l1ToL2Messages = await this.stores.messages.getL1ToL2Messages(published.checkpoint.number);
992
1069
  const computedInHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
993
1070
  const publishedInHash = published.checkpoint.header.inHash;
994
1071
  if (!computedInHash.equals(publishedInHash)) {
@@ -1009,15 +1086,31 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1009
1086
  blocks: published.checkpoint.blocks.map((b)=>b.getStats())
1010
1087
  });
1011
1088
  }
1089
+ for (const published of validCheckpoints){
1090
+ this.instrumentation.processCheckpointL1Timing({
1091
+ slotNumber: published.checkpoint.header.slotNumber,
1092
+ l1Timestamp: published.l1.timestamp,
1093
+ l1Constants: this.l1Constants
1094
+ });
1095
+ }
1012
1096
  try {
1013
1097
  const updatedValidationResult = rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
1014
- const [processDuration, result] = await elapsed(()=>execInSpan(this.tracer, 'Archiver.addCheckpoints', ()=>this.updater.addCheckpoints(validCheckpoints, updatedValidationResult)));
1015
- this.instrumentation.processNewBlocks(processDuration / validCheckpoints.length, validCheckpoints.flatMap((c)=>c.checkpoint.blocks));
1098
+ // Split valid checkpoints: the promoted one (if any) is persisted via the proposed-promotion path,
1099
+ // the rest via addCheckpoints. Both paths run within the same store transaction for atomicity.
1100
+ const [[maybeValidCheckpointToPromote], checkpointsToAdd] = partition(validCheckpoints, (c)=>c.checkpoint.number === checkpointToPromote?.checkpoint.number);
1101
+ const [processDuration, result] = await elapsed(()=>execInSpan(this.tracer, 'Archiver.addCheckpoints', ()=>this.updater.addCheckpoints(checkpointsToAdd, updatedValidationResult, maybeValidCheckpointToPromote && {
1102
+ l1: lastCalldataCheckpoint.l1,
1103
+ attestations: lastCalldataCheckpoint.attestations,
1104
+ checkpoint: maybeValidCheckpointToPromote
1105
+ }, evictProposedFrom)));
1106
+ if (checkpointsToAdd.length > 0) {
1107
+ this.instrumentation.processNewCheckpointedBlocks(processDuration / checkpointsToAdd.length, checkpointsToAdd.flatMap((c)=>c.checkpoint.blocks));
1108
+ }
1016
1109
  // If blocks were pruned due to conflict with L1 checkpoints, emit event
1017
1110
  if (result.prunedBlocks && result.prunedBlocks.length > 0) {
1018
1111
  const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
1019
1112
  const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
1020
- this.log.warn(`Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`, {
1113
+ this.log.info(`Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`, {
1021
1114
  prunedBlocks: result.prunedBlocks.map((b)=>b.toBlockInfo()),
1022
1115
  prunedSlotNumber,
1023
1116
  prunedCheckpointNumber
@@ -1034,9 +1127,9 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1034
1127
  } catch (err) {
1035
1128
  if (err instanceof InitialCheckpointNumberNotSequentialError) {
1036
1129
  const { previousCheckpointNumber, newCheckpointNumber } = err;
1037
- const previousCheckpoint = previousCheckpointNumber ? await this.store.getCheckpointData(CheckpointNumber(previousCheckpointNumber)) : undefined;
1130
+ const previousCheckpoint = previousCheckpointNumber ? await this.stores.blocks.getCheckpointData(CheckpointNumber(previousCheckpointNumber)) : undefined;
1038
1131
  const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
1039
- await this.store.setCheckpointSynchedL1BlockNumber(updatedL1SyncPoint);
1132
+ await this.stores.blocks.setSynchedL1BlockNumber(updatedL1SyncPoint);
1040
1133
  this.log.warn(`Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`, {
1041
1134
  previousCheckpointNumber,
1042
1135
  newCheckpointNumber,
@@ -1057,7 +1150,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1057
1150
  });
1058
1151
  }
1059
1152
  lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint;
1060
- lastL1BlockWithCheckpoint = retrievedCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
1153
+ lastL1BlockWithCheckpoint = calldataCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
1061
1154
  }while (searchEndBlock < currentL1BlockNumber)
1062
1155
  // Important that we update AFTER inserting the blocks.
1063
1156
  await updateProvenCheckpoint();
@@ -1067,11 +1160,71 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1067
1160
  lastL1BlockWithCheckpoint
1068
1161
  };
1069
1162
  }
1163
+ /**
1164
+ * Checks if a specific checkpoint matches a local pending entry, and if so, loads local data to build
1165
+ * a synthetic published checkpoint (skipping blob fetch).
1166
+ *
1167
+ * Returns { diverged: true, fromCheckpointNumber } when the L1 checkpoint does NOT match local pending
1168
+ * data for that number, so the caller can evict the entire pending suffix >= fromCheckpointNumber
1169
+ * (those entries chain off the now-invalid local state) within the same addCheckpoints transaction.
1170
+ */ async tryBuildPublishedCheckpointFromProposed(calldataCheckpoint) {
1171
+ if (this.config.skipPromoteProposedCheckpointDuringL1Sync || !calldataCheckpoint) {
1172
+ return undefined;
1173
+ }
1174
+ // Look up the specific pending entry for the checkpoint being mined, not just the tip
1175
+ const proposed = await this.stores.blocks.getProposedCheckpointByNumber(calldataCheckpoint.checkpointNumber);
1176
+ if (!proposed) {
1177
+ return undefined;
1178
+ }
1179
+ if (!proposed.header.equals(calldataCheckpoint.header) || !proposed.archive.root.equals(calldataCheckpoint.archiveRoot)) {
1180
+ this.log.warn(`Local proposed checkpoint ${proposed.checkpointNumber} does not match checkpoint retrieved from L1, overriding with L1 data`, {
1181
+ proposedCheckpointNumber: proposed.checkpointNumber,
1182
+ proposedHeader: proposed.header.toInspect(),
1183
+ proposedArchiveRoot: proposed.archive.root.toString(),
1184
+ calldataCheckpointNumber: calldataCheckpoint.checkpointNumber,
1185
+ calldataHeader: calldataCheckpoint.header.toInspect(),
1186
+ calldataArchiveRoot: calldataCheckpoint.archiveRoot.toString()
1187
+ });
1188
+ // Return a divergence signal so the caller can evict pending >= this number
1189
+ return {
1190
+ diverged: true,
1191
+ fromCheckpointNumber: proposed.checkpointNumber
1192
+ };
1193
+ }
1194
+ this.log.debug(`Building published checkpoint from proposed ${calldataCheckpoint.checkpointNumber} (skipping blob fetch)`, {
1195
+ proposedHeader: proposed.header.toInspect(),
1196
+ proposedArchiveRoot: proposed.archive.root.toString()
1197
+ });
1198
+ const blocks = await this.stores.blocks.getBlocks(BlockNumber(proposed.startBlock), proposed.blockCount);
1199
+ if (blocks.length !== proposed.blockCount) {
1200
+ this.log.warn(`Local proposed checkpoint ${proposed.checkpointNumber} has wrong block count (expected ${proposed.blockCount} blocks starting at ${proposed.startBlock} but got ${blocks.length})`, {
1201
+ proposedCheckpointNumber: proposed.checkpointNumber,
1202
+ proposedStartBlock: proposed.startBlock,
1203
+ proposedBlockCount: proposed.blockCount,
1204
+ retrievedBlocks: blocks.map((b)=>b.number)
1205
+ });
1206
+ return undefined;
1207
+ }
1208
+ const checkpoint = Checkpoint.from({
1209
+ archive: proposed.archive,
1210
+ header: proposed.header,
1211
+ blocks,
1212
+ number: proposed.checkpointNumber,
1213
+ feeAssetPriceModifier: proposed.feeAssetPriceModifier
1214
+ });
1215
+ const promotedCheckpoint = PublishedCheckpoint.from({
1216
+ checkpoint,
1217
+ l1: calldataCheckpoint.l1,
1218
+ attestations: calldataCheckpoint.attestations
1219
+ });
1220
+ this.instrumentation.processCheckpointPromoted();
1221
+ return promotedCheckpoint;
1222
+ }
1070
1223
  async checkForNewCheckpointsBeforeL1SyncPoint(status, blocksSynchedTo, currentL1BlockNumber) {
1071
1224
  const { lastRetrievedCheckpoint, pendingCheckpointNumber } = status;
1072
1225
  // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the
1073
1226
  // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
1074
- const latestLocalCheckpointNumber = lastRetrievedCheckpoint?.checkpoint.number ?? await this.store.getSynchedCheckpointNumber();
1227
+ const latestLocalCheckpointNumber = lastRetrievedCheckpoint?.checkpoint.number ?? await this.stores.blocks.getLatestCheckpointNumber();
1075
1228
  if (latestLocalCheckpointNumber < pendingCheckpointNumber) {
1076
1229
  // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
1077
1230
  // but still haven't reached the pending checkpoint according to the call to the rollup contract.
@@ -1084,7 +1237,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1084
1237
  latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString();
1085
1238
  targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber;
1086
1239
  } else if (latestLocalCheckpointNumber > 0) {
1087
- const checkpoint = await this.store.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c])=>c);
1240
+ const checkpoint = await this.stores.blocks.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c])=>c);
1088
1241
  latestLocalCheckpointArchive = checkpoint.archive.root.toString();
1089
1242
  targetL1BlockNumber = checkpoint.l1.blockNumber;
1090
1243
  }
@@ -1095,7 +1248,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1095
1248
  currentL1BlockNumber,
1096
1249
  ...status
1097
1250
  });
1098
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
1251
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
1099
1252
  } else {
1100
1253
  this.log.trace(`No new checkpoints behind L1 sync point to retrieve.`, {
1101
1254
  latestLocalCheckpointNumber,
@@ -1104,7 +1257,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1104
1257
  }
1105
1258
  }
1106
1259
  async getCheckpointHeader(number) {
1107
- const checkpoint = await this.store.getCheckpointData(number);
1260
+ const checkpoint = await this.stores.blocks.getCheckpointData(number);
1108
1261
  if (!checkpoint) {
1109
1262
  return undefined;
1110
1263
  }