@aztec/archiver 0.0.1-commit.d431d1c → 0.0.1-commit.d58ff9d0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (138) hide show
  1. package/README.md +28 -11
  2. package/dest/archiver.d.ts +80 -21
  3. package/dest/archiver.d.ts.map +1 -1
  4. package/dest/archiver.js +347 -167
  5. package/dest/config.d.ts +7 -3
  6. package/dest/config.d.ts.map +1 -1
  7. package/dest/config.js +38 -14
  8. package/dest/errors.d.ts +60 -9
  9. package/dest/errors.d.ts.map +1 -1
  10. package/dest/errors.js +88 -13
  11. package/dest/factory.d.ts +13 -6
  12. package/dest/factory.d.ts.map +1 -1
  13. package/dest/factory.js +59 -39
  14. package/dest/index.d.ts +12 -3
  15. package/dest/index.d.ts.map +1 -1
  16. package/dest/index.js +11 -2
  17. package/dest/l1/bin/retrieve-calldata.js +35 -32
  18. package/dest/l1/calldata_retriever.d.ts +81 -50
  19. package/dest/l1/calldata_retriever.d.ts.map +1 -1
  20. package/dest/l1/calldata_retriever.js +202 -260
  21. package/dest/l1/data_retrieval.d.ts +32 -18
  22. package/dest/l1/data_retrieval.d.ts.map +1 -1
  23. package/dest/l1/data_retrieval.js +45 -50
  24. package/dest/l1/spire_proposer.d.ts +5 -5
  25. package/dest/l1/spire_proposer.d.ts.map +1 -1
  26. package/dest/l1/spire_proposer.js +9 -17
  27. package/dest/l1/trace_tx.d.ts +12 -66
  28. package/dest/l1/trace_tx.d.ts.map +1 -1
  29. package/dest/l1/validate_historical_logs.d.ts +23 -0
  30. package/dest/l1/validate_historical_logs.d.ts.map +1 -0
  31. package/dest/l1/validate_historical_logs.js +108 -0
  32. package/dest/l1/validate_trace.d.ts +6 -3
  33. package/dest/l1/validate_trace.d.ts.map +1 -1
  34. package/dest/l1/validate_trace.js +13 -9
  35. package/dest/modules/contract_data_source_adapter.d.ts +25 -0
  36. package/dest/modules/contract_data_source_adapter.d.ts.map +1 -0
  37. package/dest/modules/contract_data_source_adapter.js +32 -0
  38. package/dest/modules/data_source_base.d.ts +75 -48
  39. package/dest/modules/data_source_base.d.ts.map +1 -1
  40. package/dest/modules/data_source_base.js +251 -220
  41. package/dest/modules/data_store_updater.d.ts +65 -29
  42. package/dest/modules/data_store_updater.d.ts.map +1 -1
  43. package/dest/modules/data_store_updater.js +242 -153
  44. package/dest/modules/instrumentation.d.ts +30 -4
  45. package/dest/modules/instrumentation.d.ts.map +1 -1
  46. package/dest/modules/instrumentation.js +83 -20
  47. package/dest/modules/l1_synchronizer.d.ts +16 -14
  48. package/dest/modules/l1_synchronizer.d.ts.map +1 -1
  49. package/dest/modules/l1_synchronizer.js +493 -225
  50. package/dest/modules/outbox_trees_resolver.d.ts +62 -0
  51. package/dest/modules/outbox_trees_resolver.d.ts.map +1 -0
  52. package/dest/modules/outbox_trees_resolver.js +162 -0
  53. package/dest/modules/validation.d.ts +30 -7
  54. package/dest/modules/validation.d.ts.map +1 -1
  55. package/dest/modules/validation.js +35 -17
  56. package/dest/store/block_store.d.ts +200 -75
  57. package/dest/store/block_store.d.ts.map +1 -1
  58. package/dest/store/block_store.js +802 -276
  59. package/dest/store/contract_class_store.d.ts +17 -4
  60. package/dest/store/contract_class_store.d.ts.map +1 -1
  61. package/dest/store/contract_class_store.js +51 -73
  62. package/dest/store/contract_instance_store.d.ts +28 -1
  63. package/dest/store/contract_instance_store.d.ts.map +1 -1
  64. package/dest/store/contract_instance_store.js +54 -2
  65. package/dest/store/data_stores.d.ts +68 -0
  66. package/dest/store/data_stores.d.ts.map +1 -0
  67. package/dest/store/data_stores.js +54 -0
  68. package/dest/store/function_names_cache.d.ts +17 -0
  69. package/dest/store/function_names_cache.d.ts.map +1 -0
  70. package/dest/store/function_names_cache.js +30 -0
  71. package/dest/store/l2_tips_cache.d.ts +25 -0
  72. package/dest/store/l2_tips_cache.d.ts.map +1 -0
  73. package/dest/store/l2_tips_cache.js +26 -0
  74. package/dest/store/log_store.d.ts +42 -37
  75. package/dest/store/log_store.d.ts.map +1 -1
  76. package/dest/store/log_store.js +262 -388
  77. package/dest/store/log_store_codec.d.ts +78 -0
  78. package/dest/store/log_store_codec.d.ts.map +1 -0
  79. package/dest/store/log_store_codec.js +110 -0
  80. package/dest/store/message_store.d.ts +11 -1
  81. package/dest/store/message_store.d.ts.map +1 -1
  82. package/dest/store/message_store.js +51 -9
  83. package/dest/test/fake_l1_state.d.ts +28 -4
  84. package/dest/test/fake_l1_state.d.ts.map +1 -1
  85. package/dest/test/fake_l1_state.js +166 -32
  86. package/dest/test/index.js +3 -1
  87. package/dest/test/mock_archiver.d.ts +1 -1
  88. package/dest/test/mock_archiver.d.ts.map +1 -1
  89. package/dest/test/mock_archiver.js +3 -2
  90. package/dest/test/mock_l2_block_source.d.ts +65 -51
  91. package/dest/test/mock_l2_block_source.d.ts.map +1 -1
  92. package/dest/test/mock_l2_block_source.js +309 -215
  93. package/dest/test/mock_structs.d.ts +6 -2
  94. package/dest/test/mock_structs.d.ts.map +1 -1
  95. package/dest/test/mock_structs.js +26 -12
  96. package/dest/test/noop_l1_archiver.d.ts +34 -0
  97. package/dest/test/noop_l1_archiver.d.ts.map +1 -0
  98. package/dest/test/noop_l1_archiver.js +88 -0
  99. package/package.json +15 -14
  100. package/src/archiver.ts +458 -198
  101. package/src/config.ts +43 -13
  102. package/src/errors.ts +134 -21
  103. package/src/factory.ts +87 -36
  104. package/src/index.ts +19 -2
  105. package/src/l1/README.md +25 -68
  106. package/src/l1/bin/retrieve-calldata.ts +45 -33
  107. package/src/l1/calldata_retriever.ts +267 -379
  108. package/src/l1/data_retrieval.ts +70 -74
  109. package/src/l1/spire_proposer.ts +7 -15
  110. package/src/l1/validate_historical_logs.ts +140 -0
  111. package/src/l1/validate_trace.ts +24 -6
  112. package/src/modules/contract_data_source_adapter.ts +46 -0
  113. package/src/modules/data_source_base.ts +336 -288
  114. package/src/modules/data_store_updater.ts +287 -187
  115. package/src/modules/instrumentation.ts +86 -21
  116. package/src/modules/l1_synchronizer.ts +667 -278
  117. package/src/modules/outbox_trees_resolver.ts +199 -0
  118. package/src/modules/validation.ts +80 -23
  119. package/src/store/block_store.ts +984 -352
  120. package/src/store/contract_class_store.ts +57 -107
  121. package/src/store/contract_instance_store.ts +68 -5
  122. package/src/store/data_stores.ts +104 -0
  123. package/src/store/function_names_cache.ts +37 -0
  124. package/src/store/l2_tips_cache.ts +35 -0
  125. package/src/store/log_store.ts +305 -500
  126. package/src/store/log_store_codec.ts +143 -0
  127. package/src/store/message_store.ts +60 -10
  128. package/src/structs/inbox_message.ts +1 -1
  129. package/src/test/fake_l1_state.ts +215 -44
  130. package/src/test/index.ts +3 -0
  131. package/src/test/mock_archiver.ts +3 -2
  132. package/src/test/mock_l2_block_source.ts +381 -246
  133. package/src/test/mock_structs.ts +50 -17
  134. package/src/test/noop_l1_archiver.ts +158 -0
  135. package/dest/store/kv_archiver_store.d.ts +0 -336
  136. package/dest/store/kv_archiver_store.d.ts.map +0 -1
  137. package/dest/store/kv_archiver_store.js +0 -443
  138. package/src/store/kv_archiver_store.ts +0 -631
@@ -371,33 +371,39 @@ function _apply_decs_2203_r(targetClass, memberDecs, classDecs, parentClass) {
371
371
  return (_apply_decs_2203_r = applyDecs2203RFactory())(targetClass, memberDecs, classDecs, parentClass);
372
372
  }
373
373
  var _dec, _dec1, _dec2, _dec3, _initProto;
374
+ import { getFinalizedL1Block } from '@aztec/ethereum/queries';
375
+ import { asyncPool } from '@aztec/foundation/async-pool';
374
376
  import { maxBigint } from '@aztec/foundation/bigint';
375
377
  import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
376
- import { Buffer32 } from '@aztec/foundation/buffer';
377
- import { pick } from '@aztec/foundation/collection';
378
+ import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
379
+ import { compactArray, partition, pick } from '@aztec/foundation/collection';
380
+ import { EthAddress } from '@aztec/foundation/eth-address';
378
381
  import { createLogger } from '@aztec/foundation/log';
382
+ import { retryTimes } from '@aztec/foundation/retry';
379
383
  import { count } from '@aztec/foundation/string';
380
384
  import { Timer, elapsed } from '@aztec/foundation/timer';
381
- import { isDefined } from '@aztec/foundation/types';
385
+ import { isDefined, isErrorClass } from '@aztec/foundation/types';
382
386
  import { L2BlockSourceEvents } from '@aztec/stdlib/block';
383
- import { getEpochAtSlot, getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
387
+ import { Checkpoint, PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
388
+ import { getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
384
389
  import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
385
390
  import { execInSpan, trackSpan } from '@aztec/telemetry-client';
386
391
  import { InitialCheckpointNumberNotSequentialError } from '../errors.js';
387
- import { retrieveCheckpointsFromRollup, retrieveL1ToL2Message, retrieveL1ToL2Messages, retrievedToPublishedCheckpoint } from '../l1/data_retrieval.js';
392
+ import { getCheckpointBlobDataFromBlobs, retrieveCheckpointCalldataFromRollup, retrieveL1ToL2Message, retrieveL1ToL2Messages, retrievedToPublishedCheckpoint } from '../l1/data_retrieval.js';
393
+ import { getArchiverSynchPoint } from '../store/data_stores.js';
394
+ import { MessageStoreError } from '../store/message_store.js';
388
395
  import { ArchiverDataStoreUpdater } from './data_store_updater.js';
389
- import { validateCheckpointAttestations } from './validation.js';
396
+ import { validateCheckpointAttestationsFromCalldata } from './validation.js';
390
397
  _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpochPrune'), _dec2 = trackSpan('Archiver.handleL1ToL2Messages'), _dec3 = trackSpan('Archiver.handleCheckpoints');
391
398
  /**
392
399
  * Handles L1 synchronization for the archiver.
393
- * Responsible for fetching checkpoints, L1L2 messages, and handling L1 reorgs.
400
+ * Responsible for fetching checkpoints, L1 to L2 messages, and handling L1 reorgs.
394
401
  */ export class ArchiverL1Synchronizer {
395
402
  publicClient;
396
403
  debugClient;
397
404
  rollup;
398
405
  inbox;
399
- l1Addresses;
400
- store;
406
+ stores;
401
407
  config;
402
408
  blobClient;
403
409
  epochCache;
@@ -435,13 +441,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
435
441
  l1Timestamp;
436
442
  updater;
437
443
  tracer;
438
- constructor(publicClient, debugClient, rollup, inbox, l1Addresses, store, config, blobClient, epochCache, dateProvider, instrumentation, l1Constants, events, tracer, log = createLogger('archiver:l1-sync')){
444
+ constructor(publicClient, debugClient, rollup, inbox, stores, config, blobClient, epochCache, dateProvider, instrumentation, l1Constants, events, tracer, l2TipsCache, log = createLogger('archiver:l1-sync')){
439
445
  this.publicClient = publicClient;
440
446
  this.debugClient = debugClient;
441
447
  this.rollup = rollup;
442
448
  this.inbox = inbox;
443
- this.l1Addresses = l1Addresses;
444
- this.store = store;
449
+ this.stores = stores;
445
450
  this.config = config;
446
451
  this.blobClient = blobClient;
447
452
  this.epochCache = epochCache;
@@ -451,7 +456,9 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
451
456
  this.events = events;
452
457
  this.log = log;
453
458
  _initProto(this);
454
- this.updater = new ArchiverDataStoreUpdater(this.store);
459
+ this.updater = new ArchiverDataStoreUpdater(this.stores, l2TipsCache, {
460
+ rollupManaLimit: l1Constants.rollupManaLimit
461
+ });
455
462
  this.tracer = tracer;
456
463
  }
457
464
  /** Sets new config */ setConfig(newConfig) {
@@ -463,6 +470,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
463
470
  /** Returns the last L1 timestamp that was synced. */ getL1Timestamp() {
464
471
  return this.l1Timestamp;
465
472
  }
473
+ getSignatureContext() {
474
+ return {
475
+ chainId: this.publicClient.chain.id,
476
+ rollupAddress: EthAddress.fromString(this.rollup.address)
477
+ };
478
+ }
466
479
  /** Checks that the ethereum node we are connected to has a latest timestamp no more than the allowed drift. Throw if not. */ async testEthereumNodeSynced() {
467
480
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
468
481
  if (maxAllowedDelay === 0) {
@@ -477,61 +490,58 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
477
490
  }
478
491
  }
479
492
  async syncFromL1(initialSyncComplete) {
493
+ const blocksAdded = [];
494
+ // In between the various calls to L1, the block number can move meaning some of the following
495
+ // calls will return data for blocks that were not present during earlier calls. To combat this
496
+ // we ensure that all data retrieval methods only retrieve data up to the currentBlockNumber
497
+ // captured at the top of this function.
480
498
  const currentL1Block = await this.publicClient.getBlock({
481
499
  includeTransactions: false
482
500
  });
483
501
  const currentL1BlockNumber = currentL1Block.number;
484
502
  const currentL1BlockHash = Buffer32.fromString(currentL1Block.hash);
485
503
  const currentL1Timestamp = currentL1Block.timestamp;
504
+ const currentL1BlockData = {
505
+ l1BlockNumber: currentL1BlockNumber,
506
+ l1BlockHash: currentL1BlockHash
507
+ };
486
508
  if (this.l1BlockHash && currentL1BlockHash.equals(this.l1BlockHash)) {
487
509
  this.log.trace(`No new L1 blocks since last sync at L1 block ${this.l1BlockNumber}`);
488
- return;
510
+ return blocksAdded;
489
511
  }
490
- // Warn if the latest L1 block timestamp is too old
512
+ // Log at error if the latest L1 block timestamp is too old
491
513
  const maxAllowedDelay = this.config.maxAllowedEthClientDriftSeconds;
492
514
  const now = this.dateProvider.nowInSeconds();
493
515
  if (maxAllowedDelay > 0 && Number(currentL1Timestamp) <= now - maxAllowedDelay) {
494
- this.log.warn(`Latest L1 block ${currentL1BlockNumber} timestamp ${currentL1Timestamp} is too old. Make sure your Ethereum node is synced.`, {
516
+ this.log.error(`Latest L1 block ${currentL1BlockNumber} timestamp ${currentL1Timestamp} is too old. Make sure your Ethereum node is synced.`, {
495
517
  currentL1BlockNumber,
496
518
  currentL1Timestamp,
497
519
  now,
498
520
  maxAllowedDelay
499
521
  });
500
522
  }
501
- // Load sync point for blocks and messages defaulting to start block
502
- const { blocksSynchedTo = this.l1Constants.l1StartBlock, messagesSynchedTo = {
503
- l1BlockNumber: this.l1Constants.l1StartBlock,
504
- l1BlockHash: this.l1Constants.l1StartBlockHash
505
- } } = await this.store.getSynchPoint();
523
+ // Query finalized block on L1
524
+ const rawFinalizedL1Block = await getFinalizedL1Block(this.publicClient);
525
+ const finalizedL1Block = rawFinalizedL1Block && {
526
+ l1BlockNumber: rawFinalizedL1Block.number,
527
+ l1BlockHash: Buffer32.fromString(rawFinalizedL1Block.hash)
528
+ };
529
+ // Load sync point for blocks defaulting to start block
530
+ const { blocksSynchedTo = this.l1Constants.l1StartBlock } = await getArchiverSynchPoint(this.stores);
506
531
  this.log.debug(`Starting new archiver sync iteration`, {
507
532
  blocksSynchedTo,
508
- messagesSynchedTo,
509
- currentL1BlockNumber,
510
- currentL1BlockHash
533
+ currentL1BlockData,
534
+ finalizedL1Block
511
535
  });
512
- // ********** Ensuring Consistency of data pulled from L1 **********
513
- /**
514
- * There are a number of calls in this sync operation to L1 for retrieving
515
- * events and transaction data. There are a couple of things we need to bear in mind
516
- * to ensure that data is read exactly once.
517
- *
518
- * The first is the problem of eventually consistent ETH service providers like Infura.
519
- * Each L1 read operation will query data from the last L1 block that it saw emit its kind of data.
520
- * (so pending L1 to L2 messages will read from the last L1 block that emitted a message and so on)
521
- * This will mean the archiver will lag behind L1 and will only advance when there's L2-relevant activity on the chain.
522
- *
523
- * The second is that in between the various calls to L1, the block number can move meaning some
524
- * of the following calls will return data for blocks that were not present during earlier calls.
525
- * To combat this for the time being we simply ensure that all data retrieval methods only retrieve
526
- * data up to the currentBlockNumber captured at the top of this function. We might want to improve on this
527
- * in future but for the time being it should give us the guarantees that we need
528
- */ // ********** Events that are processed per L1 block **********
529
- await this.handleL1ToL2Messages(messagesSynchedTo, currentL1BlockNumber);
530
- // ********** Events that are processed per checkpoint **********
536
+ // Sync L1 to L2 messages. We retry this a few times since there are error conditions that reset the sync point, requiring a new iteration.
537
+ // Note that we cannot just wait for the l1 synchronizer to loop again, since the synchronizer would report as synced up to the current L1
538
+ // block, when that wouldn't be the case, since L1 to L2 messages would need another iteration.
539
+ await retryTimes(()=>this.handleL1ToL2Messages(currentL1BlockData, finalizedL1Block), 'Handling L1 to L2 messages', 3, 0.1);
531
540
  if (currentL1BlockNumber > blocksSynchedTo) {
532
541
  // First we retrieve new checkpoints and L2 blocks and store them in the DB. This will also update the
533
542
  // pending chain validation status, proven checkpoint number, and synched L1 block number.
534
543
  const rollupStatus = await this.handleCheckpoints(blocksSynchedTo, currentL1BlockNumber, initialSyncComplete);
544
+ blocksAdded.push(...rollupStatus.blocksAdded);
535
545
  // Then we try pruning uncheckpointed blocks if a new slot was mined without checkpoints
536
546
  await this.pruneUncheckpointedBlocks(currentL1Timestamp);
537
547
  // Then we prune the current epoch if it'd reorg on next submission.
@@ -540,21 +550,16 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
540
550
  // the chain locally before we start unwinding stuff. This can be optimized by figuring out
541
551
  // up to which point we're pruning, and then requesting checkpoints up to that point only.
542
552
  const { rollupCanPrune } = await this.handleEpochPrune(rollupStatus.provenCheckpointNumber, currentL1BlockNumber, currentL1Timestamp);
543
- // If the last checkpoint we processed had an invalid attestation, we manually advance the L1 syncpoint
544
- // past it, since otherwise we'll keep downloading it and reprocessing it on every iteration until
545
- // we get a valid checkpoint to advance the syncpoint.
546
- if (!rollupStatus.validationResult?.valid && rollupStatus.lastL1BlockWithCheckpoint !== undefined) {
547
- await this.store.setCheckpointSynchedL1BlockNumber(rollupStatus.lastL1BlockWithCheckpoint);
548
- }
549
553
  // And lastly we check if we are missing any checkpoints behind us due to a possible L1 reorg.
550
554
  // We only do this if rollup cant prune on the next submission. Otherwise we will end up
551
- // re-syncing the checkpoints we have just unwound above. We also dont do this if the last checkpoint is invalid,
552
- // since the archiver will rightfully refuse to sync up to it.
553
- if (!rollupCanPrune && rollupStatus.validationResult?.valid) {
555
+ // re-syncing the checkpoints we have just unwound above.
556
+ if (!rollupCanPrune) {
554
557
  await this.checkForNewCheckpointsBeforeL1SyncPoint(rollupStatus, blocksSynchedTo, currentL1BlockNumber);
555
558
  }
556
559
  this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
557
560
  }
561
+ // Update the finalized L2 checkpoint based on L1 finality.
562
+ await this.updateFinalizedCheckpoint(finalizedL1Block);
558
563
  // After syncing has completed, update the current l1 block number and timestamp,
559
564
  // otherwise we risk announcing to the world that we've synced to a given point,
560
565
  // but the corresponding blocks have not been processed (see #12631).
@@ -567,38 +572,68 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
567
572
  l1TimestampAtStart: currentL1Timestamp,
568
573
  l1BlockNumberAtEnd
569
574
  });
575
+ return blocksAdded;
576
+ }
577
+ /** Updates the finalized checkpoint using the pre-fetched finalized L1 block from the current sync iteration. */ async updateFinalizedCheckpoint(finalizedL1Block) {
578
+ try {
579
+ if (!finalizedL1Block) {
580
+ this.log.trace(`Skipping finalized checkpoint update: L1 has no finalized block yet.`);
581
+ return;
582
+ }
583
+ const finalizedL1BlockNumber = finalizedL1Block.l1BlockNumber;
584
+ const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
585
+ blockNumber: finalizedL1BlockNumber
586
+ });
587
+ const localFinalizedCheckpointNumber = await this.stores.blocks.getFinalizedCheckpointNumber();
588
+ if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
589
+ await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
590
+ this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
591
+ finalizedCheckpointNumber,
592
+ finalizedL1BlockNumber
593
+ });
594
+ }
595
+ } catch (err) {
596
+ // The rollup contract may not exist at the finalized L1 block right after deployment.
597
+ if (!err?.message?.includes('returned no data')) {
598
+ this.log.warn(`Failed to update finalized checkpoint: ${err}`);
599
+ }
600
+ }
570
601
  }
571
602
  /** Prune all proposed local blocks that should have been checkpointed by now. */ async pruneUncheckpointedBlocks(currentL1Timestamp) {
572
603
  const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
573
- this.store.getCheckpointedL2BlockNumber(),
574
- this.store.getLatestBlockNumber()
604
+ this.stores.blocks.getCheckpointedL2BlockNumber(),
605
+ this.stores.blocks.getLatestL2BlockNumber()
575
606
  ]);
576
607
  // If there are no uncheckpointed blocks, we got nothing to do
577
608
  if (lastProposedBlockNumber === lastCheckpointedBlockNumber) {
578
609
  this.log.trace(`No uncheckpointed blocks to prune.`);
579
610
  return;
580
611
  }
581
- // What's the slot of the first uncheckpointed block?
582
- const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
583
- const [firstUncheckpointedBlockHeader] = await this.store.getBlockHeaders(firstUncheckpointedBlockNumber, 1);
584
- const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
585
612
  // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
586
- const nextL1BlockTimestamp = currentL1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration);
587
- const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
588
- // Prune provisional blocks from slots that have ended without being checkpointed
589
- if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
590
- this.log.warn(`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`, {
591
- firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(),
592
- slotAtNextL1Block
613
+ const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
614
+ const firstUncheckpointedBlockNumber = BlockNumber(lastCheckpointedBlockNumber + 1);
615
+ // What's the slot of the first uncheckpointed block?
616
+ const firstUncheckpointedBlockData = await this.stores.blocks.getBlockData({
617
+ number: firstUncheckpointedBlockNumber
618
+ });
619
+ const firstUncheckpointedBlockSlot = firstUncheckpointedBlockData?.header.getSlot();
620
+ if (firstUncheckpointedBlockSlot === undefined || firstUncheckpointedBlockSlot >= slotAtNextL1Block) {
621
+ return;
622
+ }
623
+ // Prune provisional blocks from slots that have ended without being checkpointed.
624
+ // This also clears any proposed checkpoint whose blocks are being pruned.
625
+ this.log.warn(`Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`, {
626
+ firstUncheckpointedBlockHeader: firstUncheckpointedBlockData?.header.toInspect(),
627
+ slotAtNextL1Block
628
+ });
629
+ const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
630
+ if (prunedBlocks.length > 0) {
631
+ this.instrumentation.recordPrune('uncheckpointed');
632
+ this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
633
+ type: L2BlockSourceEvents.L2PruneUncheckpointed,
634
+ slotNumber: firstUncheckpointedBlockSlot,
635
+ blocks: prunedBlocks
593
636
  });
594
- const prunedBlocks = await this.updater.removeBlocksAfter(lastCheckpointedBlockNumber);
595
- if (prunedBlocks.length > 0) {
596
- this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
597
- type: L2BlockSourceEvents.L2PruneUncheckpointed,
598
- slotNumber: firstUncheckpointedBlockSlot,
599
- blocks: prunedBlocks
600
- });
601
- }
602
637
  }
603
638
  }
604
639
  /** Queries the rollup contract on whether a prune can be executed on the immediate next L1 block. */ async canPrune(currentL1BlockNumber, currentL1Timestamp) {
@@ -617,7 +652,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
617
652
  }
618
653
  /** Checks if there'd be a reorg for the next checkpoint submission and start pruning now. */ async handleEpochPrune(provenCheckpointNumber, currentL1BlockNumber, currentL1Timestamp) {
619
654
  const rollupCanPrune = await this.canPrune(currentL1BlockNumber, currentL1Timestamp);
620
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
655
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
621
656
  const canPrune = localPendingCheckpointNumber > provenCheckpointNumber && rollupCanPrune;
622
657
  if (canPrune) {
623
658
  const timer = new Timer();
@@ -629,12 +664,14 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
629
664
  const pruneFromSlotNumber = header.slotNumber;
630
665
  const pruneFromEpochNumber = getEpochAtSlot(pruneFromSlotNumber, this.l1Constants);
631
666
  const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
632
- const checkpointPromises = Array.from({
667
+ // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
668
+ // promises when the gap between local pending and proven checkpoint numbers is large.
669
+ const BATCH_SIZE = 10;
670
+ const indices = Array.from({
633
671
  length: checkpointsToUnwind
634
- }).fill(0).map((_, i)=>this.store.getCheckpointData(CheckpointNumber(i + pruneFrom)));
635
- const checkpoints = await Promise.all(checkpointPromises);
636
- const blockPromises = await Promise.all(checkpoints.filter(isDefined).map((cp)=>this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber))));
637
- const newBlocks = blockPromises.filter(isDefined).flat();
672
+ }, (_, i)=>CheckpointNumber(i + pruneFrom));
673
+ const checkpoints = (await asyncPool(BATCH_SIZE, indices, (idx)=>this.stores.blocks.getCheckpointData(idx))).filter(isDefined);
674
+ const newBlocks = (await asyncPool(BATCH_SIZE, checkpoints, (cp)=>this.stores.blocks.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)))).filter(isDefined).flat();
638
675
  // Emit an event for listening services to react to the chain prune
639
676
  this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
640
677
  type: L2BlockSourceEvents.L2PruneUnproven,
@@ -642,12 +679,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
642
679
  blocks: newBlocks
643
680
  });
644
681
  this.log.debug(`L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`);
645
- await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
646
- this.log.warn(`Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` + `to ${provenCheckpointNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` + `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`);
682
+ await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
683
+ this.log.warn(`Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` + `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` + `Updated latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`);
647
684
  this.instrumentation.processPrune(timer.ms());
648
685
  // TODO(palla/reorg): Do we need to set the block synched L1 block number here?
649
686
  // Seems like the next iteration should handle this.
650
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
687
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
651
688
  }
652
689
  return {
653
690
  rollupCanPrune
@@ -668,59 +705,79 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
668
705
  nextEnd
669
706
  ];
670
707
  }
671
- async handleL1ToL2Messages(messagesSyncPoint, currentL1BlockNumber) {
672
- this.log.trace(`Handling L1 to L2 messages from ${messagesSyncPoint.l1BlockNumber} to ${currentL1BlockNumber}.`);
673
- if (currentL1BlockNumber <= messagesSyncPoint.l1BlockNumber) {
674
- return;
708
+ async handleL1ToL2Messages(currentL1Block, finalizedL1Block) {
709
+ // Load the syncpoint, which may have been updated in a previous iteration
710
+ const { messagesSynchedTo = {
711
+ l1BlockNumber: this.l1Constants.l1StartBlock,
712
+ l1BlockHash: this.l1Constants.l1StartBlockHash
713
+ } } = await getArchiverSynchPoint(this.stores);
714
+ // Nothing to do if L1 block number has not moved forward
715
+ const currentL1BlockNumber = currentL1Block.l1BlockNumber;
716
+ if (currentL1BlockNumber <= messagesSynchedTo.l1BlockNumber) {
717
+ return true;
675
718
  }
676
- // Load remote and local inbox states.
677
- const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
678
- const localLastMessage = await this.store.getLastL1ToL2Message();
719
+ // Compare local message store state with the remote. If they match, we just advance the match pointer.
679
720
  const remoteMessagesState = await this.inbox.getState({
680
721
  blockNumber: currentL1BlockNumber
681
722
  });
682
- this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
683
- localMessagesInserted,
684
- localLastMessage,
685
- remoteMessagesState
686
- });
687
- // Compare message count and rolling hash. If they match, no need to retrieve anything.
688
- if (remoteMessagesState.totalMessagesInserted === localMessagesInserted && remoteMessagesState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer32.ZERO)) {
689
- this.log.trace(`No L1 to L2 messages to query between L1 blocks ${messagesSyncPoint.l1BlockNumber} and ${currentL1BlockNumber}.`);
690
- return;
723
+ const localLastMessage = await this.stores.messages.getLastMessage();
724
+ if (await this.localStateMatches(localLastMessage, remoteMessagesState)) {
725
+ this.log.trace(`Local L1 to L2 messages are already in sync with remote at L1 block ${currentL1BlockNumber}`);
726
+ await this.stores.messages.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress, finalizedL1Block);
727
+ return true;
691
728
  }
692
- // Check if our syncpoint is still valid. If not, there was an L1 reorg and we need to re-retrieve messages.
693
- // Note that we need to fetch it from logs and not from inbox state at the syncpoint l1 block number, since it
694
- // could be older than 128 blocks and non-archive nodes cannot resolve it.
695
- if (localLastMessage) {
696
- const remoteLastMessage = await this.retrieveL1ToL2Message(localLastMessage.leaf);
697
- this.log.trace(`Retrieved remote message for local last`, {
698
- remoteLastMessage,
699
- localLastMessage
700
- });
701
- if (!remoteLastMessage || !remoteLastMessage.rollingHash.equals(localLastMessage.rollingHash)) {
702
- this.log.warn(`Rolling back L1 to L2 messages due to hash mismatch or msg not found.`, {
703
- remoteLastMessage,
704
- messagesSyncPoint,
705
- localLastMessage
706
- });
707
- messagesSyncPoint = await this.rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint);
708
- this.log.debug(`Rolled back L1 to L2 messages to L1 block ${messagesSyncPoint.l1BlockNumber}.`, {
709
- messagesSyncPoint
729
+ // If not, then we are out of sync. Most likely there are new messages on the inbox, so we try retrieving them.
730
+ // However, it could also be the case that there was an L1 reorg and our syncpoint is no longer valid.
731
+ // If that's the case, we'd get an exception out of the message store since the rolling hash of the first message
732
+ // we try to insert would not match the one in the db, in which case we rollback to the last common message with L1.
733
+ try {
734
+ await this.retrieveAndStoreMessages(messagesSynchedTo.l1BlockNumber, currentL1BlockNumber);
735
+ } catch (error) {
736
+ if (isErrorClass(error, MessageStoreError)) {
737
+ this.log.warn(`Failed to store L1 to L2 messages retrieved from L1: ${error.message}. Rolling back syncpoint to retry.`, {
738
+ inboxMessage: error.inboxMessage
710
739
  });
740
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
741
+ return false;
711
742
  }
743
+ throw error;
712
744
  }
713
- // Retrieve and save messages in batches. Each batch is estimated to acommodate up to L2 'blockBatchSize' blocks,
745
+ // Note that, if there are no new messages to insert, but there was an L1 reorg that pruned out last messages,
746
+ // we'd notice by comparing our local state with the remote one again, and seeing they don't match even after
747
+ // our sync attempt. In this case, we also rollback our syncpoint, and trigger a retry.
748
+ const localLastMessageAfterSync = await this.stores.messages.getLastMessage();
749
+ if (!await this.localStateMatches(localLastMessageAfterSync, remoteMessagesState)) {
750
+ this.log.warn(`Local L1 to L2 messages state does not match remote after sync attempt. Rolling back syncpoint to retry.`, {
751
+ localLastMessageAfterSync,
752
+ remoteMessagesState
753
+ });
754
+ await this.rollbackL1ToL2Messages(remoteMessagesState);
755
+ return false;
756
+ }
757
+ // Advance the syncpoint after a successful sync
758
+ await this.stores.messages.setMessageSyncState(currentL1Block, remoteMessagesState.treeInProgress, finalizedL1Block);
759
+ return true;
760
+ }
761
+ /** Checks if the local rolling hash and message count matches the remote state */ async localStateMatches(localLastMessage, remoteState) {
762
+ const localMessageCount = await this.stores.messages.getTotalL1ToL2MessageCount();
763
+ this.log.trace(`Comparing local and remote inbox state`, {
764
+ localMessageCount,
765
+ localLastMessage,
766
+ remoteState
767
+ });
768
+ return remoteState.totalMessagesInserted === localMessageCount && remoteState.messagesRollingHash.equals(localLastMessage?.rollingHash ?? Buffer16.ZERO);
769
+ }
770
+ /** Retrieves L1 to L2 messages from L1 in batches and stores them. */ async retrieveAndStoreMessages(fromL1Block, toL1Block) {
714
771
  let searchStartBlock = 0n;
715
- let searchEndBlock = messagesSyncPoint.l1BlockNumber;
772
+ let searchEndBlock = fromL1Block;
716
773
  let lastMessage;
717
774
  let messageCount = 0;
718
775
  do {
719
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
776
+ [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, toL1Block);
720
777
  this.log.trace(`Retrieving L1 to L2 messages in L1 blocks ${searchStartBlock}-${searchEndBlock}`);
721
778
  const messages = await retrieveL1ToL2Messages(this.inbox, searchStartBlock, searchEndBlock);
722
779
  const timer = new Timer();
723
- await this.store.addL1ToL2Messages(messages);
780
+ await this.stores.messages.addL1ToL2Messages(messages);
724
781
  const perMsg = timer.ms() / messages.length;
725
782
  this.instrumentation.processNewMessages(messages.length, perMsg);
726
783
  for (const msg of messages){
@@ -731,74 +788,94 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
731
788
  lastMessage = msg;
732
789
  messageCount++;
733
790
  }
734
- }while (searchEndBlock < currentL1BlockNumber)
735
- // Log stats for messages retrieved (if any).
791
+ }while (searchEndBlock < toL1Block)
736
792
  if (messageCount > 0) {
737
793
  this.log.info(`Retrieved ${messageCount} new L1 to L2 messages up to message with index ${lastMessage?.index} for checkpoint ${lastMessage?.checkpointNumber}`, {
738
794
  lastMessage,
739
795
  messageCount
740
796
  });
741
797
  }
742
- // Warn if the resulting rolling hash does not match the remote state we had retrieved.
743
- if (lastMessage && !lastMessage.rollingHash.equals(remoteMessagesState.messagesRollingHash)) {
744
- this.log.warn(`Last message retrieved rolling hash does not match remote state.`, {
745
- lastMessage,
746
- remoteMessagesState
747
- });
748
- }
749
- }
750
- async retrieveL1ToL2Message(leaf) {
751
- const currentL1BlockNumber = await this.publicClient.getBlockNumber();
752
- let searchStartBlock = 0n;
753
- let searchEndBlock = this.l1Constants.l1StartBlock - 1n;
754
- do {
755
- [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
756
- const message = await retrieveL1ToL2Message(this.inbox, leaf, searchStartBlock, searchEndBlock);
757
- if (message) {
758
- return message;
759
- }
760
- }while (searchEndBlock < currentL1BlockNumber)
761
- return undefined;
762
798
  }
763
- async rollbackL1ToL2Messages(localLastMessage, messagesSyncPoint) {
764
- // Slowly go back through our messages until we find the last common message.
765
- // We could query the logs in batch as an optimization, but the depth of the reorg should not be deep, and this
766
- // is a very rare case, so it's fine to query one log at a time.
799
+ /**
800
+ * Rolls back local L1 to L2 messages to the last common message with L1, and updates the syncpoint to the L1 block of that message.
801
+ * If no common message is found, rolls back all messages and sets the syncpoint to the start block.
802
+ */ async rollbackL1ToL2Messages(remoteMessagesState) {
803
+ const { treeInProgress: remoteTreeInProgress, messagesRollingHash: remoteRollingHash } = remoteMessagesState;
804
+ const messagesFinalizedL1Block = await this.stores.messages.getMessagesFinalizedL1Block();
805
+ const finalizedL1BlockNumber = messagesFinalizedL1Block?.l1BlockNumber;
806
+ // Slowly go back through our messages until we find the last common message. We could query the logs in
807
+ // batch as an optimization, but the depth of the reorg should not be deep, and this is a very rare case,
808
+ // so it's fine to query one log at a time.
767
809
  let commonMsg;
768
- this.log.verbose(`Searching most recent common L1 to L2 message at or before index ${localLastMessage.index}`);
769
- for await (const msg of this.store.iterateL1ToL2Messages({
770
- reverse: true,
771
- end: localLastMessage.index
810
+ let messagesToDelete = 0;
811
+ this.log.verbose(`Searching most recent common L1 to L2 message`);
812
+ for await (const localMsg of this.stores.messages.iterateL1ToL2Messages({
813
+ reverse: true
772
814
  })){
773
- const remoteMsg = await this.retrieveL1ToL2Message(msg.leaf);
774
815
  const logCtx = {
775
- remoteMsg,
776
- localMsg: msg
816
+ remoteMsg: undefined,
817
+ localMsg,
818
+ remoteMessagesState
777
819
  };
778
- if (remoteMsg && remoteMsg.rollingHash.equals(msg.rollingHash)) {
779
- this.log.verbose(`Found most recent common L1 to L2 message at index ${msg.index} on L1 block ${msg.l1BlockNumber}`, logCtx);
820
+ // First check if the local message rolling hash matches the current rolling hash of the inbox contract,
821
+ // which means we just need to rollback some local messages and we should be back in sync. This means there
822
+ // was an L1 reorg that removed some of the messages we had, but no new messages were added compared.
823
+ if (localMsg.rollingHash.equals(remoteRollingHash)) {
824
+ this.log.info(`Found common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber} matching current remote state`, logCtx);
825
+ commonMsg = localMsg;
826
+ break;
827
+ }
828
+ // Messages at or below the finalized L1 block cannot have been reorged — accept as common without querying L1.
829
+ if (finalizedL1BlockNumber !== undefined && localMsg.l1BlockNumber <= finalizedL1BlockNumber) {
830
+ this.log.info(`Found common L1 to L2 message at finalized L1 block ${localMsg.l1BlockNumber}`, logCtx);
831
+ commonMsg = localMsg;
832
+ break;
833
+ }
834
+ // If there's no match with the current remote state, check if the message exists on the inbox contract at all
835
+ // by looking at the inbox events. If the L1 reorg *added* new messages in addition to deleting existing ones,
836
+ // then the current remote state's rolling hash will not match anything we have locally, so we need to check existence
837
+ // of individual messages via logs. Note we use logs and not historical queries so we don't have to depend on
838
+ // an archival rpc node, since the message could be from a long time ago if we're catching up with syncing.
839
+ const remoteMsg = await retrieveL1ToL2Message(this.inbox, localMsg);
840
+ logCtx.remoteMsg = remoteMsg;
841
+ if (remoteMsg && remoteMsg.rollingHash.equals(localMsg.rollingHash)) {
842
+ this.log.info(`Found most recent common L1 to L2 message at index ${localMsg.index} on L1 block ${localMsg.l1BlockNumber}`, logCtx);
780
843
  commonMsg = remoteMsg;
781
844
  break;
782
845
  } else if (remoteMsg) {
783
- this.log.debug(`Local L1 to L2 message with index ${msg.index} has different rolling hash`, logCtx);
846
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} has different rolling hash`, logCtx);
847
+ messagesToDelete++;
784
848
  } else {
785
- this.log.debug(`Local L1 to L2 message with index ${msg.index} not found on L1`, logCtx);
849
+ this.log.debug(`Local L1 to L2 message with index ${localMsg.index} not found on L1`, logCtx);
850
+ messagesToDelete++;
786
851
  }
787
852
  }
788
- // Delete everything after the common message we found.
789
- const lastGoodIndex = commonMsg?.index;
790
- this.log.warn(`Deleting all local L1 to L2 messages after index ${lastGoodIndex ?? 'undefined'}`);
791
- await this.store.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
853
+ // Delete everything after the common message we found, if anything needs to be deleted.
854
+ // Do not exit early if there are no messages to delete, we still want to update the syncpoint.
855
+ if (messagesToDelete > 0) {
856
+ const lastGoodIndex = commonMsg?.index;
857
+ this.log.warn(`Rolling back all local L1 to L2 messages after index ${lastGoodIndex ?? 'initial'}`);
858
+ await this.stores.messages.removeL1ToL2Messages(lastGoodIndex !== undefined ? lastGoodIndex + 1n : 0n);
859
+ }
792
860
  // Update the syncpoint so the loop below reprocesses the changed messages. We go to the block before
793
861
  // the last common one, so we force reprocessing it, in case new messages were added on that same L1 block
794
- // after the last common message.
795
- const syncPointL1BlockNumber = commonMsg ? commonMsg.l1BlockNumber - 1n : this.l1Constants.l1StartBlock;
796
- const syncPointL1BlockHash = await this.getL1BlockHash(syncPointL1BlockNumber);
797
- messagesSyncPoint = {
862
+ // after the last common message. Cap at the finalized L1 block: messages at or below finalized cannot
863
+ // have been reorged, so there is no need to walk back any further than that.
864
+ const syncPointL1BlockNumber = maxBigint(...compactArray([
865
+ commonMsg ? commonMsg.l1BlockNumber - 1n : undefined,
866
+ finalizedL1BlockNumber,
867
+ this.l1Constants.l1StartBlock
868
+ ]));
869
+ const syncPointL1BlockHash = syncPointL1BlockNumber === finalizedL1BlockNumber ? messagesFinalizedL1Block.l1BlockHash : await this.getL1BlockHash(syncPointL1BlockNumber);
870
+ const messagesSyncPoint = {
798
871
  l1BlockNumber: syncPointL1BlockNumber,
799
872
  l1BlockHash: syncPointL1BlockHash
800
873
  };
801
- await this.store.setMessageSynchedL1Block(messagesSyncPoint);
874
+ await this.stores.messages.setMessageSyncState(messagesSyncPoint, remoteTreeInProgress);
875
+ this.log.verbose(`Updated messages syncpoint to L1 block ${messagesSyncPoint.l1BlockNumber}`, {
876
+ ...messagesSyncPoint,
877
+ remoteTreeInProgress
878
+ });
802
879
  return messagesSyncPoint;
803
880
  }
804
881
  async getL1BlockHash(l1BlockNumber) {
@@ -812,8 +889,9 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
812
889
  return Buffer32.fromString(block.hash);
813
890
  }
814
891
  async handleCheckpoints(blocksSynchedTo, currentL1BlockNumber, initialSyncComplete) {
815
- const localPendingCheckpointNumber = await this.store.getSynchedCheckpointNumber();
816
- const initialValidationResult = await this.store.getPendingChainValidationStatus();
892
+ const blocksAdded = [];
893
+ const localPendingCheckpointNumber = await this.stores.blocks.getLatestCheckpointNumber();
894
+ const initialValidationResult = await this.stores.blocks.getPendingChainValidationStatus();
817
895
  const { provenCheckpointNumber, provenArchive, pendingCheckpointNumber, pendingArchive, archiveOfMyCheckpoint: archiveForLocalPendingCheckpointNumber } = await execInSpan(this.tracer, 'Archiver.getRollupStatus', ()=>this.rollup.status(localPendingCheckpointNumber, {
818
896
  blockNumber: currentL1BlockNumber
819
897
  }));
@@ -822,7 +900,8 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
822
900
  provenArchive: provenArchive.toString(),
823
901
  pendingCheckpointNumber,
824
902
  pendingArchive: pendingArchive.toString(),
825
- validationResult: initialValidationResult
903
+ validationResult: initialValidationResult,
904
+ blocksAdded
826
905
  };
827
906
  this.log.trace(`Retrieved rollup status at current L1 block ${currentL1BlockNumber}.`, {
828
907
  localPendingCheckpointNumber,
@@ -836,39 +915,39 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
836
915
  // we need to set it to zero. This is an edge case because we dont have a checkpoint zero (initial checkpoint is one),
837
916
  // so localCheckpointForDestinationProvenCheckpointNumber would not be found below.
838
917
  if (provenCheckpointNumber === 0) {
839
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
918
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
840
919
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
841
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
920
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
842
921
  this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, {
843
922
  provenCheckpointNumber
844
923
  });
845
924
  }
846
925
  }
847
- const localCheckpointForDestinationProvenCheckpointNumber = await this.store.getCheckpointData(provenCheckpointNumber);
926
+ const localCheckpointForDestinationProvenCheckpointNumber = await this.stores.blocks.getCheckpointData(provenCheckpointNumber);
848
927
  // Sanity check. I've hit what seems to be a state where the proven checkpoint is set to a value greater than the latest
849
928
  // synched checkpoint when requesting L2Tips from the archiver. This is the only place where the proven checkpoint is set.
850
- const synched = await this.store.getSynchedCheckpointNumber();
929
+ const synched = await this.stores.blocks.getLatestCheckpointNumber();
851
930
  if (localCheckpointForDestinationProvenCheckpointNumber && synched < localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber) {
852
931
  this.log.error(`Hit local checkpoint greater than last synched checkpoint: ${localCheckpointForDestinationProvenCheckpointNumber.checkpointNumber} > ${synched}`);
853
932
  }
854
933
  this.log.trace(`Local checkpoint for remote proven checkpoint ${provenCheckpointNumber} is ${localCheckpointForDestinationProvenCheckpointNumber?.archive.root.toString() ?? 'undefined'}`);
855
934
  if (localCheckpointForDestinationProvenCheckpointNumber && provenArchive.equals(localCheckpointForDestinationProvenCheckpointNumber.archive.root)) {
856
- const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
935
+ const localProvenCheckpointNumber = await this.stores.blocks.getProvenCheckpointNumber();
857
936
  if (localProvenCheckpointNumber !== provenCheckpointNumber) {
858
- await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
937
+ await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
859
938
  this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, {
860
939
  provenCheckpointNumber
861
940
  });
862
941
  const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
863
942
  const provenEpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
864
- const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + localCheckpointForDestinationProvenCheckpointNumber.numBlocks - 1;
943
+ const lastBlockNumberInCheckpoint = localCheckpointForDestinationProvenCheckpointNumber.startBlock + localCheckpointForDestinationProvenCheckpointNumber.blockCount - 1;
865
944
  this.events.emit(L2BlockSourceEvents.L2BlockProven, {
866
945
  type: L2BlockSourceEvents.L2BlockProven,
867
946
  blockNumber: BlockNumber(lastBlockNumberInCheckpoint),
868
947
  slotNumber: provenSlotNumber,
869
948
  epochNumber: provenEpochNumber
870
949
  });
871
- this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
950
+ this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
872
951
  } else {
873
952
  this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
874
953
  }
@@ -878,7 +957,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
878
957
  // If we have 0 checkpoints locally and there are no checkpoints onchain there is nothing to do.
879
958
  const noCheckpoints = localPendingCheckpointNumber === 0 && pendingCheckpointNumber === 0;
880
959
  if (noCheckpoints) {
881
- await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
960
+ await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
882
961
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}, no checkpoints on chain`);
883
962
  return rollupStatus;
884
963
  }
@@ -886,7 +965,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
886
965
  // Related to the L2 reorgs of the pending chain. We are only interested in actually addressing a reorg if there
887
966
  // are any state that could be impacted by it. If we have no checkpoints, there is no impact.
888
967
  if (localPendingCheckpointNumber > 0) {
889
- const localPendingCheckpoint = await this.store.getCheckpointData(localPendingCheckpointNumber);
968
+ const localPendingCheckpoint = await this.stores.blocks.getCheckpointData(localPendingCheckpointNumber);
890
969
  if (localPendingCheckpoint === undefined) {
891
970
  throw new Error(`Missing checkpoint ${localPendingCheckpointNumber}`);
892
971
  }
@@ -900,7 +979,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
900
979
  // However, in the re-org scenario, our L1 node is temporarily lying to us and we end up potentially missing checkpoints.
901
980
  // We must only set this block number based on actually retrieved logs.
902
981
  // TODO(#8621): Tackle this properly when we handle L1 Re-orgs.
903
- // await this.store.setCheckpointSynchedL1BlockNumber(currentL1BlockNumber);
982
+ // await this.stores.blocks.setSynchedL1BlockNumber(currentL1BlockNumber);
904
983
  this.log.debug(`No checkpoints to retrieve from ${blocksSynchedTo + 1n} to ${currentL1BlockNumber}`);
905
984
  return rollupStatus;
906
985
  }
@@ -917,7 +996,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
917
996
  });
918
997
  let tipAfterUnwind = localPendingCheckpointNumber;
919
998
  while(true){
920
- const candidateCheckpoint = await this.store.getCheckpointData(tipAfterUnwind);
999
+ const candidateCheckpoint = await this.stores.blocks.getCheckpointData(tipAfterUnwind);
921
1000
  if (candidateCheckpoint === undefined) {
922
1001
  break;
923
1002
  }
@@ -931,9 +1010,12 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
931
1010
  }
932
1011
  tipAfterUnwind--;
933
1012
  }
934
- const checkpointsToUnwind = localPendingCheckpointNumber - tipAfterUnwind;
935
- await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
936
- this.log.warn(`Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` + `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`);
1013
+ const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
1014
+ await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
1015
+ if (checkpointsToRemove > 0) {
1016
+ this.instrumentation.recordPrune('l1_mismatch');
1017
+ }
1018
+ this.log.warn(`Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` + `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` + `Updated L2 latest checkpoint is ${await this.stores.blocks.getLatestCheckpointNumber()}.`);
937
1019
  }
938
1020
  }
939
1021
  // Retrieve checkpoints in batches. Each batch is estimated to accommodate up to 'blockBatchSize' L1 blocks,
@@ -941,41 +1023,64 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
941
1023
  let searchStartBlock = blocksSynchedTo;
942
1024
  let searchEndBlock = blocksSynchedTo;
943
1025
  let lastRetrievedCheckpoint;
944
- let lastL1BlockWithCheckpoint = undefined;
1026
+ let lastSeenCheckpoint;
945
1027
  do {
946
1028
  [searchStartBlock, searchEndBlock] = this.nextRange(searchEndBlock, currentL1BlockNumber);
947
1029
  this.log.trace(`Retrieving checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
948
- // TODO(md): Retrieve from blob client then from consensus client, then from peers
949
- const retrievedCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointsFromRollup', ()=>retrieveCheckpointsFromRollup(this.rollup, this.publicClient, this.debugClient, this.blobClient, searchStartBlock, searchEndBlock, this.l1Addresses, this.instrumentation, this.log, !initialSyncComplete));
950
- if (retrievedCheckpoints.length === 0) {
1030
+ // First fetch calldata only, no blobs yet, since we may be able to just get that data out of the proposed chain
1031
+ const calldataCheckpoints = await execInSpan(this.tracer, 'Archiver.retrieveCheckpointCalldataFromRollup', ()=>retrieveCheckpointCalldataFromRollup(this.rollup, this.publicClient, this.debugClient, searchStartBlock, searchEndBlock, this.instrumentation, this.log));
1032
+ if (calldataCheckpoints.length === 0) {
951
1033
  // We are not calling `setBlockSynchedL1BlockNumber` because it may cause sync issues if based off infura.
952
1034
  // See further details in earlier comments.
953
1035
  this.log.trace(`Retrieved no new checkpoints from L1 block ${searchStartBlock} to ${searchEndBlock}`);
954
1036
  continue;
955
1037
  }
956
- this.log.debug(`Retrieved ${retrievedCheckpoints.length} new checkpoints between L1 blocks ${searchStartBlock} and ${searchEndBlock}`, {
957
- lastProcessedCheckpoint: retrievedCheckpoints[retrievedCheckpoints.length - 1].l1,
1038
+ this.log.debug(`Retrieved ${calldataCheckpoints.length} new checkpoint calldata between L1 blocks ${searchStartBlock} and ${searchEndBlock}`, {
1039
+ lastProcessedCheckpoint: calldataCheckpoints[calldataCheckpoints.length - 1].l1,
958
1040
  searchStartBlock,
959
1041
  searchEndBlock
960
1042
  });
961
- const publishedCheckpoints = await Promise.all(retrievedCheckpoints.map((b)=>retrievedToPublishedCheckpoint(b)));
962
- const validCheckpoints = [];
963
- for (const published of publishedCheckpoints){
1043
+ // Check if the last checkpoint matches a local pending entry (so we can skip blob fetch).
1044
+ // We only check the last one; if it matches, the blob fetch is skipped for that entry.
1045
+ // TODO(palla/pipelining): We may have more than a single checkpoint to promote
1046
+ const lastCalldataCheckpoint = calldataCheckpoints[calldataCheckpoints.length - 1];
1047
+ const promoteResult = await this.tryBuildPublishedCheckpointFromProposed(lastCalldataCheckpoint);
1048
+ const checkpointToPromote = promoteResult && !('diverged' in promoteResult) ? promoteResult : undefined;
1049
+ const evictProposedFrom = promoteResult && 'diverged' in promoteResult ? promoteResult.fromCheckpointNumber : undefined;
1050
+ // Validate attestations from CALLDATA before fetching any blobs. A checkpoint with invalid
1051
+ // attestations (or one descending from a rejected ancestor) is rejected here without fetching its
1052
+ // blobs, so a malformed blob does not throw during decode before the rejection path runs and
1053
+ // stall sync. The signed consensus payload (header, archive root, fee asset price
1054
+ // modifier) is fully available from calldata.
1055
+ const checkpointsToIngest = [];
1056
+ for (const calldataCheckpoint of calldataCheckpoints){
1057
+ // Check the attestations uploaded by the publisher to L1 are correct.
1058
+ // Rollup contract does not validate attestations to save on gas, so this
1059
+ // falls on the nodes to verify offchain and skip those checkpoints.
964
1060
  const validationResult = this.config.skipValidateCheckpointAttestations ? {
965
1061
  valid: true
966
- } : await validateCheckpointAttestations(published, this.epochCache, this.l1Constants, this.log);
967
- // Only update the validation result if it has changed, so we can keep track of the first invalid checkpoint
1062
+ } : await validateCheckpointAttestationsFromCalldata(calldataCheckpoint, this.epochCache, this.l1Constants, this.getSignatureContext(), this.log);
1063
+ // Also skip the checkpoint if it builds on a previously-rejected ancestor. Without
1064
+ // this, addCheckpoints would throw InitialCheckpointNumberNotSequentialError when the
1065
+ // ancestor was skipped earlier (e.g. due to invalid attestations), the catch handler
1066
+ // would roll back the L1 sync point, and the next iteration would re-fetch and re-throw.
1067
+ const rejectedAncestor = await this.stores.blocks.getRejectedCheckpointByArchiveRoot(calldataCheckpoint.header.lastArchiveRoot);
1068
+ // Update the validation result if it has changed, so we can keep track of the first invalid checkpoint
968
1069
  // in case there is a sequence of more than one invalid checkpoint, as we need to invalidate the first one.
969
1070
  // There is an exception though: if a checkpoint is invalidated and replaced with another invalid checkpoint,
970
1071
  // we need to update the validation result, since we need to be able to invalidate the new one.
971
1072
  // See test 'chain progresses if an invalid checkpoint is invalidated with an invalid one' for more info.
972
- if (rollupStatus.validationResult?.valid !== validationResult.valid || !rollupStatus.validationResult.valid && !validationResult.valid && rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber) {
1073
+ // Do not update the validation result if there is a rejected ancestor, since in that case we want to keep the
1074
+ // original invalidation, as the new checkpoint is extending from a previous invalid one.
1075
+ const validStatusChanged = rollupStatus.validationResult?.valid !== validationResult.valid;
1076
+ const invalidStatusWithSameCheckpointNumber = !validationResult.valid && rollupStatus.validationResult && !rollupStatus.validationResult.valid && rollupStatus.validationResult.checkpoint.checkpointNumber === validationResult.checkpoint.checkpointNumber;
1077
+ if (!rejectedAncestor && (validStatusChanged || invalidStatusWithSameCheckpointNumber)) {
973
1078
  rollupStatus.validationResult = validationResult;
974
1079
  }
975
1080
  if (!validationResult.valid) {
976
- this.log.warn(`Skipping checkpoint ${published.checkpoint.number} due to invalid attestations`, {
977
- checkpointHash: published.checkpoint.hash(),
978
- l1BlockNumber: published.l1.blockNumber,
1081
+ this.log.warn(`Skipping checkpoint ${calldataCheckpoint.checkpointNumber} due to invalid attestations`, {
1082
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1083
+ l1BlockNumber: calldataCheckpoint.l1.blockNumber,
979
1084
  ...pick(validationResult, 'reason')
980
1085
  });
981
1086
  // Emit event for invalid checkpoint detection
@@ -983,12 +1088,80 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
983
1088
  type: L2BlockSourceEvents.InvalidAttestationsCheckpointDetected,
984
1089
  validationResult
985
1090
  });
1091
+ // Persist a rejected-ancestor entry so any later checkpoint that builds on this one
1092
+ // is detected and skipped (rather than tripping the addCheckpoints consecutive-number
1093
+ // check and causing the sync point to roll back in a loop).
1094
+ await this.stores.blocks.addRejectedCheckpoint({
1095
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1096
+ archiveRoot: calldataCheckpoint.archiveRoot,
1097
+ parentArchiveRoot: calldataCheckpoint.header.lastArchiveRoot,
1098
+ slotNumber: calldataCheckpoint.header.slotNumber,
1099
+ l1: calldataCheckpoint.l1,
1100
+ reason: 'invalid-attestations'
1101
+ });
1102
+ continue;
1103
+ }
1104
+ if (rejectedAncestor) {
1105
+ const descendantInfo = {
1106
+ archive: calldataCheckpoint.archiveRoot,
1107
+ lastArchive: calldataCheckpoint.header.lastArchiveRoot,
1108
+ slotNumber: calldataCheckpoint.header.slotNumber,
1109
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1110
+ timestamp: calldataCheckpoint.header.timestamp
1111
+ };
1112
+ this.log.warn(`Skipping checkpoint ${calldataCheckpoint.checkpointNumber} as it is a descendant of ` + `rejected checkpoint ${rejectedAncestor.checkpointNumber} (${rejectedAncestor.reason})`, {
1113
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1114
+ l1BlockNumber: calldataCheckpoint.l1.blockNumber,
1115
+ l1BlockHash: calldataCheckpoint.l1.blockHash,
1116
+ ancestorCheckpointNumber: rejectedAncestor.checkpointNumber,
1117
+ ancestorArchiveRoot: rejectedAncestor.archiveRoot.toString(),
1118
+ ancestorReason: rejectedAncestor.reason
1119
+ });
1120
+ this.events.emit(L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected, {
1121
+ type: L2BlockSourceEvents.DescendentOfInvalidAttestationsCheckpointDetected,
1122
+ checkpoint: descendantInfo,
1123
+ ancestorArchiveRoot: rejectedAncestor.archiveRoot,
1124
+ ancestorCheckpointNumber: rejectedAncestor.checkpointNumber
1125
+ });
1126
+ // Persist this chainpoint as rejected as well, so we can construct a chain of
1127
+ // skipped checkpoints starting from the first one with invalid attestations.
1128
+ await this.stores.blocks.addRejectedCheckpoint({
1129
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1130
+ archiveRoot: calldataCheckpoint.archiveRoot,
1131
+ parentArchiveRoot: calldataCheckpoint.header.lastArchiveRoot,
1132
+ slotNumber: calldataCheckpoint.header.slotNumber,
1133
+ l1: calldataCheckpoint.l1,
1134
+ reason: 'descends-from-invalid-attestations'
1135
+ });
986
1136
  continue;
987
1137
  }
1138
+ checkpointsToIngest.push(calldataCheckpoint);
1139
+ }
1140
+ // Fetch blobs in parallel only for the surviving (attestation-valid, non-descendant) checkpoints,
1141
+ // then build the full published checkpoints. The last calldata checkpoint may be promotable from a
1142
+ // local proposed block (checkpointToPromote), in which case it carries no blob to fetch. A missing or
1143
+ // undecodable blob throws and propagates, rolling back the L1 sync point so the fetch is retried.
1144
+ const toFetchBlobs = checkpointToPromote ? checkpointsToIngest.filter((c)=>c.checkpointNumber !== checkpointToPromote.checkpoint.number) : checkpointsToIngest;
1145
+ const blobFetched = await asyncPool(10, toFetchBlobs, async (checkpoint)=>retrievedToPublishedCheckpoint({
1146
+ ...checkpoint,
1147
+ checkpointBlobData: await getCheckpointBlobDataFromBlobs(this.blobClient, checkpoint.l1.blockHash, checkpoint.blobHashes, checkpoint.checkpointNumber, this.log, !initialSyncComplete, checkpoint.parentBeaconBlockRoot, checkpoint.l1.timestamp)
1148
+ }));
1149
+ // Index the built checkpoints by number so we can ingest them in calldata order, slotting in the
1150
+ // promoted checkpoint (built from a local proposed block rather than blobs).
1151
+ const publishedByNumber = new Map(blobFetched.map((published)=>[
1152
+ published.checkpoint.number,
1153
+ published
1154
+ ]));
1155
+ if (checkpointToPromote) {
1156
+ publishedByNumber.set(checkpointToPromote.checkpoint.number, checkpointToPromote);
1157
+ }
1158
+ const validCheckpoints = [];
1159
+ for (const calldataCheckpoint of checkpointsToIngest){
1160
+ const published = publishedByNumber.get(calldataCheckpoint.checkpointNumber);
988
1161
  // Check the inHash of the checkpoint against the l1->l2 messages.
989
1162
  // The messages should've been synced up to the currentL1BlockNumber and must be available for the published
990
1163
  // checkpoints we just retrieved.
991
- const l1ToL2Messages = await this.store.getL1ToL2Messages(published.checkpoint.number);
1164
+ const l1ToL2Messages = await this.stores.messages.getL1ToL2Messages(published.checkpoint.number);
992
1165
  const computedInHash = computeInHashFromL1ToL2Messages(l1ToL2Messages);
993
1166
  const publishedInHash = published.checkpoint.header.inHash;
994
1167
  if (!computedInHash.equals(publishedInHash)) {
@@ -1009,19 +1182,40 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1009
1182
  blocks: published.checkpoint.blocks.map((b)=>b.getStats())
1010
1183
  });
1011
1184
  }
1185
+ for (const published of validCheckpoints){
1186
+ this.instrumentation.processCheckpointL1Timing({
1187
+ slotNumber: published.checkpoint.header.slotNumber,
1188
+ l1Timestamp: published.l1.timestamp,
1189
+ l1Constants: this.l1Constants
1190
+ });
1191
+ }
1012
1192
  try {
1013
1193
  const updatedValidationResult = rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
1014
- const [processDuration, result] = await elapsed(()=>execInSpan(this.tracer, 'Archiver.setCheckpointData', ()=>this.updater.setNewCheckpointData(validCheckpoints, updatedValidationResult)));
1015
- this.instrumentation.processNewBlocks(processDuration / validCheckpoints.length, validCheckpoints.flatMap((c)=>c.checkpoint.blocks));
1194
+ // Split valid checkpoints: the promoted one (if any) is persisted via the proposed-promotion path,
1195
+ // the rest via addCheckpoints. Both paths run within the same store transaction for atomicity.
1196
+ const [[maybeValidCheckpointToPromote], checkpointsToAdd] = partition(validCheckpoints, (c)=>c.checkpoint.number === checkpointToPromote?.checkpoint.number);
1197
+ const [processDuration, result] = await elapsed(()=>execInSpan(this.tracer, 'Archiver.addCheckpoints', ()=>this.updater.addCheckpoints(checkpointsToAdd, updatedValidationResult, maybeValidCheckpointToPromote && {
1198
+ l1: lastCalldataCheckpoint.l1,
1199
+ attestations: lastCalldataCheckpoint.attestations,
1200
+ checkpoint: maybeValidCheckpointToPromote
1201
+ }, evictProposedFrom)));
1202
+ if (validCheckpoints.length > 0) {
1203
+ this.instrumentation.processNewCheckpointedBlocks(processDuration / validCheckpoints.length, validCheckpoints.flatMap((c)=>c.checkpoint.blocks));
1204
+ }
1205
+ // Record blocks newly fetched from L1 checkpoint payloads as added. The promoted checkpoint (if any) is
1206
+ // excluded: its blocks were already in local archiver storage (added via the proposed-block path) so the
1207
+ // block stream does not need them re-downloaded.
1208
+ blocksAdded.push(...checkpointsToAdd.flatMap((c)=>c.checkpoint.blocks));
1016
1209
  // If blocks were pruned due to conflict with L1 checkpoints, emit event
1017
1210
  if (result.prunedBlocks && result.prunedBlocks.length > 0) {
1018
1211
  const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
1019
1212
  const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
1020
- this.log.warn(`Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`, {
1213
+ this.log.info(`Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`, {
1021
1214
  prunedBlocks: result.prunedBlocks.map((b)=>b.toBlockInfo()),
1022
1215
  prunedSlotNumber,
1023
1216
  prunedCheckpointNumber
1024
1217
  });
1218
+ this.instrumentation.recordPrune('l1_conflict');
1025
1219
  // Emit event for listening services to react to the prune.
1026
1220
  // Note: slotNumber comes from the first pruned block. If pruned blocks theoretically spanned multiple slots,
1027
1221
  // only one slot number would be reported (though in practice all blocks in a checkpoint span a single slot).
@@ -1034,11 +1228,15 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1034
1228
  } catch (err) {
1035
1229
  if (err instanceof InitialCheckpointNumberNotSequentialError) {
1036
1230
  const { previousCheckpointNumber, newCheckpointNumber } = err;
1037
- const previousCheckpoint = previousCheckpointNumber ? await this.store.getCheckpointData(CheckpointNumber(previousCheckpointNumber)) : undefined;
1038
- const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
1039
- await this.store.setCheckpointSynchedL1BlockNumber(updatedL1SyncPoint);
1231
+ const previousCheckpoint = previousCheckpointNumber ? await this.stores.blocks.getCheckpointData(CheckpointNumber(previousCheckpointNumber)) : undefined;
1232
+ const lastFinalizedCheckpoint = await this.stores.blocks.getCheckpointData(await this.stores.blocks.getFinalizedCheckpointNumber());
1233
+ const updatedL1SyncPoint = previousCheckpoint?.l1.blockNumber ?? lastFinalizedCheckpoint?.l1.blockNumber ?? this.l1Constants.l1StartBlock;
1234
+ await this.stores.blocks.setSynchedL1BlockNumber(updatedL1SyncPoint);
1040
1235
  this.log.warn(`Attempting to insert checkpoint ${newCheckpointNumber} with previous block ${previousCheckpointNumber}. Rolling back L1 sync point to ${updatedL1SyncPoint} to try and fetch the missing blocks.`, {
1041
1236
  previousCheckpointNumber,
1237
+ previousCheckpoint: previousCheckpoint?.header.toInspect(),
1238
+ lastFinalizedCheckpoint: lastFinalizedCheckpoint?.header.toInspect(),
1239
+ l1StartBlock: this.l1Constants.l1StartBlock,
1042
1240
  newCheckpointNumber,
1043
1241
  updatedL1SyncPoint
1044
1242
  });
@@ -1057,45 +1255,115 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1057
1255
  });
1058
1256
  }
1059
1257
  lastRetrievedCheckpoint = validCheckpoints.at(-1) ?? lastRetrievedCheckpoint;
1060
- lastL1BlockWithCheckpoint = retrievedCheckpoints.at(-1)?.l1.blockNumber ?? lastL1BlockWithCheckpoint;
1258
+ // The last checkpoint seen on L1 this batch (valid or rejected), tracked from calldata since
1259
+ // rejected checkpoints are no longer built into PublishedCheckpoints.
1260
+ lastSeenCheckpoint = lastCalldataCheckpoint;
1061
1261
  }while (searchEndBlock < currentL1BlockNumber)
1062
1262
  // Important that we update AFTER inserting the blocks.
1063
1263
  await updateProvenCheckpoint();
1064
1264
  return {
1065
1265
  ...rollupStatus,
1066
1266
  lastRetrievedCheckpoint,
1067
- lastL1BlockWithCheckpoint
1267
+ lastSeenCheckpoint
1068
1268
  };
1069
1269
  }
1270
+ /**
1271
+ * Checks if a specific checkpoint matches a local pending entry, and if so, loads local data to build
1272
+ * a synthetic published checkpoint (skipping blob fetch).
1273
+ *
1274
+ * Returns { diverged: true, fromCheckpointNumber } when the L1 checkpoint does NOT match local pending
1275
+ * data for that number, so the caller can evict the entire pending suffix >= fromCheckpointNumber
1276
+ * (those entries chain off the now-invalid local state) within the same addCheckpoints transaction.
1277
+ */ async tryBuildPublishedCheckpointFromProposed(calldataCheckpoint) {
1278
+ if (this.config.skipPromoteProposedCheckpointDuringL1Sync || !calldataCheckpoint) {
1279
+ return undefined;
1280
+ }
1281
+ // Look up the specific pending entry for the checkpoint being mined, not just the tip
1282
+ const proposed = await this.stores.blocks.getProposedCheckpointByNumber(calldataCheckpoint.checkpointNumber);
1283
+ if (!proposed) {
1284
+ return undefined;
1285
+ }
1286
+ if (!proposed.header.equals(calldataCheckpoint.header) || !proposed.archive.root.equals(calldataCheckpoint.archiveRoot)) {
1287
+ this.log.warn(`Local proposed checkpoint ${proposed.checkpointNumber} does not match checkpoint retrieved from L1, overriding with L1 data`, {
1288
+ proposedCheckpointNumber: proposed.checkpointNumber,
1289
+ proposedHeader: proposed.header.toInspect(),
1290
+ proposedArchiveRoot: proposed.archive.root.toString(),
1291
+ calldataCheckpointNumber: calldataCheckpoint.checkpointNumber,
1292
+ calldataHeader: calldataCheckpoint.header.toInspect(),
1293
+ calldataArchiveRoot: calldataCheckpoint.archiveRoot.toString()
1294
+ });
1295
+ // Both the locally-proposed checkpoint and the L1-confirmed one are signed by the
1296
+ // slot proposer; emit a divergence event so the slasher can attribute equivocation.
1297
+ // Only emit when the slots match — uncheckpointed entries are pruned above so this
1298
+ // should always hold, but guard defensively to avoid mis-attributing a slash.
1299
+ if (proposed.header.slotNumber === calldataCheckpoint.header.slotNumber) {
1300
+ this.events.emit(L2BlockSourceEvents.CheckpointEquivocationDetected, {
1301
+ type: L2BlockSourceEvents.CheckpointEquivocationDetected,
1302
+ slotNumber: calldataCheckpoint.header.slotNumber,
1303
+ checkpointNumber: calldataCheckpoint.checkpointNumber,
1304
+ l1ArchiveRoot: calldataCheckpoint.archiveRoot,
1305
+ proposedArchiveRoot: proposed.archive.root
1306
+ });
1307
+ }
1308
+ // Return a divergence signal so the caller can evict pending >= this number
1309
+ return {
1310
+ diverged: true,
1311
+ fromCheckpointNumber: proposed.checkpointNumber
1312
+ };
1313
+ }
1314
+ this.log.debug(`Building published checkpoint from proposed ${calldataCheckpoint.checkpointNumber} (skipping blob fetch)`, {
1315
+ proposedHeader: proposed.header.toInspect(),
1316
+ proposedArchiveRoot: proposed.archive.root.toString()
1317
+ });
1318
+ const blocks = await this.stores.blocks.getBlocks({
1319
+ from: BlockNumber(proposed.startBlock),
1320
+ limit: proposed.blockCount
1321
+ });
1322
+ if (blocks.length !== proposed.blockCount) {
1323
+ this.log.warn(`Local proposed checkpoint ${proposed.checkpointNumber} has wrong block count (expected ${proposed.blockCount} blocks starting at ${proposed.startBlock} but got ${blocks.length})`, {
1324
+ proposedCheckpointNumber: proposed.checkpointNumber,
1325
+ proposedStartBlock: proposed.startBlock,
1326
+ proposedBlockCount: proposed.blockCount,
1327
+ retrievedBlocks: blocks.map((b)=>b.number)
1328
+ });
1329
+ return undefined;
1330
+ }
1331
+ const checkpoint = Checkpoint.from({
1332
+ archive: proposed.archive,
1333
+ header: proposed.header,
1334
+ blocks,
1335
+ number: proposed.checkpointNumber,
1336
+ feeAssetPriceModifier: proposed.feeAssetPriceModifier
1337
+ });
1338
+ const promotedCheckpoint = PublishedCheckpoint.from({
1339
+ checkpoint,
1340
+ l1: calldataCheckpoint.l1,
1341
+ attestations: calldataCheckpoint.attestations
1342
+ });
1343
+ this.instrumentation.processCheckpointPromoted();
1344
+ return promotedCheckpoint;
1345
+ }
1070
1346
  async checkForNewCheckpointsBeforeL1SyncPoint(status, blocksSynchedTo, currentL1BlockNumber) {
1071
- const { lastRetrievedCheckpoint, pendingCheckpointNumber } = status;
1072
- // Compare the last checkpoint we have (either retrieved in this round or loaded from store) with what the
1073
- // rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
1074
- const latestLocalCheckpointNumber = lastRetrievedCheckpoint?.checkpoint.number ?? await this.store.getSynchedCheckpointNumber();
1347
+ const { lastSeenCheckpoint, pendingCheckpointNumber } = status;
1348
+ // Compare the last checkpoint (valid or not) we have (either retrieved in this round or loaded from store)
1349
+ // with what the rollup contract told us was the latest one (pinned at the currentL1BlockNumber).
1350
+ const latestLocalCheckpointNumber = lastSeenCheckpoint?.checkpointNumber ?? CheckpointNumber.max(await this.stores.blocks.getLatestCheckpointNumber(), await this.stores.blocks.getLatestRejectedCheckpointNumber()) ?? CheckpointNumber.ZERO;
1075
1351
  if (latestLocalCheckpointNumber < pendingCheckpointNumber) {
1076
1352
  // Here we have consumed all logs until the `currentL1Block` we pinned at the beginning of the archiver loop,
1077
1353
  // but still haven't reached the pending checkpoint according to the call to the rollup contract.
1078
1354
  // We suspect an L1 reorg that added checkpoints *behind* us. If that is the case, it must have happened between
1079
1355
  // the last checkpoint we saw and the current one, so we reset the last synched L1 block number. In the edge case
1080
1356
  // we don't have one, we go back 2 L1 epochs, which is the deepest possible reorg (assuming Casper is working).
1081
- let latestLocalCheckpointArchive = undefined;
1082
- let targetL1BlockNumber = maxBigint(currentL1BlockNumber - 64n, 0n);
1083
- if (lastRetrievedCheckpoint) {
1084
- latestLocalCheckpointArchive = lastRetrievedCheckpoint.checkpoint.archive.root.toString();
1085
- targetL1BlockNumber = lastRetrievedCheckpoint.l1.blockNumber;
1086
- } else if (latestLocalCheckpointNumber > 0) {
1087
- const checkpoint = await this.store.getRangeOfCheckpoints(latestLocalCheckpointNumber, 1).then(([c])=>c);
1088
- latestLocalCheckpointArchive = checkpoint.archive.root.toString();
1089
- targetL1BlockNumber = checkpoint.l1.blockNumber;
1090
- }
1357
+ const latestLocalCheckpoint = lastSeenCheckpoint ?? await this.stores.blocks.getCheckpointData(latestLocalCheckpointNumber) ?? await this.stores.blocks.getRejectedCheckpointByNumber(latestLocalCheckpointNumber);
1358
+ const targetL1BlockNumber = latestLocalCheckpoint?.l1.blockNumber ?? maxBigint(currentL1BlockNumber - 64n, this.l1Constants.l1StartBlock, 0n);
1091
1359
  this.log.warn(`Failed to reach checkpoint ${pendingCheckpointNumber} at ${currentL1BlockNumber} (latest is ${latestLocalCheckpointNumber}). ` + `Rolling back last synched L1 block number to ${targetL1BlockNumber}.`, {
1092
1360
  latestLocalCheckpointNumber,
1093
- latestLocalCheckpointArchive,
1361
+ latestLocalCheckpointL1: latestLocalCheckpoint?.l1,
1094
1362
  blocksSynchedTo,
1095
1363
  currentL1BlockNumber,
1096
1364
  ...status
1097
1365
  });
1098
- await this.store.setCheckpointSynchedL1BlockNumber(targetL1BlockNumber);
1366
+ await this.stores.blocks.setSynchedL1BlockNumber(targetL1BlockNumber);
1099
1367
  } else {
1100
1368
  this.log.trace(`No new checkpoints behind L1 sync point to retrieve.`, {
1101
1369
  latestLocalCheckpointNumber,
@@ -1104,7 +1372,7 @@ _dec = trackSpan('Archiver.syncFromL1'), _dec1 = trackSpan('Archiver.handleEpoch
1104
1372
  }
1105
1373
  }
1106
1374
  async getCheckpointHeader(number) {
1107
- const checkpoint = await this.store.getCheckpointData(number);
1375
+ const checkpoint = await this.stores.blocks.getCheckpointData(number);
1108
1376
  if (!checkpoint) {
1109
1377
  return undefined;
1110
1378
  }