@aztec/archiver 0.0.1-commit.3469e52 → 0.0.1-commit.35158ae7e

package/src/modules/l1_synchronizer.ts

@@ -1,22 +1,21 @@
 import type { BlobClientInterface } from '@aztec/blob-client/client';
 import { EpochCache } from '@aztec/epoch-cache';
 import { InboxContract, RollupContract } from '@aztec/ethereum/contracts';
-import type { L1ContractAddresses } from '@aztec/ethereum/l1-contract-addresses';
 import type { L1BlockId } from '@aztec/ethereum/l1-types';
 import type { ViemPublicClient, ViemPublicDebugClient } from '@aztec/ethereum/types';
+import { asyncPool } from '@aztec/foundation/async-pool';
 import { maxBigint } from '@aztec/foundation/bigint';
 import { BlockNumber, CheckpointNumber, EpochNumber } from '@aztec/foundation/branded-types';
 import { Buffer32 } from '@aztec/foundation/buffer';
 import { pick } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
-import { EthAddress } from '@aztec/foundation/eth-address';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 import { count } from '@aztec/foundation/string';
 import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
 import { isDefined } from '@aztec/foundation/types';
 import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
 import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
-import { type L1RollupConstants, getEpochAtSlot, getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
+import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
 import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
 import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
 
@@ -28,6 +27,7 @@ import {
   retrievedToPublishedCheckpoint,
 } from '../l1/data_retrieval.js';
 import type { KVArchiverDataStore } from '../store/kv_archiver_store.js';
+import type { L2TipsCache } from '../store/l2_tips_cache.js';
 import type { InboxMessage } from '../structs/inbox_message.js';
 import { ArchiverDataStoreUpdater } from './data_store_updater.js';
 import type { ArchiverInstrumentation } from './instrumentation.js';
@@ -60,10 +60,6 @@ export class ArchiverL1Synchronizer implements Traceable {
     private readonly debugClient: ViemPublicDebugClient,
     private readonly rollup: RollupContract,
     private readonly inbox: InboxContract,
-    private readonly l1Addresses: Pick<
-      L1ContractAddresses,
-      'registryAddress' | 'governanceProposerAddress' | 'slashFactoryAddress'
-    > & { slashingProposerAddress: EthAddress },
     private readonly store: KVArchiverDataStore,
     private config: {
       batchSize: number;
@@ -74,12 +70,18 @@ export class ArchiverL1Synchronizer implements Traceable {
     private readonly epochCache: EpochCache,
     private readonly dateProvider: DateProvider,
     private readonly instrumentation: ArchiverInstrumentation,
-    private readonly l1Constants: L1RollupConstants & { l1StartBlockHash: Buffer32; genesisArchiveRoot: Fr },
+    private readonly l1Constants: L1RollupConstants & {
+      l1StartBlockHash: Buffer32;
+      genesisArchiveRoot: Fr;
+    },
     private readonly events: ArchiverEmitter,
     tracer: Tracer,
+    l2TipsCache?: L2TipsCache,
     private readonly log: Logger = createLogger('archiver:l1-sync'),
   ) {
-    this.updater = new ArchiverDataStoreUpdater(this.store);
+    this.updater = new ArchiverDataStoreUpdater(this.store, l2TipsCache, {
+      rollupManaLimit: l1Constants.rollupManaLimit,
+    });
     this.tracer = tracer;
   }
 
@@ -215,6 +217,9 @@ export class ArchiverL1Synchronizer implements Traceable {
       this.instrumentation.updateL1BlockHeight(currentL1BlockNumber);
     }
 
+    // Update the finalized L2 checkpoint based on L1 finality.
+    await this.updateFinalizedCheckpoint();
+
     // After syncing has completed, update the current l1 block number and timestamp,
     // otherwise we risk announcing to the world that we've synced to a given point,
     // but the corresponding blocks have not been processed (see #12631).
@@ -230,6 +235,27 @@ export class ArchiverL1Synchronizer implements Traceable {
     });
   }
 
+  /** Query L1 for its finalized block and update the finalized checkpoint accordingly. */
+  private async updateFinalizedCheckpoint(): Promise<void> {
+    try {
+      const finalizedL1Block = await this.publicClient.getBlock({ blockTag: 'finalized', includeTransactions: false });
+      const finalizedL1BlockNumber = finalizedL1Block.number;
+      const finalizedCheckpointNumber = await this.rollup.getProvenCheckpointNumber({
+        blockNumber: finalizedL1BlockNumber,
+      });
+      const localFinalizedCheckpointNumber = await this.store.getFinalizedCheckpointNumber();
+      if (localFinalizedCheckpointNumber !== finalizedCheckpointNumber) {
+        await this.updater.setFinalizedCheckpointNumber(finalizedCheckpointNumber);
+        this.log.info(`Updated finalized chain to checkpoint ${finalizedCheckpointNumber}`, {
+          finalizedCheckpointNumber,
+          finalizedL1BlockNumber,
+        });
+      }
+    } catch (err) {
+      this.log.warn(`Failed to update finalized checkpoint: ${err}`);
+    }
+  }
+
   /** Prune all proposed local blocks that should have been checkpointed by now. */
   private async pruneUncheckpointedBlocks(currentL1Timestamp: bigint) {
     const [lastCheckpointedBlockNumber, lastProposedBlockNumber] = await Promise.all([
@@ -249,8 +275,7 @@ export class ArchiverL1Synchronizer implements Traceable {
     const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
 
     // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
-    const nextL1BlockTimestamp = currentL1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration);
-    const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
+    const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
 
     // Prune provisional blocks from slots that have ended without being checkpointed
     if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
@@ -258,7 +283,7 @@ export class ArchiverL1Synchronizer implements Traceable {
         `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
         { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
       );
-      const prunedBlocks = await this.updater.removeBlocksAfter(lastCheckpointedBlockNumber);
+      const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
 
       if (prunedBlocks.length > 0) {
         this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
@@ -309,17 +334,20 @@ export class ArchiverL1Synchronizer implements Traceable {
 
       const checkpointsToUnwind = localPendingCheckpointNumber - provenCheckpointNumber;
 
-      const checkpointPromises = Array.from({ length: checkpointsToUnwind })
-        .fill(0)
-        .map((_, i) => this.store.getCheckpointData(CheckpointNumber(i + pruneFrom)));
-      const checkpoints = await Promise.all(checkpointPromises);
-
-      const blockPromises = await Promise.all(
-        checkpoints
-          .filter(isDefined)
-          .map(cp => this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber))),
+      // Fetch checkpoints and blocks in bounded batches to avoid unbounded concurrent
+      // promises when the gap between local pending and proven checkpoint numbers is large.
+      const BATCH_SIZE = 10;
+      const indices = Array.from({ length: checkpointsToUnwind }, (_, i) => CheckpointNumber(i + pruneFrom));
+      const checkpoints = (await asyncPool(BATCH_SIZE, indices, idx => this.store.getCheckpointData(idx))).filter(
+        isDefined,
       );
-      const newBlocks = blockPromises.filter(isDefined).flat();
+      const newBlocks = (
+        await asyncPool(BATCH_SIZE, checkpoints, cp =>
+          this.store.getBlocksForCheckpoint(CheckpointNumber(cp.checkpointNumber)),
+        )
+      )
+        .filter(isDefined)
+        .flat();
 
       // Emit an event for listening services to react to the chain prune
       this.events.emit(L2BlockSourceEvents.L2PruneUnproven, {
@@ -331,10 +359,10 @@ export class ArchiverL1Synchronizer implements Traceable {
       this.log.debug(
         `L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
       );
-      await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
+      await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
       this.log.warn(
-        `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
-          `to ${provenCheckpointNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
+        `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
+          `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
           `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
       );
       this.instrumentation.processPrune(timer.ms());
@@ -367,6 +395,7 @@ export class ArchiverL1Synchronizer implements Traceable {
     const localMessagesInserted = await this.store.getTotalL1ToL2MessageCount();
     const localLastMessage = await this.store.getLastL1ToL2Message();
     const remoteMessagesState = await this.inbox.getState({ blockNumber: currentL1BlockNumber });
+    await this.store.setInboxTreeInProgress(remoteMessagesState.treeInProgress);
 
     this.log.trace(`Retrieved remote inbox state at L1 block ${currentL1BlockNumber}.`, {
       localMessagesInserted,
@@ -551,7 +580,7 @@ export class ArchiverL1Synchronizer implements Traceable {
       if (provenCheckpointNumber === 0) {
         const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
         if (localProvenCheckpointNumber !== provenCheckpointNumber) {
-          await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
+          await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
           this.log.info(`Rolled back proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
         }
       }
@@ -583,13 +612,13 @@ export class ArchiverL1Synchronizer implements Traceable {
       ) {
         const localProvenCheckpointNumber = await this.store.getProvenCheckpointNumber();
         if (localProvenCheckpointNumber !== provenCheckpointNumber) {
-          await this.store.setProvenCheckpointNumber(provenCheckpointNumber);
+          await this.updater.setProvenCheckpointNumber(provenCheckpointNumber);
           this.log.info(`Updated proven chain to checkpoint ${provenCheckpointNumber}`, { provenCheckpointNumber });
           const provenSlotNumber = localCheckpointForDestinationProvenCheckpointNumber.header.slotNumber;
           const provenEpochNumber: EpochNumber = getEpochAtSlot(provenSlotNumber, this.l1Constants);
           const lastBlockNumberInCheckpoint =
             localCheckpointForDestinationProvenCheckpointNumber.startBlock +
-            localCheckpointForDestinationProvenCheckpointNumber.numBlocks -
+            localCheckpointForDestinationProvenCheckpointNumber.blockCount -
             1;
 
           this.events.emit(L2BlockSourceEvents.L2BlockProven, {
@@ -598,7 +627,7 @@ export class ArchiverL1Synchronizer implements Traceable {
             slotNumber: provenSlotNumber,
             epochNumber: provenEpochNumber,
           });
-          this.instrumentation.updateLastProvenBlock(lastBlockNumberInCheckpoint);
+          this.instrumentation.updateLastProvenCheckpoint(localCheckpointForDestinationProvenCheckpointNumber);
         } else {
           this.log.trace(`Proven checkpoint ${provenCheckpointNumber} already stored.`);
         }
@@ -675,11 +704,11 @@ export class ArchiverL1Synchronizer implements Traceable {
           tipAfterUnwind--;
         }
 
-        const checkpointsToUnwind = localPendingCheckpointNumber - tipAfterUnwind;
-        await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
+        const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
+        await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
 
         this.log.warn(
-          `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
+          `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
             `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
             `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
         );
@@ -707,7 +736,6 @@ export class ArchiverL1Synchronizer implements Traceable {
           this.blobClient,
           searchStartBlock, // TODO(palla/reorg): If the L2 reorg was due to an L1 reorg, we need to start search earlier
           searchEndBlock,
-          this.l1Addresses,
           this.instrumentation,
           this.log,
           !initialSyncComplete, // isHistoricalSync
@@ -802,12 +830,20 @@ export class ArchiverL1Synchronizer implements Traceable {
         );
       }
 
+      for (const published of validCheckpoints) {
+        this.instrumentation.processCheckpointL1Timing({
+          slotNumber: published.checkpoint.header.slotNumber,
+          l1Timestamp: published.l1.timestamp,
+          l1Constants: this.l1Constants,
+        });
+      }
+
       try {
         const updatedValidationResult =
           rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
         const [processDuration, result] = await elapsed(() =>
-          execInSpan(this.tracer, 'Archiver.setCheckpointData', () =>
-            this.updater.setNewCheckpointData(validCheckpoints, updatedValidationResult),
+          execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
+            this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
           ),
         );
         this.instrumentation.processNewBlocks(
@@ -820,7 +856,7 @@ export class ArchiverL1Synchronizer implements Traceable {
           const prunedCheckpointNumber = result.prunedBlocks[0].checkpointNumber;
           const prunedSlotNumber = result.prunedBlocks[0].header.globalVariables.slotNumber;
 
-          this.log.warn(
+          this.log.info(
             `Pruned ${result.prunedBlocks.length} mismatching blocks for checkpoint ${prunedCheckpointNumber}`,
             { prunedBlocks: result.prunedBlocks.map(b => b.toBlockInfo()), prunedSlotNumber, prunedCheckpointNumber },
           );