@aztec/stdlib 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97

package/src/block/l2_block_stream/l2_block_stream.ts

@@ -1,10 +1,15 @@
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
 import { AbortError } from '@aztec/foundation/error';
 import { createLogger } from '@aztec/foundation/log';
 import { RunningPromise } from '@aztec/foundation/running-promise';
 
+import type { PublishedCheckpoint } from '../../checkpoint/published_checkpoint.js';
 import { type L2BlockId, type L2BlockSource, makeL2BlockId } from '../l2_block_source.js';
 import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider } from './interfaces.js';
 
+/** Maximum number of checkpoints to prefetch at once during sync. Matches MAX_RPC_CHECKPOINTS_LEN. */
+export const CHECKPOINT_PREFETCH_LIMIT = 50;
+
 /** Creates a stream of events for new blocks, chain tips updates, and reorgs, out of polling an archiver or a node. */
 export class L2BlockStream {
   private readonly runningPromise: RunningPromise;
@@ -12,19 +17,26 @@ export class L2BlockStream {
   private hasStarted = false;
 
   constructor(
-    private l2BlockSource: Pick<L2BlockSource, 'getPublishedBlocks' | 'getBlockHeader' | 'getL2Tips'>,
+    private l2BlockSource: Pick<L2BlockSource, 'getBlocks' | 'getBlockData' | 'getL2Tips' | 'getCheckpoints'>,
     private localData: L2BlockStreamLocalDataProvider,
     private handler: L2BlockStreamEventHandler,
     private readonly log = createLogger('types:block_stream'),
     private opts: {
-      proven?: boolean;
       pollIntervalMS?: number;
       batchSize?: number;
       startingBlock?: number;
       /** Instead of downloading all blocks, only fetch the smallest subset that results in reliable reorg detection. */
       skipFinalized?: boolean;
+      /** When true, checkpoint events will not be emitted. Blocks are still fetched via checkpoints but only blocks-added events are emitted. */
+      ignoreCheckpoints?: boolean;
+      /** Maximum number of checkpoints to prefetch at once during sync. Defaults to CHECKPOINT_PREFETCH_LIMIT (50). */
+      checkpointPrefetchLimit?: number;
     } = {},
   ) {
+    // Note that RunningPromise is in stopped state by default. This promise won't run until someone invokes `start`,
+    // which makes it run periodically, or `sync`, which triggers it once.
+    // Users of L2BlockStream decide what mode to run it in (_periodically_ vs _manually triggered_).
+    // The default is _manually triggered_.
     this.runningPromise = new RunningPromise(() => this.work(), log, this.opts.pollIntervalMS ?? 1000);
   }
 
@@ -41,6 +53,11 @@ export class L2BlockStream {
     return this.runningPromise.isRunning();
   }
 
+  /**
+   * Runs the synchronization process once.
+   *
+   * If you want to run this process continuously use `start` and `stop` instead.
+   */
   public async sync() {
     this.isSyncing = true;
     await this.runningPromise.trigger();
@@ -51,41 +68,51 @@ export class L2BlockStream {
     try {
       const sourceTips = await this.l2BlockSource.getL2Tips();
       const localTips = await this.localData.getL2Tips();
-      this.log.trace(`Running L2 block stream`, {
-        sourceLatest: sourceTips.latest.number,
-        localLatest: localTips.latest.number,
-        sourceFinalized: sourceTips.finalized.number,
-        localFinalized: localTips.finalized.number,
-        sourceProven: sourceTips.proven.number,
-        localProven: localTips.proven.number,
-        sourceLatestHash: sourceTips.latest.hash,
-        localLatestHash: localTips.latest.hash,
-        sourceProvenHash: sourceTips.proven.hash,
-        localProvenHash: localTips.proven.hash,
-        sourceFinalizedHash: sourceTips.finalized.hash,
-        localFinalizedHash: localTips.finalized.hash,
-      });
+      this.log.trace(`Running L2 block stream`, { sourceTips, localTips });
 
       // Check if there was a reorg and emit a chain-pruned event if so.
-      let latestBlockNumber = localTips.latest.number;
-      const sourceCache = new BlockHashCache([sourceTips.latest]);
+      let latestBlockNumber = localTips.proposed.number;
+      const sourceCache = new BlockHashCache([sourceTips.proposed]);
       while (!(await this.areBlockHashesEqualAt(latestBlockNumber, { sourceCache }))) {
+        if (latestBlockNumber === 0) {
+          // We walked all the way back to genesis and the hashes still differ. This means the
+          // local store and the source disagree on the genesis block itself — typically because
+          // they were configured with different `genesisTimestamp`/prefilled state. Continuing
+          // would underflow into negative block numbers and surface as "block hash not found
+          // for -1" further down. Fail loudly with a meaningful error instead.
+          this.log.error(`Genesis block hash mismatch between local store and source`, {
+            localBlockHash: await this.localData.getL2BlockHash(BlockNumber.ZERO),
+            sourceBlockHash: sourceCache.get(0) ?? (await this.getBlockHashFromSource(BlockNumber.ZERO)),
+          });
+          throw new Error(
+            'Genesis block hash mismatch between local store and source: refusing to walk past block 0. ' +
+              'This usually indicates the two sides were configured with different genesis values ' +
+              '(e.g. genesisTimestamp or prefilled public data).',
+          );
+        }
         latestBlockNumber--;
       }
 
-      if (latestBlockNumber < localTips.latest.number) {
-        latestBlockNumber = Math.min(latestBlockNumber, sourceTips.latest.number); // see #13471
+      if (latestBlockNumber < localTips.proposed.number) {
+        latestBlockNumber = BlockNumber(Math.min(latestBlockNumber, sourceTips.proposed.number)); // see #13471
         const hash = sourceCache.get(latestBlockNumber) ?? (await this.getBlockHashFromSource(latestBlockNumber));
         if (latestBlockNumber !== 0 && !hash) {
           throw new Error(`Block hash not found in block source for block number ${latestBlockNumber}`);
         }
-        this.log.verbose(`Reorg detected. Pruning blocks from ${latestBlockNumber + 1} to ${localTips.latest.number}.`);
-        await this.emitEvent({ type: 'chain-pruned', block: makeL2BlockId(latestBlockNumber, hash) });
+        this.log.verbose(
+          `Reorg detected. Pruning blocks from ${latestBlockNumber + 1} to ${localTips.proposed.number}.`,
+        );
+        await this.emitEvent({
+          type: 'chain-pruned',
+          block: makeL2BlockId(latestBlockNumber, hash),
+          checkpoint: sourceTips.checkpointed.checkpoint,
+        });
       }
 
       // If we are just starting, use the starting block number from the options.
-      if (latestBlockNumber === 0 && this.opts.startingBlock !== undefined) {
-        latestBlockNumber = Math.max(this.opts.startingBlock - 1, 0);
+      const startingBlock = this.opts.startingBlock !== undefined ? BlockNumber(this.opts.startingBlock) : undefined;
+      if (latestBlockNumber === 0 && startingBlock !== undefined) {
+        latestBlockNumber = BlockNumber(Math.max(startingBlock - 1, 0));
       }
 
       // Only log this entry once (for sanity)
@@ -95,36 +122,149 @@ export class L2BlockStream {
       }
 
       let nextBlockNumber = latestBlockNumber + 1;
+      let nextCheckpointToEmit = CheckpointNumber(localTips.checkpointed.checkpoint.number + 1);
+
+      // When startingBlock is set, also skip ahead for checkpoints.
+      if (
+        startingBlock !== undefined &&
+        startingBlock >= 1 &&
+        nextCheckpointToEmit <= sourceTips.checkpointed.checkpoint.number
+      ) {
+        if (startingBlock > sourceTips.checkpointed.block.number) {
+          // startingBlock is past all checkpointed blocks; skip Loop 1 entirely.
+          nextCheckpointToEmit = CheckpointNumber(sourceTips.checkpointed.checkpoint.number + 1);
+        } else {
+          const startingBlockData = await this.l2BlockSource.getBlockData({ number: startingBlock });
+          if (startingBlockData) {
+            nextCheckpointToEmit = CheckpointNumber(Math.max(nextCheckpointToEmit, startingBlockData.checkpointNumber));
+          }
+        }
+      }
+
       if (this.opts.skipFinalized) {
         // When skipping finalized blocks we need to provide reliable reorg detection while fetching as few blocks as
         // possible. Finalized blocks cannot be reorged by definition, so we can skip most of them. We do need the very
         // last finalized block however in order to guarantee that we will eventually find a block in which our local
         // store matches the source.
         // If the last finalized block is behind our local tip, there is nothing to skip.
-        nextBlockNumber = Math.max(sourceTips.finalized.number, nextBlockNumber);
+        nextBlockNumber = Math.max(sourceTips.finalized.block.number, nextBlockNumber);
+        // If the next checkpoint to emit is behind the finalized tip then skip forward
+        nextCheckpointToEmit = CheckpointNumber(Math.max(nextCheckpointToEmit, sourceTips.finalized.checkpoint.number));
       }
 
-      // Request new blocks from the source.
-      while (nextBlockNumber <= sourceTips.latest.number) {
-        const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.latest.number - nextBlockNumber + 1);
-        this.log.trace(`Requesting blocks from ${nextBlockNumber} limit ${limit} proven=${this.opts.proven}`);
-        const blocks = await this.l2BlockSource.getPublishedBlocks(nextBlockNumber, limit, this.opts.proven);
+      // Loop 1: Emit checkpoint events for checkpoints whose blocks are already in local storage.
+      // This handles the case where blocks were synced as uncheckpointed and later became checkpointed.
+      // The guard `lastBlockInCheckpoint.number > localTips.proposed.number` ensures we don't emit
+      // checkpoints for blocks we don't have (e.g., when startingBlock skips earlier blocks).
+      // Since only one checkpoint can ever be uncheckpointed, this loop should iterate at most once.
+      if (!this.opts.ignoreCheckpoints) {
+        let loop1Iterations = 0;
+        while (nextCheckpointToEmit <= sourceTips.checkpointed.checkpoint.number) {
+          const checkpoints = await this.l2BlockSource.getCheckpoints({ from: nextCheckpointToEmit, limit: 1 });
+          if (checkpoints.length === 0) {
+            break;
+          }
+          const lastBlockInCheckpoint = checkpoints[0].checkpoint.blocks.at(-1)!;
+          // If this checkpoint has blocks we haven't seen yet, stop - they need to be fetched first
+          if (lastBlockInCheckpoint.number > localTips.proposed.number) {
+            break;
+          }
+          loop1Iterations++;
+          if (loop1Iterations > 1) {
+            this.log.warn(
+              `Emitting multiple checkpoints (${loop1Iterations}) for already-local blocks. ` +
+                `Next checkpoint: ${nextCheckpointToEmit}, source checkpointed: ${sourceTips.checkpointed.checkpoint.number}`,
+            );
+          }
+          const lastBlockHash = await lastBlockInCheckpoint.hash();
+          await this.emitEvent({
+            type: 'chain-checkpointed',
+            checkpoint: checkpoints[0],
+            block: makeL2BlockId(lastBlockInCheckpoint.number, lastBlockHash.toString()),
+          });
+          nextCheckpointToEmit = CheckpointNumber(nextCheckpointToEmit + 1);
+        }
+      }
+
+      // Loop 2: Fetch new checkpointed blocks. For each checkpoint, emit all blocks
+      // from that checkpoint that we need, then emit the checkpoint event.
+      // We prefetch multiple checkpoints, then process them one by one.
+      let prefetchedCheckpoints: PublishedCheckpoint[] = [];
+      let prefetchIdx = 0;
+      let nextCheckpointNumber: CheckpointNumber | undefined;
+
+      // Find the starting checkpoint number
+      if (nextBlockNumber <= sourceTips.checkpointed.block.number) {
+        const blockData = await this.l2BlockSource.getBlockData({ number: BlockNumber(nextBlockNumber) });
+        if (blockData) {
+          nextCheckpointNumber = blockData.checkpointNumber;
+        }
+      }
+
+      while (nextBlockNumber <= sourceTips.checkpointed.block.number && nextCheckpointNumber !== undefined) {
+        // Refill the prefetch buffer when exhausted
+        if (prefetchIdx >= prefetchedCheckpoints.length) {
+          const prefetchLimit = this.opts.checkpointPrefetchLimit ?? CHECKPOINT_PREFETCH_LIMIT;
+          prefetchedCheckpoints = await this.l2BlockSource.getCheckpoints({
+            from: nextCheckpointNumber,
+            limit: prefetchLimit,
+          });
+          prefetchIdx = 0;
+          if (prefetchedCheckpoints.length === 0) {
+            break;
+          }
+        }
+
+        const checkpoint = prefetchedCheckpoints[prefetchIdx]!;
+
+        // Get all blocks from this checkpoint that we need, respecting batchSize
+        const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.checkpointed.block.number - nextBlockNumber + 1);
+        const blocksForCheckpoint = checkpoint.checkpoint.blocks
+          .filter(b => b.number >= nextBlockNumber)
+          .slice(0, limit);
+        if (blocksForCheckpoint.length === 0) {
+          break;
+        }
+        await this.emitEvent({ type: 'blocks-added', blocks: blocksForCheckpoint });
+        nextBlockNumber = blocksForCheckpoint.at(-1)!.number + 1;
+
+        // If we've reached the end of this checkpoint, emit the checkpoint event and move to next
+        const lastBlockInCheckpoint = checkpoint.checkpoint.blocks.at(-1)!;
+        if (nextBlockNumber > lastBlockInCheckpoint.number) {
+          if (!this.opts.ignoreCheckpoints) {
+            const lastBlockHash = await lastBlockInCheckpoint.hash();
+            await this.emitEvent({
+              type: 'chain-checkpointed',
+              checkpoint,
+              block: makeL2BlockId(lastBlockInCheckpoint.number, lastBlockHash.toString()),
+            });
+          }
+          prefetchIdx++;
+          nextCheckpointNumber = CheckpointNumber(nextCheckpointNumber + 1);
+        }
+      }
+
+      // Loop 3: Fetch any remaining uncheckpointed (proposed) blocks.
+      while (nextBlockNumber <= sourceTips.proposed.number) {
+        const limit = Math.min(this.opts.batchSize ?? 50, sourceTips.proposed.number - nextBlockNumber + 1);
+        this.log.trace(`Requesting blocks from ${nextBlockNumber} limit ${limit}`);
+        const blocks = await this.l2BlockSource.getBlocks({ from: BlockNumber(nextBlockNumber), limit });
         if (blocks.length === 0) {
           break;
         }
         await this.emitEvent({ type: 'blocks-added', blocks });
-        nextBlockNumber = blocks.at(-1)!.block.number + 1;
+        nextBlockNumber = blocks.at(-1)!.number + 1;
       }
 
       // Update the proven and finalized tips.
-      if (localTips.proven !== undefined && sourceTips.proven.number !== localTips.proven.number) {
+      if (localTips.proven !== undefined && sourceTips.proven.block.number !== localTips.proven.block.number) {
         await this.emitEvent({
           type: 'chain-proven',
-          block: sourceTips.proven,
+          block: sourceTips.proven.block,
         });
       }
-      if (localTips.finalized !== undefined && sourceTips.finalized.number !== localTips.finalized.number) {
-        await this.emitEvent({ type: 'chain-finalized', block: sourceTips.finalized });
+      if (localTips.finalized !== undefined && sourceTips.finalized.block.number !== localTips.finalized.block.number) {
+        await this.emitEvent({ type: 'chain-finalized', block: sourceTips.finalized.block });
       }
     } catch (err: any) {
       if (err.name === 'AbortError') {
@@ -139,10 +279,7 @@ export class L2BlockStream {
    * @param blockNumber - The block number to test.
    * @param args - A cache of data already requested from source, to avoid re-requesting it.
    */
-  private async areBlockHashesEqualAt(blockNumber: number, args: { sourceCache: BlockHashCache }) {
-    if (blockNumber === 0) {
-      return true;
-    }
+  private async areBlockHashesEqualAt(blockNumber: BlockNumber, args: { sourceCache: BlockHashCache }) {
     const localBlockHash = await this.localData.getL2BlockHash(blockNumber);
     if (!localBlockHash && this.opts.skipFinalized) {
       // Failing to find a block hash when skipping finalized blocks can be highly problematic as we'd potentially need
@@ -163,16 +300,16 @@ export class L2BlockStream {
     return localBlockHash === sourceBlockHash;
   }
 
-  private getBlockHashFromSource(blockNumber: number) {
+  private getBlockHashFromSource(blockNumber: BlockNumber) {
     return this.l2BlockSource
-      .getBlockHeader(blockNumber)
-      .then(h => h?.hash())
+      .getBlockData({ number: blockNumber })
+      .then(d => d?.header.hash())
       .then(hash => hash?.toString());
   }
 
   private async emitEvent(event: L2BlockStreamEvent) {
     this.log.debug(
-      `Emitting ${event.type} (${event.type === 'blocks-added' ? event.blocks.length : event.block.number})`,
+      `Emitting ${event.type} (${event.type === 'blocks-added' ? event.blocks.length : event.type === 'chain-checkpointed' ? event.checkpoint.checkpoint.number : event.block.number})`,
     );
     await this.handler.handleBlockStreamEvent(event);
     if (!this.isRunning() && !this.isSyncing) {

package/src/block/l2_block_stream/l2_tips_memory_store.ts

@@ -1,75 +1,89 @@
-import type { L2Block } from '../l2_block.js';
-import type { L2BlockId, L2BlockTag, L2Tips } from '../l2_block_source.js';
-import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider } from './interfaces.js';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+
+import type { PublishedCheckpoint } from '../../checkpoint/published_checkpoint.js';
+import type { BlockHash } from '../block_hash.js';
+import type { L2BlockTag } from '../l2_block_source.js';
+import { L2TipsStoreBase } from './l2_tips_store_base.js';
 
 /**
- * Stores currently synced L2 tips and unfinalized block hashes.
- * @dev tests in kv-store/src/stores/l2_tips_memory_store.test.ts
+ * In-memory implementation of L2 tips store. Useful for testing and lightweight clients.
+ * @dev Tests in kv-store/src/stores/l2_tips_memory_store.test.ts
  */
-export class L2TipsMemoryStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
-  protected readonly l2TipsStore: Map<L2BlockTag, number> = new Map();
-  protected readonly l2BlockHashesStore: Map<number, string> = new Map();
+export class L2TipsMemoryStore extends L2TipsStoreBase {
+  constructor(initialBlockHash: BlockHash) {
+    super(initialBlockHash);
+  }
+
+  private readonly tips = new Map<L2BlockTag, BlockNumber>();
+  private readonly blockHashes = new Map<number, string>();
+  private readonly blockToCheckpoint = new Map<number, CheckpointNumber>();
+  private readonly checkpoints = new Map<number, PublishedCheckpoint>();
 
-  public getL2BlockHash(number: number): Promise<string | undefined> {
-    return Promise.resolve(this.l2BlockHashesStore.get(number));
+  protected getTip(tag: L2BlockTag): Promise<BlockNumber | undefined> {
+    return Promise.resolve(this.tips.get(tag));
   }
 
-  public getL2Tips(): Promise<L2Tips> {
-    return Promise.resolve({
-      latest: this.getL2Tip('latest'),
-      finalized: this.getL2Tip('finalized'),
-      proven: this.getL2Tip('proven'),
-    });
+  protected setTip(tag: L2BlockTag, blockNumber: BlockNumber): Promise<void> {
+    this.tips.set(tag, blockNumber);
+    return Promise.resolve();
   }
 
-  private getL2Tip(tag: L2BlockTag): L2BlockId {
-    const blockNumber = this.l2TipsStore.get(tag);
-    if (blockNumber === undefined || blockNumber === 0) {
-      return { number: 0, hash: undefined };
-    }
-    const blockHash = this.l2BlockHashesStore.get(blockNumber);
-    if (!blockHash) {
-      throw new Error(`Block hash not found for block number ${blockNumber}`);
+  protected getStoredBlockHash(blockNumber: BlockNumber): Promise<string | undefined> {
+    return Promise.resolve(this.blockHashes.get(blockNumber));
+  }
+
+  protected setBlockHash(blockNumber: BlockNumber, hash: string): Promise<void> {
+    this.blockHashes.set(blockNumber, hash);
+    return Promise.resolve();
+  }
+
+  protected deleteBlockHashesBefore(blockNumber: BlockNumber): Promise<void> {
+    for (const key of this.blockHashes.keys()) {
+      if (key < blockNumber) {
+        this.blockHashes.delete(key);
+      }
     }
+    return Promise.resolve();
+  }
+
+  protected getCheckpointNumberForBlock(blockNumber: BlockNumber): Promise<CheckpointNumber | undefined> {
+    return Promise.resolve(this.blockToCheckpoint.get(blockNumber));
+  }
 
-    return { number: blockNumber, hash: blockHash };
+  protected setCheckpointNumberForBlock(blockNumber: BlockNumber, checkpointNumber: CheckpointNumber): Promise<void> {
+    this.blockToCheckpoint.set(blockNumber, checkpointNumber);
+    return Promise.resolve();
   }
 
-  public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
-    switch (event.type) {
-      case 'blocks-added': {
-        const blocks = event.blocks.map(b => b.block);
-        for (const block of blocks) {
-          this.l2BlockHashesStore.set(block.number, await this.computeBlockHash(block));
-        }
-        this.l2TipsStore.set('latest', blocks.at(-1)!.number);
-        break;
+  protected deleteBlockToCheckpointBefore(blockNumber: BlockNumber): Promise<void> {
+    for (const key of this.blockToCheckpoint.keys()) {
+      if (key < blockNumber) {
+        this.blockToCheckpoint.delete(key);
       }
-      case 'chain-pruned':
-        this.saveTag('latest', event.block);
-        break;
-      case 'chain-proven':
-        this.saveTag('proven', event.block);
-        break;
-      case 'chain-finalized':
-        this.saveTag('finalized', event.block);
-        for (const key of this.l2BlockHashesStore.keys()) {
-          if (key < event.block.number) {
-            this.l2BlockHashesStore.delete(key);
-          }
-        }
-        break;
     }
+    return Promise.resolve();
+  }
+
+  protected getCheckpoint(checkpointNumber: CheckpointNumber): Promise<PublishedCheckpoint | undefined> {
+    return Promise.resolve(this.checkpoints.get(checkpointNumber));
+  }
+
+  protected saveCheckpointData(checkpoint: PublishedCheckpoint): Promise<void> {
+    this.checkpoints.set(checkpoint.checkpoint.number, checkpoint);
+    return Promise.resolve();
   }
 
-  protected saveTag(name: L2BlockTag, block: L2BlockId) {
-    this.l2TipsStore.set(name, block.number);
-    if (block.hash) {
-      this.l2BlockHashesStore.set(block.number, block.hash);
+  protected deleteCheckpointsBefore(checkpointNumber: CheckpointNumber): Promise<void> {
+    for (const key of this.checkpoints.keys()) {
+      if (key < checkpointNumber) {
+        this.checkpoints.delete(key);
+      }
     }
+    return Promise.resolve();
   }
 
-  protected computeBlockHash(block: L2Block) {
-    return block.hash().then(hash => hash.toString());
+  protected runInTransaction<T>(fn: () => Promise<T>): Promise<T> {
+    // Memory store doesn't need transactions - just execute immediately
+    return fn();
   }
 }