@aztec/archiver 0.0.1-commit.d431d1c → 0.0.1-commit.e3c1de76

package/src/modules/data_store_updater.ts

@@ -1,4 +1,4 @@
-import { BlockNumber, type CheckpointNumber } from '@aztec/foundation/branded-types';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { createLogger } from '@aztec/foundation/log';
 import {
@@ -10,7 +10,7 @@ import {
   ContractInstancePublishedEvent,
   ContractInstanceUpdatedEvent,
 } from '@aztec/protocol-contracts/instance-registry';
-import type { L2BlockNew, ValidateCheckpointResult } from '@aztec/stdlib/block';
+import type { L2Block, ValidateCheckpointResult } from '@aztec/stdlib/block';
 import type { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 import {
   type ExecutablePrivateFunctionWithMembershipProof,
@@ -35,7 +35,7 @@ enum Operation {
 /** Result of adding checkpoints with information about any pruned blocks. */
 type ReconcileCheckpointsResult = {
   /** Blocks that were pruned due to conflict with L1 checkpoints. */
-  prunedBlocks: L2BlockNew[] | undefined;
+  prunedBlocks: L2Block[] | undefined;
   /** Last block number that was already inserted locally, or undefined if none. */
   lastAlreadyInsertedBlockNumber: BlockNumber | undefined;
 };
@@ -47,17 +47,21 @@ export class ArchiverDataStoreUpdater {
   constructor(private store: KVArchiverDataStore) {}
 
   /**
-   * Adds blocks to the store with contract class/instance extraction from logs.
+   * Adds proposed blocks to the store with contract class/instance extraction from logs.
+   * These are uncheckpointed blocks that have been proposed by the sequencer but not yet included in a checkpoint on L1.
    * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
    * and individually broadcasted functions from the block logs.
    *
-   * @param blocks - The L2 blocks to add.
+   * @param blocks - The proposed L2 blocks to add.
    * @param pendingChainValidationStatus - Optional validation status to set.
    * @returns True if the operation is successful.
    */
-  public addBlocks(blocks: L2BlockNew[], pendingChainValidationStatus?: ValidateCheckpointResult): Promise<boolean> {
+  public addProposedBlocks(
+    blocks: L2Block[],
+    pendingChainValidationStatus?: ValidateCheckpointResult,
+  ): Promise<boolean> {
     return this.store.transactionAsync(async () => {
-      await this.store.addBlocks(blocks);
+      await this.store.addProposedBlocks(blocks);
 
       const opResults = await Promise.all([
         // Update the pending chain validation status if provided
@@ -65,7 +69,7 @@ export class ArchiverDataStoreUpdater {
         // Add any logs emitted during the retrieved blocks
         this.store.addLogs(blocks),
         // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
-        ...blocks.map(block => this.addBlockDataToDB(block)),
+        ...blocks.map(block => this.addContractDataToDb(block)),
       ]);
 
       return opResults.every(Boolean);
@@ -74,7 +78,7 @@ export class ArchiverDataStoreUpdater {
 
   /**
    * Reconciles local blocks with incoming checkpoints from L1.
-   * Adds checkpoints to the store with contract class/instance extraction from logs.
+   * Adds new checkpoints to the store with contract class/instance extraction from logs.
    * Prunes any local blocks that conflict with checkpoint data (by comparing archive roots).
    * Extracts ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated events,
    * and individually broadcasted functions from the checkpoint block logs.
@@ -83,7 +87,7 @@ export class ArchiverDataStoreUpdater {
    * @param pendingChainValidationStatus - Optional validation status to set.
    * @returns Result with information about any pruned blocks.
    */
-  public setNewCheckpointData(
+  public addCheckpoints(
     checkpoints: PublishedCheckpoint[],
     pendingChainValidationStatus?: ValidateCheckpointResult,
   ): Promise<ReconcileCheckpointsResult> {
@@ -93,7 +97,7 @@ export class ArchiverDataStoreUpdater {
 
       await this.store.addCheckpoints(checkpoints);
 
-      // Filter out blocks that were already inserted via addBlocks() to avoid duplicating logs/contract data
+      // Filter out blocks that were already inserted via addProposedBlocks() to avoid duplicating logs/contract data
       const newBlocks = checkpoints
         .flatMap((ch: PublishedCheckpoint) => ch.checkpoint.blocks)
         .filter(b => lastAlreadyInsertedBlockNumber === undefined || b.number > lastAlreadyInsertedBlockNumber);
@@ -104,7 +108,7 @@ export class ArchiverDataStoreUpdater {
         // Add any logs emitted during the retrieved blocks
         this.store.addLogs(newBlocks),
         // Unroll all logs emitted during the retrieved blocks and extract any contract classes and instances from them
-        ...newBlocks.map(block => this.addBlockDataToDB(block)),
+        ...newBlocks.map(block => this.addContractDataToDb(block)),
       ]);
 
       return { prunedBlocks, lastAlreadyInsertedBlockNumber };
@@ -185,80 +189,80 @@ export class ArchiverDataStoreUpdater {
   }
 
   /**
-   * Removes all blocks strictly after the specified block number and cleans up associated contract data.
+   * Removes all uncheckpointed blocks strictly after the specified block number and cleans up associated contract data.
    * This handles removal of provisionally added blocks along with their contract classes/instances.
+   * Verifies that each block being removed is not part of a stored checkpoint.
    *
    * @param blockNumber - Remove all blocks with number greater than this.
    * @returns The removed blocks.
+   * @throws Error if any block to be removed is checkpointed.
    */
-  public removeBlocksAfter(blockNumber: BlockNumber): Promise<L2BlockNew[]> {
+  public removeUncheckpointedBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
     return this.store.transactionAsync(async () => {
-      // First get the blocks to be removed so we can clean up contract data
-      const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
-
-      // Clean up contract data and logs for the removed blocks
-      await Promise.all([
-        this.store.deleteLogs(removedBlocks),
-        ...removedBlocks.map(block => this.removeBlockDataFromDB(block)),
-      ]);
+      // Verify we're only removing uncheckpointed blocks
+      const lastCheckpointedBlockNumber = await this.store.getCheckpointedL2BlockNumber();
+      if (blockNumber < lastCheckpointedBlockNumber) {
+        throw new Error(
+          `Cannot remove blocks after ${blockNumber} because checkpointed blocks exist up to ${lastCheckpointedBlockNumber}`,
+        );
+      }
 
-      return removedBlocks;
+      return await this.removeBlocksAfter(blockNumber);
     });
   }
 
   /**
-   * Unwinds checkpoints from the store with reverse contract extraction.
+   * Removes all blocks strictly after the given block number along with any logs and contract data.
+   * Does not remove their checkpoints.
+   */
+  private async removeBlocksAfter(blockNumber: BlockNumber): Promise<L2Block[]> {
+    // First get the blocks to be removed so we can clean up contract data
+    const removedBlocks = await this.store.removeBlocksAfter(blockNumber);
+
+    // Clean up contract data and logs for the removed blocks
+    await Promise.all([
+      this.store.deleteLogs(removedBlocks),
+      ...removedBlocks.map(block => this.removeContractDataFromDb(block)),
+    ]);
+
+    return removedBlocks;
+  }
+
+  /**
+   * Removes all checkpoints after the given checkpoint number.
    * Deletes ContractClassPublished, ContractInstancePublished, ContractInstanceUpdated data
-   * that was stored for the unwound checkpoints.
+   * that was stored for the removed checkpoints. Also removes ALL blocks (both checkpointed
+   * and uncheckpointed) after the last block of the given checkpoint.
    *
-   * @param from - The checkpoint number to unwind from (must be the current tip).
-   * @param checkpointsToUnwind - The number of checkpoints to unwind.
+   * @param checkpointNumber - Remove all checkpoints strictly after this one.
    * @returns True if the operation is successful.
    */
-  public async unwindCheckpoints(from: CheckpointNumber, checkpointsToUnwind: number): Promise<boolean> {
-    if (checkpointsToUnwind <= 0) {
-      throw new Error(`Cannot unwind ${checkpointsToUnwind} blocks`);
-    }
-
-    const last = await this.store.getSynchedCheckpointNumber();
-    if (from != last) {
-      throw new Error(`Cannot unwind checkpoints from checkpoint ${from} when the last checkpoint is ${last}`);
-    }
-
-    const blocks = [];
-    const lastCheckpointNumber = from + checkpointsToUnwind - 1;
-    for (let checkpointNumber = from; checkpointNumber <= lastCheckpointNumber; checkpointNumber++) {
-      const blocksForCheckpoint = await this.store.getBlocksForCheckpoint(checkpointNumber);
-      if (!blocksForCheckpoint) {
-        continue;
-      }
-      blocks.push(...blocksForCheckpoint);
-    }
+  public async removeCheckpointsAfter(checkpointNumber: CheckpointNumber): Promise<boolean> {
+    const { blocksRemoved = [] } = await this.store.removeCheckpointsAfter(checkpointNumber);
 
     const opResults = await Promise.all([
       // Prune rolls back to the last proven block, which is by definition valid
       this.store.setPendingChainValidationStatus({ valid: true }),
-      // Remove contract data for all blocks being unwound
-      ...blocks.map(block => this.removeBlockDataFromDB(block)),
-      this.store.deleteLogs(blocks),
-      this.store.unwindCheckpoints(from, checkpointsToUnwind),
+      // Remove contract data for all blocks being removed
+      ...blocksRemoved.map(block => this.removeContractDataFromDb(block)),
+      this.store.deleteLogs(blocksRemoved),
     ]);
 
     return opResults.every(Boolean);
   }
 
   /** Extracts and stores contract data from a single block. */
-  private addBlockDataToDB(block: L2BlockNew): Promise<boolean> {
-    return this.editContractBlockData(block, Operation.Store);
+  private addContractDataToDb(block: L2Block): Promise<boolean> {
+    return this.updateContractDataOnDb(block, Operation.Store);
   }
 
   /** Removes contract data associated with a block. */
-  private removeBlockDataFromDB(block: L2BlockNew): Promise<boolean> {
-    return this.editContractBlockData(block, Operation.Delete);
+  private removeContractDataFromDb(block: L2Block): Promise<boolean> {
+    return this.updateContractDataOnDb(block, Operation.Delete);
   }
 
   /** Adds or remove contract data associated with a block. */
-  private async editContractBlockData(block: L2BlockNew, operation: Operation): Promise<boolean> {
+  private async updateContractDataOnDb(block: L2Block, operation: Operation): Promise<boolean> {
     const contractClassLogs = block.body.txEffects.flatMap(txEffect => txEffect.contractClassLogs);
     const privateLogs = block.body.txEffects.flatMap(txEffect => txEffect.privateLogs);
     const publicLogs = block.body.txEffects.flatMap(txEffect => txEffect.publicLogs);

package/src/modules/instrumentation.ts

@@ -1,5 +1,5 @@
 import { createLogger } from '@aztec/foundation/log';
-import type { L2BlockNew } from '@aztec/stdlib/block';
+import type { L2Block } from '@aztec/stdlib/block';
 import {
   Attributes,
   type Gauge,
@@ -10,6 +10,7 @@ import {
   type TelemetryClient,
   type Tracer,
   type UpDownCounter,
+  createUpDownCounterWithDefault,
 } from '@aztec/telemetry-client';
 
 export class ArchiverInstrumentation {
@@ -48,15 +49,17 @@ export class ArchiverInstrumentation {
 
     this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT);
 
-    this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS);
+    this.txCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_TOTAL_TXS);
 
-    this.proofsSubmittedCount = meter.createUpDownCounter(Metrics.ARCHIVER_ROLLUP_PROOF_COUNT);
+    this.proofsSubmittedCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_ROLLUP_PROOF_COUNT, {
+      [Attributes.PROOF_TIMED_OUT]: [true, false],
+    });
 
     this.proofsSubmittedDelay = meter.createHistogram(Metrics.ARCHIVER_ROLLUP_PROOF_DELAY);
 
     this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK);
 
-    this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
+    this.syncBlockCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_BLOCK_COUNT);
 
     this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK);
 
@@ -64,13 +67,19 @@ export class ArchiverInstrumentation {
 
     this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE);
 
-    this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
+    this.syncMessageCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_SYNC_MESSAGE_COUNT);
 
     this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION);
 
-    this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT);
+    this.pruneCount = createUpDownCounterWithDefault(meter, Metrics.ARCHIVER_PRUNE_COUNT);
 
-    this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT);
+    this.blockProposalTxTargetCount = createUpDownCounterWithDefault(
+      meter,
+      Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT,
+      {
+        [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: [true, false],
+      },
+    );
 
     this.dbMetrics = new LmdbMetrics(
       meter,
@@ -84,10 +93,6 @@ export class ArchiverInstrumentation {
   public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
     const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
 
-    instance.syncBlockCount.add(0);
-    instance.syncMessageCount.add(0);
-    instance.pruneCount.add(0);
-
     await instance.telemetry.flush();
 
     return instance;
@@ -97,7 +102,7 @@ export class ArchiverInstrumentation {
     return this.telemetry.isEnabled();
   }
 
-  public processNewBlocks(syncTimePerBlock: number, blocks: L2BlockNew[]) {
+  public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
     this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
     this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
     this.syncBlockCount.add(blocks.length);

package/src/modules/l1_synchronizer.ts

@@ -16,7 +16,7 @@ import { DateProvider, Timer, elapsed } from '@aztec/foundation/timer';
 import { isDefined } from '@aztec/foundation/types';
 import { type ArchiverEmitter, L2BlockSourceEvents, type ValidateCheckpointResult } from '@aztec/stdlib/block';
 import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
-import { type L1RollupConstants, getEpochAtSlot, getSlotAtTimestamp } from '@aztec/stdlib/epoch-helpers';
+import { type L1RollupConstants, getEpochAtSlot, getSlotAtNextL1Block } from '@aztec/stdlib/epoch-helpers';
 import { computeInHashFromL1ToL2Messages } from '@aztec/stdlib/messaging';
 import { type Traceable, type Tracer, execInSpan, trackSpan } from '@aztec/telemetry-client';
 
@@ -249,8 +249,7 @@ export class ArchiverL1Synchronizer implements Traceable {
     const firstUncheckpointedBlockSlot = firstUncheckpointedBlockHeader?.getSlot();
 
     // What's the slot at the next L1 block? All blocks for slots strictly before this one should've been checkpointed by now.
-    const nextL1BlockTimestamp = currentL1Timestamp + BigInt(this.l1Constants.ethereumSlotDuration);
-    const slotAtNextL1Block = getSlotAtTimestamp(nextL1BlockTimestamp, this.l1Constants);
+    const slotAtNextL1Block = getSlotAtNextL1Block(currentL1Timestamp, this.l1Constants);
 
     // Prune provisional blocks from slots that have ended without being checkpointed
     if (firstUncheckpointedBlockSlot !== undefined && firstUncheckpointedBlockSlot < slotAtNextL1Block) {
@@ -258,7 +257,7 @@ export class ArchiverL1Synchronizer implements Traceable {
         `Pruning blocks after block ${lastCheckpointedBlockNumber} due to slot ${firstUncheckpointedBlockSlot} not being checkpointed`,
         { firstUncheckpointedBlockHeader: firstUncheckpointedBlockHeader.toInspect(), slotAtNextL1Block },
       );
-      const prunedBlocks = await this.updater.removeBlocksAfter(lastCheckpointedBlockNumber);
+      const prunedBlocks = await this.updater.removeUncheckpointedBlocksAfter(lastCheckpointedBlockNumber);
 
       if (prunedBlocks.length > 0) {
         this.events.emit(L2BlockSourceEvents.L2PruneUncheckpointed, {
@@ -331,10 +330,10 @@ export class ArchiverL1Synchronizer implements Traceable {
       this.log.debug(
         `L2 prune from ${provenCheckpointNumber + 1} to ${localPendingCheckpointNumber} will occur on next checkpoint submission.`,
       );
-      await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
+      await this.updater.removeCheckpointsAfter(provenCheckpointNumber);
       this.log.warn(
-        `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
-          `to ${provenCheckpointNumber} due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
+        `Removed ${count(checkpointsToUnwind, 'checkpoint')} after checkpoint ${provenCheckpointNumber} ` +
+          `due to predicted reorg at L1 block ${currentL1BlockNumber}. ` +
           `Updated latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
       );
       this.instrumentation.processPrune(timer.ms());
@@ -675,11 +674,11 @@ export class ArchiverL1Synchronizer implements Traceable {
           tipAfterUnwind--;
         }
 
-        const checkpointsToUnwind = localPendingCheckpointNumber - tipAfterUnwind;
-        await this.updater.unwindCheckpoints(localPendingCheckpointNumber, checkpointsToUnwind);
+        const checkpointsToRemove = localPendingCheckpointNumber - tipAfterUnwind;
+        await this.updater.removeCheckpointsAfter(CheckpointNumber(tipAfterUnwind));
 
         this.log.warn(
-          `Unwound ${count(checkpointsToUnwind, 'checkpoint')} from checkpoint ${localPendingCheckpointNumber} ` +
+          `Removed ${count(checkpointsToRemove, 'checkpoint')} after checkpoint ${tipAfterUnwind} ` +
             `due to mismatched checkpoint hashes at L1 block ${currentL1BlockNumber}. ` +
             `Updated L2 latest checkpoint is ${await this.store.getSynchedCheckpointNumber()}.`,
         );
@@ -806,8 +805,8 @@ export class ArchiverL1Synchronizer implements Traceable {
         const updatedValidationResult =
           rollupStatus.validationResult === initialValidationResult ? undefined : rollupStatus.validationResult;
         const [processDuration, result] = await elapsed(() =>
-          execInSpan(this.tracer, 'Archiver.setCheckpointData', () =>
-            this.updater.setNewCheckpointData(validCheckpoints, updatedValidationResult),
+          execInSpan(this.tracer, 'Archiver.addCheckpoints', () =>
+            this.updater.addCheckpoints(validCheckpoints, updatedValidationResult),
           ),
         );
         this.instrumentation.processNewBlocks(