@aztec/archiver 0.0.0-test.1 → 0.0.1-commit.03f7ef2

package/src/archiver/config.ts

@@ -1,62 +1,31 @@
-import { type BlobSinkConfig, blobSinkConfigMapping } from '@aztec/blob-sink/client';
+import { type BlobClientConfig, blobClientConfigMapping } from '@aztec/blob-client/client/config';
+import { type L1ContractsConfig, l1ContractsConfigMappings } from '@aztec/ethereum/config';
+import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
+import { type L1ReaderConfig, l1ReaderConfigMappings } from '@aztec/ethereum/l1-reader';
 import {
-  type L1ContractAddresses,
-  type L1ContractsConfig,
-  type L1ReaderConfig,
-  l1ContractsConfigMappings,
-  l1ReaderConfigMappings,
-} from '@aztec/ethereum';
-import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
+  type ConfigMappingsType,
+  booleanConfigHelper,
+  getConfigFromMappings,
+  numberConfigHelper,
+} from '@aztec/foundation/config';
 import { type ChainConfig, chainConfigMappings } from '@aztec/stdlib/config';
+import type { ArchiverSpecificConfig } from '@aztec/stdlib/interfaces/server';
 
 /**
+ * The archiver configuration.
  * There are 2 polling intervals used in this configuration. The first is the archiver polling interval, archiverPollingIntervalMS.
  * This is the interval between successive calls to eth_blockNumber via viem.
  * Results of calls to eth_blockNumber are cached by viem with this cache being updated periodically at the interval specified by viemPollingIntervalMS.
  * As a result the maximum observed polling time for new blocks will be viemPollingIntervalMS + archiverPollingIntervalMS.
  */
-
-/**
- * The archiver configuration.
- */
-export type ArchiverConfig = {
-  /** URL for an archiver service. If set, will return an archiver client as opposed to starting a new one. */
-  archiverUrl?: string;
-
-  /** URL for an L1 consensus client */
-  l1ConsensusHostUrl?: string;
-
-  /** The polling interval in ms for retrieving new L2 blocks and encrypted logs. */
-  archiverPollingIntervalMS?: number;
-
-  /** The number of L2 blocks the archiver will attempt to download at a time. */
-  archiverBatchSize?: number;
-
-  /** The polling interval viem uses in ms */
-  viemPollingIntervalMS?: number;
-
-  /** The deployed L1 contract addresses */
-  l1Contracts: L1ContractAddresses;
-
-  /** The max number of logs that can be obtained in 1 "getPublicLogs" call. */
-  maxLogs?: number;
-} & L1ReaderConfig &
+export type ArchiverConfig = ArchiverSpecificConfig &
+  L1ReaderConfig &
   L1ContractsConfig &
-  BlobSinkConfig &
+  BlobClientConfig &
   ChainConfig;
 
 export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
-  ...blobSinkConfigMapping,
-  archiverUrl: {
-    env: 'ARCHIVER_URL',
-    description:
-      'URL for an archiver service. If set, will return an archiver client as opposed to starting a new one.',
-  },
-  l1ConsensusHostUrl: {
-    env: 'L1_CONSENSUS_HOST_URL',
-    description: 'URL for an L1 consensus client.',
-    parseEnv: (val: string) => val,
-  },
+  ...blobClientConfigMapping,
   archiverPollingIntervalMS: {
     env: 'ARCHIVER_POLLING_INTERVAL_MS',
     description: 'The polling interval in ms for retrieving new L2 blocks and encrypted logs.',
@@ -72,6 +41,25 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
     description: 'The max number of logs that can be obtained in 1 "getPublicLogs" call.',
     ...numberConfigHelper(1_000),
   },
+  archiverStoreMapSizeKb: {
+    env: 'ARCHIVER_STORE_MAP_SIZE_KB',
+    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+    description: 'The maximum possible size of the archiver DB in KB. Overwrites the general dataStoreMapSizeKb.',
+  },
+  skipValidateBlockAttestations: {
+    description: 'Whether to skip validating block attestations (use only for testing).',
+    ...booleanConfigHelper(false),
+  },
+  maxAllowedEthClientDriftSeconds: {
+    env: 'MAX_ALLOWED_ETH_CLIENT_DRIFT_SECONDS',
+    description: 'Maximum allowed drift in seconds between the Ethereum client and current time.',
+    ...numberConfigHelper(300),
+  },
+  ethereumAllowNoDebugHosts: {
+    env: 'ETHEREUM_ALLOW_NO_DEBUG_HOSTS',
+    description: 'Whether to allow starting the archiver without debug/trace method support on Ethereum hosts',
+    ...booleanConfigHelper(true),
+  },
   ...chainConfigMappings,
   ...l1ReaderConfigMappings,
   viemPollingIntervalMS: {
@@ -80,6 +68,10 @@ export const archiverConfigMappings: ConfigMappingsType<ArchiverConfig> = {
     ...numberConfigHelper(1000),
   },
   ...l1ContractsConfigMappings,
+  l1Contracts: {
+    description: 'The deployed L1 contract addresses',
+    nested: l1ContractAddressesMapping,
+  },
 };
 
 /**

package/src/archiver/errors.ts

@@ -1,5 +1,90 @@
+import type { Fr } from '@aztec/foundation/schemas';
+
 export class NoBlobBodiesFoundError extends Error {
   constructor(l2BlockNum: number) {
     super(`No blob bodies found for block ${l2BlockNum}`);
   }
 }
+
+export class InitialBlockNumberNotSequentialError extends Error {
+  constructor(
+    public readonly newBlockNumber: number,
+    public readonly previousBlockNumber: number | undefined,
+  ) {
+    super(
+      `Cannot insert new block ${newBlockNumber} given previous block number in store is ${
+        previousBlockNumber ?? 'undefined'
+      }`,
+    );
+  }
+}
+
+export class BlockNumberNotSequentialError extends Error {
+  constructor(newBlockNumber: number, previous: number | undefined) {
+    super(
+      `Cannot insert new block ${newBlockNumber} given previous block number in batch is ${previous ?? 'undefined'}`,
+    );
+  }
+}
+
+export class InitialCheckpointNumberNotSequentialError extends Error {
+  constructor(
+    public readonly newCheckpointNumber: number,
+    public readonly previousCheckpointNumber: number | undefined,
+  ) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous checkpoint number in store is ${
+        previousCheckpointNumber ?? 'undefined'
+      }`,
+    );
+  }
+}
+
+export class CheckpointNumberNotSequentialError extends Error {
+  constructor(newCheckpointNumber: number, previous: number | undefined) {
+    super(
+      `Cannot insert new checkpoint ${newCheckpointNumber} given previous checkpoint number in batch is ${previous ?? 'undefined'}`,
+    );
+  }
+}
+
+export class CheckpointNumberNotConsistentError extends Error {
+  constructor(newCheckpointNumber: number, previous: number | undefined) {
+    super(
+      `Cannot insert block for new checkpoint ${newCheckpointNumber} given previous block was checkpoint ${previous ?? 'undefined'}`,
+    );
+  }
+}
+
+export class BlockIndexNotSequentialError extends Error {
+  constructor(newBlockIndex: number, previousBlockIndex: number | undefined) {
+    super(
+      `Cannot insert new block at checkpoint index ${newBlockIndex} given previous block index is ${previousBlockIndex ?? 'undefined'}`,
+    );
+  }
+}
+
+export class BlockArchiveNotConsistentError extends Error {
+  constructor(
+    newBlockNumber: number,
+    previousBlockNumber: number | undefined,
+    newBlockArchive: Fr,
+    previousBlockArchive: Fr,
+  ) {
+    super(
+      `Cannot insert new block number ${newBlockNumber} with archive ${newBlockArchive.toString()} previous block number is ${previousBlockNumber ?? 'undefined'}, previous archive is ${previousBlockArchive?.toString() ?? 'undefined'}`,
+    );
+  }
+}
+
+export class CheckpointNotFoundError extends Error {
+  constructor(checkpointNumber: number) {
+    super(`Failed to find expected checkpoint number ${checkpointNumber}`);
+  }
+}
+
+export class BlockNotFoundError extends Error {
+  constructor(blockNumber: number) {
+    super(`Failed to find expected block number ${blockNumber}`);
+  }
+}

package/src/archiver/index.ts

@@ -1,7 +1,6 @@
 export * from './archiver.js';
 export * from './config.js';
-export { type L1Published, type L1PublishedData } from './structs/published.js';
-export { MemoryArchiverStore } from './memory_archiver_store/memory_archiver_store.js';
+export { type L1PublishedData } from './structs/published.js';
 export type { ArchiverDataStore } from './archiver_store.js';
-export { KVArchiverDataStore } from './kv_archiver_store/kv_archiver_store.js';
+export { KVArchiverDataStore, ARCHIVER_DB_VERSION } from './kv_archiver_store/kv_archiver_store.js';
 export { ContractInstanceStore } from './kv_archiver_store/contract_instance_store.js';

package/src/archiver/instrumentation.ts

@@ -1,5 +1,5 @@
 import { createLogger } from '@aztec/foundation/log';
-import type { L2Block } from '@aztec/stdlib/block';
+import type { L2BlockNew } from '@aztec/stdlib/block';
 import {
   Attributes,
   type Gauge,
@@ -18,32 +18,45 @@ export class ArchiverInstrumentation {
 
   private blockHeight: Gauge;
   private txCount: UpDownCounter;
-  private syncDuration: Histogram;
-  private l1BlocksSynced: UpDownCounter;
   private l1BlockHeight: Gauge;
   private proofsSubmittedDelay: Histogram;
   private proofsSubmittedCount: UpDownCounter;
   private dbMetrics: LmdbMetrics;
+
+  private pruneDuration: Histogram;
   private pruneCount: UpDownCounter;
 
+  private syncDurationPerBlock: Histogram;
+  private syncBlockCount: UpDownCounter;
+  private manaPerBlock: Histogram;
+  private txsPerBlock: Histogram;
+
+  private syncDurationPerMessage: Histogram;
+  private syncMessageCount: UpDownCounter;
+
+  private blockProposalTxTargetCount: UpDownCounter;
+
   private log = createLogger('archiver:instrumentation');
 
-  private constructor(private telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
+  private constructor(
+    private telemetry: TelemetryClient,
+    lmdbStats?: LmdbStatsCallback,
+  ) {
     this.tracer = telemetry.getTracer('Archiver');
     const meter = telemetry.getMeter('Archiver');
+
     this.blockHeight = meter.createGauge(Metrics.ARCHIVER_BLOCK_HEIGHT, {
       description: 'The height of the latest block processed by the archiver',
       valueType: ValueType.INT,
     });
 
-    this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TX_COUNT, {
-      description: 'The total number of transactions',
+    this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT, {
+      description: 'The height of the latest L1 block processed by the archiver',
       valueType: ValueType.INT,
     });
 
-    this.syncDuration = meter.createHistogram(Metrics.ARCHIVER_SYNC_DURATION, {
-      unit: 'ms',
-      description: 'Duration to sync a block',
+    this.txCount = meter.createUpDownCounter(Metrics.ARCHIVER_TOTAL_TXS, {
+      description: 'The total number of transactions',
       valueType: ValueType.INT,
     });
 
@@ -58,13 +71,53 @@ export class ArchiverInstrumentation {
       valueType: ValueType.INT,
     });
 
-    this.l1BlocksSynced = meter.createUpDownCounter(Metrics.ARCHIVER_L1_BLOCKS_SYNCED, {
+    this.syncDurationPerBlock = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_BLOCK, {
+      unit: 'ms',
+      description: 'Duration to sync a block',
+      valueType: ValueType.INT,
+    });
+
+    this.syncBlockCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_BLOCK_COUNT, {
       description: 'Number of blocks synced from L1',
       valueType: ValueType.INT,
     });
 
-    this.l1BlockHeight = meter.createGauge(Metrics.ARCHIVER_L1_BLOCK_HEIGHT, {
-      description: 'The height of the latest L1 block processed by the archiver',
+    this.manaPerBlock = meter.createHistogram(Metrics.ARCHIVER_MANA_PER_BLOCK, {
+      description: 'The mana consumed by blocks',
+      valueType: ValueType.DOUBLE,
+      unit: 'Mmana',
+    });
+
+    this.txsPerBlock = meter.createHistogram(Metrics.ARCHIVER_TXS_PER_BLOCK, {
+      description: 'The block tx count',
+      valueType: ValueType.INT,
+      unit: 'tx',
+    });
+
+    this.syncDurationPerMessage = meter.createHistogram(Metrics.ARCHIVER_SYNC_PER_MESSAGE, {
+      unit: 'ms',
+      description: 'Duration to sync a message',
+      valueType: ValueType.INT,
+    });
+
+    this.syncMessageCount = meter.createUpDownCounter(Metrics.ARCHIVER_SYNC_MESSAGE_COUNT, {
+      description: 'Number of L1 to L2 messages synced',
+      valueType: ValueType.INT,
+    });
+
+    this.pruneDuration = meter.createHistogram(Metrics.ARCHIVER_PRUNE_DURATION, {
+      unit: 'ms',
+      description: 'Duration to sync a message',
+      valueType: ValueType.INT,
+    });
+
+    this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT, {
+      description: 'Number of prunes detected',
+      valueType: ValueType.INT,
+    });
+
+    this.blockProposalTxTargetCount = meter.createUpDownCounter(Metrics.ARCHIVER_BLOCK_PROPOSAL_TX_TARGET_COUNT, {
+      description: 'Number of block proposals by tx target',
       valueType: ValueType.INT,
     });
 
@@ -75,17 +128,14 @@ export class ArchiverInstrumentation {
       },
       lmdbStats,
     );
-
-    this.pruneCount = meter.createUpDownCounter(Metrics.ARCHIVER_PRUNE_COUNT, {
-      description: 'Number of prunes detected',
-      valueType: ValueType.INT,
-    });
   }
 
   public static async new(telemetry: TelemetryClient, lmdbStats?: LmdbStatsCallback) {
     const instance = new ArchiverInstrumentation(telemetry, lmdbStats);
 
-    instance.l1BlocksSynced.add(0);
+    instance.syncBlockCount.add(0);
+    instance.syncMessageCount.add(0);
+    instance.pruneCount.add(0);
 
     await instance.telemetry.flush();
 
@@ -96,17 +146,29 @@ export class ArchiverInstrumentation {
     return this.telemetry.isEnabled();
   }
 
-  public processNewBlocks(syncTimePerBlock: number, blocks: L2Block[]) {
-    this.syncDuration.record(Math.ceil(syncTimePerBlock));
+  public processNewBlocks(syncTimePerBlock: number, blocks: L2BlockNew[]) {
+    this.syncDurationPerBlock.record(Math.ceil(syncTimePerBlock));
     this.blockHeight.record(Math.max(...blocks.map(b => b.number)));
-    this.l1BlocksSynced.add(blocks.length);
+    this.syncBlockCount.add(blocks.length);
+
     for (const block of blocks) {
       this.txCount.add(block.body.txEffects.length);
+      this.txsPerBlock.record(block.body.txEffects.length);
+      this.manaPerBlock.record(block.header.totalManaUsed.toNumber() / 1e6);
     }
   }
 
-  public processPrune() {
+  public processNewMessages(count: number, syncPerMessageMs: number) {
+    if (count === 0) {
+      return;
+    }
+    this.syncMessageCount.add(count);
+    this.syncDurationPerMessage.record(Math.ceil(syncPerMessageMs));
+  }
+
+  public processPrune(duration: number) {
     this.pruneCount.add(1);
+    this.pruneDuration.record(Math.ceil(duration));
   }
 
   public updateLastProvenBlock(blockNumber: number) {
@@ -129,4 +191,11 @@ export class ArchiverInstrumentation {
   public updateL1BlockHeight(blockNumber: bigint) {
     this.l1BlockHeight.record(Number(blockNumber));
   }
+
+  public recordBlockProposalTxTarget(target: string, usedTrace: boolean) {
+    this.blockProposalTxTargetCount.add(1, {
+      [Attributes.L1_BLOCK_PROPOSAL_TX_TARGET]: target.toLowerCase(),
+      [Attributes.L1_BLOCK_PROPOSAL_USED_TRACE]: usedTrace,
+    });
+  }
 }