@aztec/world-state 0.0.0-test.1 → 0.0.1-commit.017a351

package/src/native/native_world_state.ts

@@ -1,42 +1,48 @@
 import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
-import { padArrayEnd } from '@aztec/foundation/collection';
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { Fr } from '@aztec/foundation/fields';
-import { createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import type { L2Block } from '@aztec/stdlib/block';
-import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
+import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
 import type {
   IndexedTreeId,
   MerkleTreeReadOperations,
   MerkleTreeWriteOperations,
 } from '@aztec/stdlib/interfaces/server';
+import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
 import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
-import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
+import { BlockHeader, GlobalVariables, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
+import { EMPTY_GENESIS_DATA, type GenesisData, WorldStateRevision } from '@aztec/stdlib/world-state';
 import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert/strict';
-import { mkdtemp, rm } from 'fs/promises';
+import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
 import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
+import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
 import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
+import { IpcWorldState } from './ipc_world_state_instance.js';
 import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
 import {
   WorldStateMessageType,
   type WorldStateStatusFull,
   type WorldStateStatusSummary,
   blockStateReference,
-  sanitiseFullStatus,
-  sanitiseSummary,
+  sanitizeFullStatus,
+  sanitizeSummary,
   treeStateReferenceToSnapshot,
-  worldStateRevision,
 } from './message.js';
-import { NativeWorldState } from './native_world_state_instance.js';
+import type { NativeWorldStateInstance } from './native_world_state_instance.js';
 
 // The current version of the world state database schema
 // Increment this when making incompatible changes to the database schema
-export const WORLD_STATE_DB_VERSION = 1; // The initial version
+export const WORLD_STATE_DB_VERSION = 2; // The initial version
+
+export const WORLD_STATE_DIR = 'world_state';
 
 export class NativeWorldStateService implements MerkleTreeDatabase {
   protected initialHeader: BlockHeader | undefined;
@@ -44,38 +50,53 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   private cachedStatusSummary: WorldStateStatusSummary | undefined;
 
   protected constructor(
-    protected readonly instance: NativeWorldState,
+    protected instance: NativeWorldStateInstance,
     protected readonly worldStateInstrumentation: WorldStateInstrumentation,
-    protected readonly log = createLogger('world-state:database'),
+    protected readonly log: Logger,
+    private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
     private readonly cleanup = () => Promise.resolve(),
+    /** Factory to recreate a fresh IpcWorldState after clear(). */
+    private readonly recreateInstance?: () => Promise<NativeWorldStateInstance>,
   ) {}
 
   static async new(
     rollupAddress: EthAddress,
     dataDir: string,
-    dbMapSizeKb: number,
-    prefilledPublicData: PublicDataTreeLeaf[] = [],
+    wsTreeMapSizes: WorldStateTreeMapSizes,
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
-    log = createLogger('world-state:database'),
+    bindings?: LoggerBindings,
     cleanup = () => Promise.resolve(),
   ): Promise<NativeWorldStateService> {
-    const worldStateDirectory = join(dataDir, 'world_state');
-    // Create a version manager to handle versioning
-    const versionManager = new DatabaseVersionManager(
-      WORLD_STATE_DB_VERSION,
+    for (const [key, value] of Object.entries(wsTreeMapSizes)) {
+      if (value <= 0) {
+        throw new Error(`Map size must be a positive number, got ${value} for ${key}`);
+      }
+    }
+
+    const log = createLogger('world-state:database', bindings);
+    const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
+
+    const versionManager = new DatabaseVersionManager({
+      schemaVersion: WORLD_STATE_DB_VERSION,
       rollupAddress,
-      worldStateDirectory,
-      (dir: string) => {
-        return Promise.resolve(new NativeWorldState(dir, dbMapSizeKb, prefilledPublicData, instrumentation));
-      },
-    );
+      dataDirectory: worldStateDirectory,
+      onOpen: dir => IpcWorldState.spawn(dir, wsTreeMapSizes, genesis, instrumentation, bindings),
+    });
 
     const [instance] = await versionManager.open();
-    const worldState = new this(instance, instrumentation, log, cleanup);
+
+    const recreateInstance = async () => {
+      await rm(worldStateDirectory, { recursive: true, force: true, maxRetries: 3 });
+      await mkdir(worldStateDirectory, { recursive: true });
+      return IpcWorldState.spawn(worldStateDirectory, wsTreeMapSizes, genesis, instrumentation, bindings);
+    };
+
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
     try {
       await worldState.init();
     } catch (e) {
-      log.error(`Error initialising world state: ${e}`);
+      log.error(`Error initializing world state: ${e}`);
       throw e;
     }
 
@@ -83,17 +104,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   static async tmp(
-    rollupAddress = EthAddress.ZERO,
     cleanupTmpDir = true,
-    prefilledPublicData: PublicDataTreeLeaf[] = [],
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
   ): Promise<NativeWorldStateService> {
-    const log = createLogger('world-state:database');
+    const log = createLogger('world-state:database', bindings);
     const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
     const dbMapSizeKb = 10 * 1024 * 1024;
-    log.debug(`Created temporary world state database at: ${dataDir} with size: ${dbMapSizeKb}`);
+    const worldStateTreeMapSizes: WorldStateTreeMapSizes = {
+      archiveTreeMapSizeKb: dbMapSizeKb,
+      nullifierTreeMapSizeKb: dbMapSizeKb,
+      noteHashTreeMapSizeKb: dbMapSizeKb,
+      messageTreeMapSizeKb: dbMapSizeKb,
+      publicDataTreeMapSizeKb: dbMapSizeKb,
+    };
+    log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
+
+    const instance = await IpcWorldState.spawn(dataDir, worldStateTreeMapSizes, genesis, instrumentation, bindings);
 
-    // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
     const cleanup = async () => {
       if (cleanupTmpDir) {
         await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
@@ -103,7 +132,43 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       }
     };
 
-    return this.new(rollupAddress, dataDir, dbMapSizeKb, prefilledPublicData, instrumentation, log, cleanup);
+    const recreateInstance = async () => {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      await mkdir(dataDir, { recursive: true });
+      return IpcWorldState.spawn(dataDir, worldStateTreeMapSizes, genesis, instrumentation, bindings);
+    };
+
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
+    try {
+      await worldState.init();
+    } catch (e) {
+      log.error(`Error initializing tmp world state: ${e}`);
+      throw e;
+    }
+    return worldState;
+  }
+
+  static ephemeral(
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
+    instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
+  ): Promise<NativeWorldStateService> {
+    return this.tmp(/*cleanupTmpDir=*/ true, genesis, instrumentation, bindings);
+  }
+
+  static async fromIpc(
+    wsdbBackend: ConstructorParameters<typeof IpcWorldState>[0],
+    instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
+    cleanup = () => Promise.resolve(),
+    recreateInstance?: () => Promise<NativeWorldStateInstance>,
+  ): Promise<NativeWorldStateService> {
+    const log = createLogger('world-state:database', bindings);
+    const instance = new IpcWorldState(wsdbBackend, instrumentation, bindings);
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
+    await worldState.init();
+    return worldState;
   }
 
   protected async init() {
@@ -127,21 +192,57 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
   }
 
+  public async clear(): Promise<void> {
+    if (!this.recreateInstance) {
+      throw new Error('clear() is not available for externally-managed IPC backends');
+    }
+    this.log.warn('Clearing world state: shutting down WSDB, deleting data, and recreating');
+    await this.instance.close();
+    this.cachedStatusSummary = undefined;
+    this.instance = await this.recreateInstance();
+    await this.init();
+    this.log.info('World state cleared and reinitialized from genesis');
+  }
+
+  /** Returns the socket path of the underlying IPC backend, if available. */
+  public getSocketPath(): string {
+    if (this.instance instanceof IpcWorldState) {
+      return this.instance.getSocketPath();
+    }
+    throw new Error('getSocketPath() is only available with IPC world state');
+  }
+
   public getCommitted(): MerkleTreeReadOperations {
-    return new MerkleTreesFacade(this.instance, this.initialHeader!, worldStateRevision(false, 0, 0));
+    return new MerkleTreesFacade(this.instance, this.initialHeader!, WorldStateRevision.empty());
   }
 
-  public getSnapshot(blockNumber: number): MerkleTreeReadOperations {
-    return new MerkleTreesFacade(this.instance, this.initialHeader!, worldStateRevision(false, 0, blockNumber));
+  public getSnapshot(blockNumber: BlockNumber): MerkleTreeReadOperations {
+    return new MerkleTreesFacade(
+      this.instance,
+      this.initialHeader!,
+      new WorldStateRevision(/*forkId=*/ 0, /* blockNumber=*/ blockNumber, /* includeUncommitted=*/ false),
+    );
   }
 
-  public async fork(blockNumber?: number): Promise<MerkleTreeWriteOperations> {
+  public async fork(
+    blockNumber?: BlockNumber,
+    opts: { closeDelayMs?: number } = {},
+  ): Promise<MerkleTreeWriteOperations> {
     const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, {
       latest: blockNumber === undefined,
-      blockNumber: blockNumber ?? 0,
+      blockNumber: blockNumber ?? BlockNumber.ZERO,
       canonical: true,
     });
-    return new MerkleTreesForkFacade(this.instance, this.initialHeader!, worldStateRevision(true, resp.forkId, 0));
+    return new MerkleTreesForkFacade(
+      this.instance,
+      this.initialHeader!,
+      new WorldStateRevision(
+        /*forkId=*/ resp.forkId,
+        /* blockNumber=*/ WorldStateRevision.LATEST,
+        /* includeUncommitted=*/ true,
+      ),
+      opts,
+    );
   }
 
   public getInitialHeader(): BlockHeader {
@@ -149,11 +250,22 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   public async handleL2BlockAndMessages(l2Block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull> {
-    // We have to pad both the values within tx effects because that's how the trees are built by circuits.
+    const isFirstBlock = l2Block.indexWithinCheckpoint === 0;
+    if (!isFirstBlock && l1ToL2Messages.length > 0) {
+      throw new Error(
+        `L1 to L2 messages must be empty for non-first blocks, but got ${l1ToL2Messages.length} messages for block ${l2Block.number}.`,
+      );
+    }
+
+    // We have to pad the given l1 to l2 messages, and the note hashes and nullifiers within tx effects, because that's
+    // how the trees are built by circuits.
+    const paddedL1ToL2Messages = isFirstBlock
+      ? padArrayEnd<Fr, number>(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP)
+      : [];
+
     const paddedNoteHashes = l2Block.body.txEffects.flatMap(txEffect =>
       padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX),
     );
-    const paddedL1ToL2Messages = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP);
 
     const paddedNullifiers = l2Block.body.txEffects
       .flatMap(txEffect => padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX))
@@ -173,7 +285,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
         WorldStateMessageType.SYNC_BLOCK,
         {
           blockNumber: l2Block.number,
-          blockHeaderHash: await l2Block.header.hash(),
+          blockHeaderHash: (await l2Block.hash()).toBuffer(),
           paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
           paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
           paddedNullifiers: paddedNullifiers.map(serializeLeaf),
@@ -181,7 +293,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
           blockStateRef: blockStateReference(l2Block.header.state),
           canonical: true,
         },
-        this.sanitiseAndCacheSummaryFromFull.bind(this),
+        this.sanitizeAndCacheSummaryFromFull.bind(this),
         this.deleteCachedSummary.bind(this),
       );
     } catch (err) {
@@ -191,25 +303,35 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   public async close(): Promise<void> {
-    await this.instance.close();
-    await this.cleanup();
+    try {
+      await this.instance.close();
+    } finally {
+      await this.cleanup();
+    }
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.close();
   }
 
   private async buildInitialHeader(): Promise<BlockHeader> {
     const state = await this.getInitialStateReference();
-    return BlockHeader.empty({ state });
+    return BlockHeader.empty({
+      state,
+      globalVariables: GlobalVariables.empty({ timestamp: this.genesis.genesisTimestamp }),
+    });
   }
 
-  private sanitiseAndCacheSummaryFromFull(response: WorldStateStatusFull) {
-    const sanitised = sanitiseFullStatus(response);
-    this.cachedStatusSummary = { ...sanitised.summary };
-    return sanitised;
+  private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {
+    const sanitized = sanitizeFullStatus(response);
+    this.cachedStatusSummary = { ...sanitized.summary };
+    return sanitized;
   }
 
-  private sanitiseAndCacheSummary(response: WorldStateStatusSummary) {
-    const sanitised = sanitiseSummary(response);
-    this.cachedStatusSummary = { ...sanitised };
-    return sanitised;
+  private sanitizeAndCacheSummary(response: WorldStateStatusSummary) {
+    const sanitized = sanitizeSummary(response);
+    this.cachedStatusSummary = { ...sanitized };
+    return sanitized;
   }
 
   private deleteCachedSummary(_: string) {
@@ -217,19 +339,19 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   /**
-   * Advances the finalised block number to be the number provided
-   * @param toBlockNumber The block number that is now the tip of the finalised chain
+   * Advances the finalized block number to be the number provided
+   * @param toBlockNumber The block number that is now the tip of the finalized chain
    * @returns The new WorldStateStatus
    */
-  public async setFinalised(toBlockNumber: bigint) {
+  public async setFinalized(toBlockNumber: BlockNumber) {
     try {
       await this.instance.call(
-        WorldStateMessageType.FINALISE_BLOCKS,
+        WorldStateMessageType.FINALIZE_BLOCKS,
         {
           toBlockNumber,
           canonical: true,
         },
-        this.sanitiseAndCacheSummary.bind(this),
+        this.sanitizeAndCacheSummary.bind(this),
         this.deleteCachedSummary.bind(this),
       );
     } catch (err) {
@@ -244,7 +366,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
    * @param toBlockNumber The block number of the new oldest historical block
    * @returns The new WorldStateStatus
    */
-  public async removeHistoricalBlocks(toBlockNumber: bigint) {
+  public async removeHistoricalBlocks(toBlockNumber: BlockNumber) {
     try {
       return await this.instance.call(
         WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
@@ -252,7 +374,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
           toBlockNumber,
           canonical: true,
         },
-        this.sanitiseAndCacheSummaryFromFull.bind(this),
+        this.sanitizeAndCacheSummaryFromFull.bind(this),
         this.deleteCachedSummary.bind(this),
       );
     } catch (err) {
@@ -266,7 +388,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
    * @param toBlockNumber The block number of the new tip of the pending chain,
    * @returns The new WorldStateStatus
    */
-  public async unwindBlocks(toBlockNumber: bigint) {
+  public async unwindBlocks(toBlockNumber: BlockNumber) {
     try {
       return await this.instance.call(
         WorldStateMessageType.UNWIND_BLOCKS,
@@ -274,7 +396,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
           toBlockNumber,
           canonical: true,
         },
-        this.sanitiseAndCacheSummaryFromFull.bind(this),
+        this.sanitizeAndCacheSummaryFromFull.bind(this),
         this.deleteCachedSummary.bind(this),
       );
     } catch (err) {
@@ -290,7 +412,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     return await this.instance.call(
       WorldStateMessageType.GET_STATUS,
       { canonical: true },
-      this.sanitiseAndCacheSummary.bind(this),
+      this.sanitizeAndCacheSummary.bind(this),
     );
   }
 
@@ -314,4 +436,25 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       ),
     );
   }
+
+  public async backupTo(
+    dstPath: string,
+    compact: boolean = true,
+  ): Promise<Record<Exclude<SnapshotDataKeys, 'archiver'>, string>> {
+    await this.instance.call(WorldStateMessageType.COPY_STORES, {
+      dstPath,
+      compact,
+      canonical: true,
+    });
+    return fromEntries(NATIVE_WORLD_STATE_DBS.map(([name, dir]) => [name, join(dstPath, dir, 'data.mdb')] as const));
+  }
 }
+
+// The following paths are defined in cpp-land
+export const NATIVE_WORLD_STATE_DBS = [
+  ['l1-to-l2-message-tree', 'L1ToL2MessageTree'],
+  ['archive-tree', 'ArchiveTree'],
+  ['public-data-tree', 'PublicDataTree'],
+  ['note-hash-tree', 'NoteHashTree'],
+  ['nullifier-tree', 'NullifierTree'],
+] as const;