@aztec/world-state 0.0.1-commit.001888fc → 0.0.1-commit.017a351

package/src/native/merkle_trees_facade.ts

@@ -4,6 +4,7 @@ import { createLogger } from '@aztec/foundation/log';
 import { serializeToBuffer } from '@aztec/foundation/serialize';
 import { sleep } from '@aztec/foundation/sleep';
 import { type IndexedTreeLeafPreimage, SiblingPath } from '@aztec/foundation/trees';
+import { BlockHash } from '@aztec/stdlib/block';
 import type {
   BatchInsertionResult,
   IndexedTreeId,
@@ -21,7 +22,7 @@ import {
   PublicDataTreeLeafPreimage,
 } from '@aztec/stdlib/trees';
 import { type BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
-import { type WorldStateRevision, WorldStateRevisionWithHandle } from '@aztec/stdlib/world-state';
+import type { WorldStateRevision } from '@aztec/stdlib/world-state';
 
 import assert from 'assert';
 
@@ -45,8 +46,12 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations {
     return this.initialHeader;
   }
 
-  getRevision(): WorldStateRevisionWithHandle {
-    return WorldStateRevisionWithHandle.fromWorldStateRevision(this.revision, this.instance.getHandle());
+  getRevision(): WorldStateRevision {
+    return this.revision;
+  }
+
+  getSocketPath(): string {
+    return this.instance.getSocketPath();
   }
 
   findLeafIndices(treeId: MerkleTreeId, values: MerkleTreeLeafType<MerkleTreeId>[]): Promise<(bigint | undefined)[]> {
@@ -118,7 +123,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations {
 
     const leaf = deserializeLeafValue(resp);
     if (leaf instanceof Fr) {
-      return leaf as any;
+      return treeId === MerkleTreeId.ARCHIVE ? (new BlockHash(leaf) as any) : (leaf as any);
     } else {
       return leaf.toBuffer() as any;
     }
@@ -207,6 +212,7 @@ export class MerkleTreesFacade implements MerkleTreeReadOperations {
 
 export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTreeWriteOperations {
   private log = createLogger('world-state:merkle-trees-fork-facade');
+  private closePromise: Promise<void> | undefined;
 
   constructor(
     instance: NativeWorldStateInstance,
@@ -228,7 +234,7 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
 
   async appendLeaves<ID extends MerkleTreeId>(treeId: ID, leaves: MerkleTreeLeafType<ID>[]): Promise<void> {
     await this.instance.call(WorldStateMessageType.APPEND_LEAVES, {
-      leaves: leaves.map(leaf => leaf as any),
+      leaves: leaves.map(leaf => serializeLeaf(hydrateLeaf(treeId, leaf as any))),
       forkId: this.revision.forkId,
       treeId,
     });
@@ -249,15 +255,15 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
 
     return {
       newSubtreeSiblingPath: new SiblingPath<SubtreeSiblingPathHeight>(
-        resp.subtree_path.length as any,
-        resp.subtree_path,
+        resp.subtreePath.length as any,
+        resp.subtreePath,
       ),
-      sortedNewLeaves: resp.sorted_leaves
+      sortedNewLeaves: resp.sortedLeaves
         .map(([leaf]) => leaf)
         .map(deserializeLeafValue)
         .map(serializeToBuffer),
-      sortedNewLeavesIndexes: resp.sorted_leaves.map(([, index]) => index),
-      lowLeavesWitnessData: resp.low_leaf_witness_data.map(data => ({
+      sortedNewLeavesIndexes: resp.sortedLeaves.map(([, index]) => index),
+      lowLeavesWitnessData: resp.lowLeafWitnessData.map(data => ({
         index: BigInt(data.index),
         leafPreimage: deserializeIndexedLeaf(data.leaf),
         siblingPath: new SiblingPath<TreeHeight>(data.path.length as any, data.path),
@@ -277,12 +283,12 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
     });
 
     return {
-      lowLeavesWitnessData: resp.low_leaf_witness_data.map(data => ({
+      lowLeavesWitnessData: resp.lowLeafWitnessData.map(data => ({
         index: BigInt(data.index),
         leafPreimage: deserializeIndexedLeaf(data.leaf),
         siblingPath: new SiblingPath<TreeHeight>(data.path.length as any, data.path),
       })),
-      insertionWitnessData: resp.insertion_witness_data.map(data => ({
+      insertionWitnessData: resp.insertionWitnessData.map(data => ({
         index: BigInt(data.index),
         leafPreimage: deserializeIndexedLeaf(data.leaf),
         siblingPath: new SiblingPath<TreeHeight>(data.path.length as any, data.path),
@@ -290,8 +296,17 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
     };
   }
 
-  public async close(): Promise<void> {
+  public close(): Promise<void> {
     assert.notEqual(this.revision.forkId, 0, 'Fork ID must be set');
+    // Share the in-flight close promise across duplicate dispose calls so DELETE_FORK is sent at most once.
+    if (this.closePromise) {
+      return this.closePromise;
+    }
+    this.closePromise = this.doClose();
+    return this.closePromise;
+  }
+
+  private async doClose(): Promise<void> {
     try {
       await this.instance.call(WorldStateMessageType.DELETE_FORK, { forkId: this.revision.forkId });
     } catch (err: any) {
@@ -300,6 +315,12 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
       if (err?.message === 'Native instance is closed') {
         return;
       }
+      // Ignore "Fork not found": the native fork was already destroyed by a pending-chain unwind or a
+      // historical prune (both call C++ remove_forks_for_block). Fork IDs are monotonic and never reused,
+      // so swallowing this on close cannot mask a deletion of a different fork.
+      if (err?.message === 'Fork not found') {
+        return;
+      }
       throw err;
     }
   }
@@ -309,9 +330,6 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
       void sleep(this.opts.closeDelayMs)
         .then(() => this.close())
         .catch(err => {
-          if (err && 'message' in err && err.message === 'Native instance is closed') {
-            return; // Ignore errors due to native instance being closed
-          }
           this.log.warn('Error closing MerkleTreesForkFacade after delay', { err });
         });
     } else {
@@ -352,9 +370,11 @@ export class MerkleTreesForkFacade extends MerkleTreesFacade implements MerkleTr
   }
 }
 
-function hydrateLeaf<ID extends MerkleTreeId>(treeId: ID, leaf: Fr | Buffer) {
+function hydrateLeaf(treeId: MerkleTreeId, leaf: Fr | BlockHash | Buffer) {
   if (leaf instanceof Fr) {
     return leaf;
+  } else if (leaf instanceof BlockHash) {
+    return leaf.toFr();
   } else if (treeId === MerkleTreeId.NULLIFIER_TREE) {
     return NullifierLeaf.fromBuffer(leaf);
   } else if (treeId === MerkleTreeId.PUBLIC_DATA_TREE) {
@@ -364,8 +384,10 @@ function hydrateLeaf<ID extends MerkleTreeId>(treeId: ID, leaf: Fr | Buffer) {
   }
 }
 
-export function serializeLeaf(leaf: Fr | NullifierLeaf | PublicDataTreeLeaf): SerializedLeafValue {
-  if (leaf instanceof Fr) {
+export function serializeLeaf(leaf: Fr | BlockHash | NullifierLeaf | PublicDataTreeLeaf): SerializedLeafValue {
+  if (leaf instanceof BlockHash) {
+    return leaf.toBuffer();
+  } else if (leaf instanceof Fr) {
     return leaf.toBuffer();
   } else if (leaf instanceof NullifierLeaf) {
     return { nullifier: leaf.nullifier.toBuffer() };

package/src/native/message.ts

@@ -1,7 +1,6 @@
 import { BlockNumber } from '@aztec/foundation/branded-types';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import type { Tuple } from '@aztec/foundation/serialize';
-import type { BlockHash } from '@aztec/stdlib/block';
 import { AppendOnlyTreeSnapshot, MerkleTreeId } from '@aztec/stdlib/trees';
 import type { StateReference } from '@aztec/stdlib/tx';
 import type { UInt32 } from '@aztec/stdlib/types';
@@ -391,24 +390,24 @@ interface BatchInsertRequest extends WithTreeId, WithForkId, WithLeaves {
 }
 
 interface BatchInsertResponse {
-  low_leaf_witness_data: ReadonlyArray<{
+  lowLeafWitnessData: ReadonlyArray<{
     leaf: SerializedIndexedLeaf;
     index: bigint | number;
     path: Tuple<Buffer, number>;
   }>;
-  sorted_leaves: ReadonlyArray<[SerializedLeafValue, UInt32]>;
-  subtree_path: Tuple<Buffer, number>;
+  sortedLeaves: ReadonlyArray<[SerializedLeafValue, UInt32]>;
+  subtreePath: Tuple<Buffer, number>;
 }
 
 interface SequentialInsertRequest extends WithTreeId, WithForkId, WithLeaves {}
 
 interface SequentialInsertResponse {
-  low_leaf_witness_data: ReadonlyArray<{
+  lowLeafWitnessData: ReadonlyArray<{
     leaf: SerializedIndexedLeaf;
     index: bigint | number;
     path: Tuple<Buffer, number>;
   }>;
-  insertion_witness_data: ReadonlyArray<{
+  insertionWitnessData: ReadonlyArray<{
     leaf: SerializedIndexedLeaf;
     index: bigint | number;
     path: Tuple<Buffer, number>;
@@ -423,7 +422,7 @@ interface UpdateArchiveRequest extends WithForkId {
 interface SyncBlockRequest extends WithCanonicalForkId {
   blockNumber: BlockNumber;
   blockStateRef: BlockStateReference;
-  blockHeaderHash: BlockHash;
+  blockHeaderHash: Buffer;
   paddedNoteHashes: readonly SerializedLeafValue[];
   paddedL1ToL2Messages: readonly SerializedLeafValue[];
   paddedNullifiers: readonly SerializedLeafValue[];

package/src/native/native_world_state.ts

@@ -3,7 +3,6 @@ import { BlockNumber } from '@aztec/foundation/branded-types';
 import { fromEntries, padArrayEnd } from '@aztec/foundation/collection';
 import { Fr } from '@aztec/foundation/curves/bn254';
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { tryRmDir } from '@aztec/foundation/fs';
 import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import type { L2Block } from '@aztec/stdlib/block';
 import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
@@ -14,18 +13,19 @@ import type {
 } from '@aztec/stdlib/interfaces/server';
 import type { SnapshotDataKeys } from '@aztec/stdlib/snapshots';
 import { MerkleTreeId, NullifierLeaf, type NullifierLeafPreimage, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
-import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
-import { WorldStateRevision } from '@aztec/stdlib/world-state';
+import { BlockHeader, GlobalVariables, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
+import { EMPTY_GENESIS_DATA, type GenesisData, WorldStateRevision } from '@aztec/stdlib/world-state';
 import { getTelemetryClient } from '@aztec/telemetry-client';
 
 import assert from 'assert/strict';
-import { mkdtemp, rm } from 'fs/promises';
+import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
 import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
 import type { WorldStateTreeMapSizes } from '../synchronizer/factory.js';
 import type { MerkleTreeAdminDatabase as MerkleTreeDatabase } from '../world-state-db/merkle_tree_db.js';
+import { IpcWorldState } from './ipc_world_state_instance.js';
 import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
 import {
   WorldStateMessageType,
@@ -36,7 +36,7 @@ import {
   sanitizeSummary,
   treeStateReferenceToSnapshot,
 } from './message.js';
-import { NativeWorldState } from './native_world_state_instance.js';
+import type { NativeWorldStateInstance } from './native_world_state_instance.js';
 
 // The current version of the world state database schema
 // Increment this when making incompatible changes to the database schema
@@ -50,37 +50,49 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   private cachedStatusSummary: WorldStateStatusSummary | undefined;
 
   protected constructor(
-    protected instance: NativeWorldState,
+    protected instance: NativeWorldStateInstance,
     protected readonly worldStateInstrumentation: WorldStateInstrumentation,
     protected readonly log: Logger,
+    private readonly genesis: GenesisData = EMPTY_GENESIS_DATA,
     private readonly cleanup = () => Promise.resolve(),
+    /** Factory to recreate a fresh IpcWorldState after clear(). */
+    private readonly recreateInstance?: () => Promise<NativeWorldStateInstance>,
   ) {}
 
   static async new(
     rollupAddress: EthAddress,
     dataDir: string,
     wsTreeMapSizes: WorldStateTreeMapSizes,
-    prefilledPublicData: PublicDataTreeLeaf[] = [],
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
     bindings?: LoggerBindings,
     cleanup = () => Promise.resolve(),
   ): Promise<NativeWorldStateService> {
+    for (const [key, value] of Object.entries(wsTreeMapSizes)) {
+      if (value <= 0) {
+        throw new Error(`Map size must be a positive number, got ${value} for ${key}`);
+      }
+    }
+
     const log = createLogger('world-state:database', bindings);
     const worldStateDirectory = join(dataDir, WORLD_STATE_DIR);
-    // Create a version manager to handle versioning
+
     const versionManager = new DatabaseVersionManager({
       schemaVersion: WORLD_STATE_DB_VERSION,
       rollupAddress,
       dataDirectory: worldStateDirectory,
-      onOpen: (dir: string) => {
-        return Promise.resolve(
-          new NativeWorldState(dir, wsTreeMapSizes, prefilledPublicData, instrumentation, bindings),
-        );
-      },
+      onOpen: dir => IpcWorldState.spawn(dir, wsTreeMapSizes, genesis, instrumentation, bindings),
     });
 
     const [instance] = await versionManager.open();
-    const worldState = new this(instance, instrumentation, log, cleanup);
+
+    const recreateInstance = async () => {
+      await rm(worldStateDirectory, { recursive: true, force: true, maxRetries: 3 });
+      await mkdir(worldStateDirectory, { recursive: true });
+      return IpcWorldState.spawn(worldStateDirectory, wsTreeMapSizes, genesis, instrumentation, bindings);
+    };
+
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
     try {
       await worldState.init();
     } catch (e) {
@@ -92,9 +104,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   static async tmp(
-    rollupAddress = EthAddress.ZERO,
     cleanupTmpDir = true,
-    prefilledPublicData: PublicDataTreeLeaf[] = [],
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
     instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
     bindings?: LoggerBindings,
   ): Promise<NativeWorldStateService> {
@@ -110,7 +121,8 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
     };
     log.debug(`Created temporary world state database at: ${dataDir} with tree map size: ${dbMapSizeKb}`);
 
-    // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
+    const instance = await IpcWorldState.spawn(dataDir, worldStateTreeMapSizes, genesis, instrumentation, bindings);
+
     const cleanup = async () => {
       if (cleanupTmpDir) {
         await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
@@ -120,15 +132,43 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       }
     };
 
-    return this.new(
-      rollupAddress,
-      dataDir,
-      worldStateTreeMapSizes,
-      prefilledPublicData,
-      instrumentation,
-      bindings,
-      cleanup,
-    );
+    const recreateInstance = async () => {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      await mkdir(dataDir, { recursive: true });
+      return IpcWorldState.spawn(dataDir, worldStateTreeMapSizes, genesis, instrumentation, bindings);
+    };
+
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
+    try {
+      await worldState.init();
+    } catch (e) {
+      log.error(`Error initializing tmp world state: ${e}`);
+      throw e;
+    }
+    return worldState;
+  }
+
+  static ephemeral(
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
+    instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
+  ): Promise<NativeWorldStateService> {
+    return this.tmp(/*cleanupTmpDir=*/ true, genesis, instrumentation, bindings);
+  }
+
+  static async fromIpc(
+    wsdbBackend: ConstructorParameters<typeof IpcWorldState>[0],
+    instrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+    bindings?: LoggerBindings,
+    genesis: GenesisData = EMPTY_GENESIS_DATA,
+    cleanup = () => Promise.resolve(),
+    recreateInstance?: () => Promise<NativeWorldStateInstance>,
+  ): Promise<NativeWorldStateService> {
+    const log = createLogger('world-state:database', bindings);
+    const instance = new IpcWorldState(wsdbBackend, instrumentation, bindings);
+    const worldState = new this(instance, instrumentation, log, genesis, cleanup, recreateInstance);
+    await worldState.init();
+    return worldState;
   }
 
   protected async init() {
@@ -147,16 +187,29 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
 
     // the initial header _must_ be the first element in the archive tree
     // if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
-    const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [(await this.initialHeader.hash()).toFr()]);
+    const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [await this.initialHeader.hash()]);
     const initialHeaderIndex = indices[0];
     assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
   }
 
-  public async clear() {
+  public async clear(): Promise<void> {
+    if (!this.recreateInstance) {
+      throw new Error('clear() is not available for externally-managed IPC backends');
+    }
+    this.log.warn('Clearing world state: shutting down WSDB, deleting data, and recreating');
     await this.instance.close();
     this.cachedStatusSummary = undefined;
-    await tryRmDir(this.instance.getDataDir(), this.log);
-    this.instance = this.instance.clone();
+    this.instance = await this.recreateInstance();
+    await this.init();
+    this.log.info('World state cleared and reinitialized from genesis');
+  }
+
+  /** Returns the socket path of the underlying IPC backend, if available. */
+  public getSocketPath(): string {
+    if (this.instance instanceof IpcWorldState) {
+      return this.instance.getSocketPath();
+    }
+    throw new Error('getSocketPath() is only available with IPC world state');
   }
 
   public getCommitted(): MerkleTreeReadOperations {
@@ -185,7 +238,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
       this.initialHeader!,
       new WorldStateRevision(
         /*forkId=*/ resp.forkId,
-        /* blockNumber=*/ BlockNumber.ZERO,
+        /* blockNumber=*/ WorldStateRevision.LATEST,
         /* includeUncommitted=*/ true,
       ),
       opts,
@@ -232,7 +285,7 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
         WorldStateMessageType.SYNC_BLOCK,
         {
           blockNumber: l2Block.number,
-          blockHeaderHash: await l2Block.hash(),
+          blockHeaderHash: (await l2Block.hash()).toBuffer(),
           paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
           paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
           paddedNullifiers: paddedNullifiers.map(serializeLeaf),
@@ -250,13 +303,23 @@ export class NativeWorldStateService implements MerkleTreeDatabase {
   }
 
   public async close(): Promise<void> {
-    await this.instance.close();
-    await this.cleanup();
+    try {
+      await this.instance.close();
+    } finally {
+      await this.cleanup();
+    }
+  }
+
+  async [Symbol.asyncDispose](): Promise<void> {
+    await this.close();
   }
 
   private async buildInitialHeader(): Promise<BlockHeader> {
     const state = await this.getInitialStateReference();
-    return BlockHeader.empty({ state });
+    return BlockHeader.empty({
+      state,
+      globalVariables: GlobalVariables.empty({ timestamp: this.genesis.genesisTimestamp }),
+    });
   }
 
   private sanitizeAndCacheSummaryFromFull(response: WorldStateStatusFull) {