@aztec/world-state 0.0.0-test.1 → 0.0.1-commit.017a351

package/src/native/native_world_state_instance.ts

@@ -1,238 +1,43 @@
-import {
-  ARCHIVE_HEIGHT,
-  GeneratorIndex,
-  L1_TO_L2_MSG_TREE_HEIGHT,
-  MAX_NULLIFIERS_PER_TX,
-  MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
-  NOTE_HASH_TREE_HEIGHT,
-  NULLIFIER_TREE_HEIGHT,
-  PUBLIC_DATA_TREE_HEIGHT,
-} from '@aztec/constants';
-import { createLogger } from '@aztec/foundation/log';
-import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
-import { MerkleTreeId } from '@aztec/stdlib/trees';
-import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
-
-import assert from 'assert';
-import { cpus } from 'os';
-
-import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
-import {
+import type {
   WorldStateMessageType,
-  type WorldStateRequest,
-  type WorldStateRequestCategories,
-  type WorldStateResponse,
-  isWithCanonical,
-  isWithForkId,
-  isWithRevision,
+  WorldStateRequest,
+  WorldStateRequestCategories,
+  WorldStateResponse,
 } from './message.js';
-import { WorldStateOpsQueue } from './world_state_ops_queue.js';
-
-const MAX_WORLD_STATE_THREADS = +(process.env.HARDWARE_CONCURRENCY || '16');
 
+/**
+ * Backend-agnostic handle to a running aztec-wsdb world state, accessed by the TS layer.
+ *
+ * Two implementations exist:
+ *   - {@link IpcWorldState} — talks to a standalone aztec-wsdb process over UDS or shared memory.
+ *
+ * The legacy in-process NAPI implementation has been removed; the C++ AVM (NAPI) now connects to
+ * the same aztec-wsdb process via UDS using the socket path returned by {@link getSocketPath}.
+ */
 export interface NativeWorldStateInstance {
+  /**
+   * Send a typed msgpack message to the backing world state and await its response.
+   *
+   * @param responseHandler — optional pre-resolution hook executed on the per-fork queue, useful
+   *   for caching responses while the queue still holds the fork lock.
+   * @param errorHandler — optional pre-rejection hook executed on the per-fork queue.
+   */
   call<T extends WorldStateMessageType>(
     messageType: T,
     body: WorldStateRequest[T] & WorldStateRequestCategories,
+    responseHandler?: (response: WorldStateResponse[T]) => WorldStateResponse[T],
+    errorHandler?: (error: string) => void,
   ): Promise<WorldStateResponse[T]>;
-}
-
-/**
- * Strongly-typed interface to access the WorldState class in the native world_state_napi module.
- */
-export class NativeWorldState implements NativeWorldStateInstance {
-  private open = true;
-
-  // We maintain a map of queue to fork
-  private queues = new Map<number, WorldStateOpsQueue>();
-
-  private instance: MsgpackChannel<WorldStateMessageType, WorldStateRequest, WorldStateResponse>;
-
-  /** Creates a new native WorldState instance */
-  constructor(
-    dataDir: string,
-    dbMapSizeKb: number,
-    prefilledPublicData: PublicDataTreeLeaf[] = [],
-    private instrumentation: WorldStateInstrumentation,
-    private log = createLogger('world-state:database'),
-  ) {
-    const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
-    log.info(
-      `Creating world state data store at directory ${dataDir} with map size ${dbMapSizeKb} KB and ${threads} threads.`,
-    );
-    const prefilledPublicDataBufferArray = prefilledPublicData.map(d => [d.slot.toBuffer(), d.value.toBuffer()]);
-    const ws = new BaseNativeWorldState(
-      dataDir,
-      {
-        [MerkleTreeId.NULLIFIER_TREE]: NULLIFIER_TREE_HEIGHT,
-        [MerkleTreeId.NOTE_HASH_TREE]: NOTE_HASH_TREE_HEIGHT,
-        [MerkleTreeId.PUBLIC_DATA_TREE]: PUBLIC_DATA_TREE_HEIGHT,
-        [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: L1_TO_L2_MSG_TREE_HEIGHT,
-        [MerkleTreeId.ARCHIVE]: ARCHIVE_HEIGHT,
-      },
-      {
-        [MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX,
-        [MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX,
-      },
-      prefilledPublicDataBufferArray,
-      GeneratorIndex.BLOCK_HASH,
-      dbMapSizeKb,
-      threads,
-    );
-    this.instance = new MsgpackChannel(ws);
-    // Manually create the queue for the canonical fork
-    this.queues.set(0, new WorldStateOpsQueue());
-  }
 
   /**
-   * Sends a message to the native instance and returns the response.
-   * @param messageType - The type of message to send
-   * @param body - The message body
-   * @param responseHandler - A callback accepting the response, executed on the job queue
-   * @param errorHandler - A callback called on request error, executed on the job queue
-   * @returns The response to the message
+   * UDS path the underlying aztec-wsdb process listens on. The C++ AVM uses this to attach to the
+   * same world state instance the TS layer is using.
    */
-  public async call<T extends WorldStateMessageType>(
-    messageType: T,
-    body: WorldStateRequest[T] & WorldStateRequestCategories,
-    // allows for the pre-processing of responses on the job queue before being passed back
-    responseHandler = (response: WorldStateResponse[T]): WorldStateResponse[T] => response,
-    errorHandler = (_: string) => {},
-  ): Promise<WorldStateResponse[T]> {
-    // Here we determine which fork the request is being executed against and whether it requires uncommitted data
-    // We use the fork Id to select the appropriate request queue and the uncommitted data flag to pass to the queue
-    let forkId = -1;
-    // We assume it includes uncommitted unless explicitly told otherwise
-    let committedOnly = false;
-
-    // Canonical requests ALWAYS go against the canonical fork
-    // These include things like block syncs/unwinds etc
-    // These requests don't contain a fork ID
-    if (isWithCanonical(body)) {
-      forkId = 0;
-    } else if (isWithForkId(body)) {
-      forkId = body.forkId;
-    } else if (isWithRevision(body)) {
-      forkId = body.revision.forkId;
-      committedOnly = body.revision.includeUncommitted === false;
-    } else {
-      const _: never = body;
-      throw new Error(`Unable to determine forkId for message=${WorldStateMessageType[messageType]}`);
-    }
-
-    // Get the queue or create a new one
-    let requestQueue = this.queues.get(forkId);
-    if (requestQueue === undefined) {
-      requestQueue = new WorldStateOpsQueue();
-      this.queues.set(forkId, requestQueue);
-    }
-
-    // Enqueue the request and wait for the response
-    const response = await requestQueue.execute(
-      async () => {
-        assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
-        assert.equal(this.open, true, 'Native instance is closed');
-        let response: WorldStateResponse[T];
-        try {
-          response = await this._sendMessage(messageType, body);
-        } catch (error: any) {
-          errorHandler(error.message);
-          throw error;
-        }
-        return responseHandler(response);
-      },
-      messageType,
-      committedOnly,
-    );
-
-    // If the request was to delete the fork then we clean it up here
-    if (messageType === WorldStateMessageType.DELETE_FORK) {
-      await requestQueue.stop();
-      this.queues.delete(forkId);
-    }
-    return response;
-  }
+  getSocketPath(): string;
 
   /**
-   * Stops the native instance.
+   * Shut down the world state instance. Cancels any in-flight queues, closes the IPC channel, and
+   * terminates the underlying aztec-wsdb process. Idempotent.
    */
-  public async close(): Promise<void> {
-    if (!this.open) {
-      return;
-    }
-    this.open = false;
-    const queue = this.queues.get(0)!;
-
-    await queue.execute(
-      async () => {
-        await this._sendMessage(WorldStateMessageType.CLOSE, { canonical: true });
-      },
-      WorldStateMessageType.CLOSE,
-      false,
-    );
-    await queue.stop();
-  }
-
-  private async _sendMessage<T extends WorldStateMessageType>(
-    messageType: T,
-    body: WorldStateRequest[T] & WorldStateRequestCategories,
-  ): Promise<WorldStateResponse[T]> {
-    let logMetadata: Record<string, any> = {};
-
-    if (body) {
-      if ('treeId' in body) {
-        logMetadata['treeId'] = MerkleTreeId[body.treeId];
-      }
-
-      if ('revision' in body) {
-        logMetadata = { ...logMetadata, ...body.revision };
-      }
-
-      if ('forkId' in body) {
-        logMetadata['forkId'] = body.forkId;
-      }
-
-      if ('blockNumber' in body) {
-        logMetadata['blockNumber'] = body.blockNumber;
-      }
-
-      if ('toBlockNumber' in body) {
-        logMetadata['toBlockNumber'] = body.toBlockNumber;
-      }
-
-      if ('leafIndex' in body) {
-        logMetadata['leafIndex'] = body.leafIndex;
-      }
-
-      if ('blockHeaderHash' in body) {
-        logMetadata['blockHeaderHash'] = '0x' + body.blockHeaderHash.toString('hex');
-      }
-
-      if ('leaves' in body) {
-        logMetadata['leavesCount'] = body.leaves.length;
-      }
-
-      // sync operation
-      if ('paddedNoteHashes' in body) {
-        logMetadata['notesCount'] = body.paddedNoteHashes.length;
-        logMetadata['nullifiersCount'] = body.paddedNullifiers.length;
-        logMetadata['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length;
-        logMetadata['publicDataWritesCount'] = body.publicDataWrites.length;
-      }
-    }
-
-    try {
-      const { duration, response } = await this.instance.sendMessage(messageType, body);
-      this.log.trace(`Call ${WorldStateMessageType[messageType]} took (ms)`, {
-        duration,
-        ...logMetadata,
-      });
-
-      this.instrumentation.recordRoundTrip(duration.totalUs, messageType);
-      return response;
-    } catch (error) {
-      this.log.error(`Call ${WorldStateMessageType[messageType]} failed: ${error}`, error, logMetadata);
-      throw error;
-    }
-  }
+  close(): Promise<void>;
 }

package/src/native/world_state_ops_queue.ts

@@ -35,7 +35,7 @@ export const MUTATING_MSG_TYPES = new Set([
   WorldStateMessageType.SYNC_BLOCK,
   WorldStateMessageType.CREATE_FORK,
   WorldStateMessageType.DELETE_FORK,
-  WorldStateMessageType.FINALISE_BLOCKS,
+  WorldStateMessageType.FINALIZE_BLOCKS,
   WorldStateMessageType.UNWIND_BLOCKS,
   WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
   WorldStateMessageType.CREATE_CHECKPOINT,
@@ -96,7 +96,7 @@ export class WorldStateOpsQueue {
     // then send the request immediately
     // If a mutating request is in flight then we must wait
     // If a mutating request is not in flight but something is queued then it must be a mutating request
-    if (this.inFlightMutatingCount == 0 && this.requests.length == 0) {
+    if (this.inFlightMutatingCount === 0 && this.requests.length === 0) {
       this.sendEnqueuedRequest(op);
     } else {
       this.requests.push(op);
@@ -122,7 +122,7 @@ export class WorldStateOpsQueue {
     --this.inFlightCount;
 
     // If there are still requests in flight then do nothing further
-    if (this.inFlightCount != 0) {
+    if (this.inFlightCount !== 0) {
       return;
     }
 
@@ -134,7 +134,7 @@ export class WorldStateOpsQueue {
     while (this.requests.length > 0) {
       const next = this.requests[0];
       if (next.mutating) {
-        if (this.inFlightCount == 0) {
+        if (this.inFlightCount === 0) {
           // send the mutating request
           this.requests.shift();
           this.sendEnqueuedRequest(next);
@@ -149,7 +149,7 @@ export class WorldStateOpsQueue {
     }
 
     // If the queue is empty, there is nothing in flight and we have been told to stop, then resolve the stop promise
-    if (this.inFlightCount == 0 && this.stopResolve !== undefined) {
+    if (this.inFlightCount === 0 && this.stopResolve !== undefined) {
       this.stopResolve();
     }
   }
@@ -182,7 +182,7 @@ export class WorldStateOpsQueue {
     });
 
     // If no outstanding requests then immediately resolve the promise
-    if (this.requests.length == 0 && this.inFlightCount == 0 && this.stopResolve !== undefined) {
+    if (this.requests.length === 0 && this.inFlightCount === 0 && this.stopResolve !== undefined) {
       this.stopResolve();
     }
     return this.stopPromise;

package/src/synchronizer/config.ts

@@ -1,8 +1,8 @@
 import {
   type ConfigMappingsType,
-  booleanConfigHelper,
   getConfigFromMappings,
   numberConfigHelper,
+  optionalNumberConfigHelper,
 } from '@aztec/foundation/config';
 
 /** World State synchronizer configuration values. */
@@ -10,51 +10,89 @@ export interface WorldStateConfig {
   /** The frequency in which to check. */
   worldStateBlockCheckIntervalMS: number;
 
-  /** Whether to follow only the proven chain. */
-  worldStateProvenBlocksOnly: boolean;
-
   /** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
   worldStateBlockRequestBatchSize?: number;
 
-  /** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKB if not specified*/
+  /** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKb if not specified*/
   worldStateDbMapSizeKb?: number;
 
+  /** The map size to be provided to LMDB for each world state archive tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
+  archiveTreeMapSizeKb?: number;
+
+  /** The map size to be provided to LMDB for each world state nullifier tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
+  nullifierTreeMapSizeKb?: number;
+
+  /** The map size to be provided to LMDB for each world state note hash tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
+  noteHashTreeMapSizeKb?: number;
+
+  /** The map size to be provided to LMDB for each world state message tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
+  messageTreeMapSizeKb?: number;
+
+  /** The map size to be provided to LMDB for each world state public data tree, optional, will inherit from the general worldStateDbMapSizeKb if not specified*/
+  publicDataTreeMapSizeKb?: number;
+
   /** Optional directory for the world state DB, if unspecified will default to the general data directory */
   worldStateDataDirectory?: string;
 
-  /** The number of historic blocks to maintain */
-  worldStateBlockHistory: number;
+  /** The number of historic checkpoints worth of blocks to maintain */
+  worldStateCheckpointHistory: number;
 }
 
 export const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig> = {
   worldStateBlockCheckIntervalMS: {
     env: 'WS_BLOCK_CHECK_INTERVAL_MS',
-    parseEnv: (val: string) => +val,
-    defaultValue: 100,
+    ...numberConfigHelper(100),
     description: 'The frequency in which to check.',
   },
-  worldStateProvenBlocksOnly: {
-    env: 'WS_PROVEN_BLOCKS_ONLY',
-    description: 'Whether to follow only the proven chain.',
-    ...booleanConfigHelper(),
-  },
   worldStateBlockRequestBatchSize: {
     env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
-    parseEnv: (val: string | undefined) => (val ? +val : undefined),
+    ...optionalNumberConfigHelper(),
     description: 'Size of the batch for each get-blocks request from the synchronizer to the archiver.',
   },
   worldStateDbMapSizeKb: {
     env: 'WS_DB_MAP_SIZE_KB',
-    parseEnv: (val: string | undefined) => (val ? +val : undefined),
-    description: 'The maximum possible size of the world state DB',
+    ...optionalNumberConfigHelper(),
+    description: 'The maximum possible size of the world state DB in KB. Overwrites the general dataStoreMapSizeKb.',
+  },
+  archiveTreeMapSizeKb: {
+    env: 'ARCHIVE_TREE_MAP_SIZE_KB',
+    ...optionalNumberConfigHelper(),
+    description:
+      'The maximum possible size of the world state archive tree in KB. Overwrites the general worldStateDbMapSizeKb.',
+  },
+  nullifierTreeMapSizeKb: {
+    env: 'NULLIFIER_TREE_MAP_SIZE_KB',
+    ...optionalNumberConfigHelper(),
+    description:
+      'The maximum possible size of the world state nullifier tree in KB. Overwrites the general worldStateDbMapSizeKb.',
+  },
+  noteHashTreeMapSizeKb: {
+    env: 'NOTE_HASH_TREE_MAP_SIZE_KB',
+    ...optionalNumberConfigHelper(),
+    description:
+      'The maximum possible size of the world state note hash tree in KB. Overwrites the general worldStateDbMapSizeKb.',
+  },
+  messageTreeMapSizeKb: {
+    env: 'MESSAGE_TREE_MAP_SIZE_KB',
+    ...optionalNumberConfigHelper(),
+    description:
+      'The maximum possible size of the world state message tree in KB. Overwrites the general worldStateDbMapSizeKb.',
+  },
+  publicDataTreeMapSizeKb: {
+    env: 'PUBLIC_DATA_TREE_MAP_SIZE_KB',
+    ...optionalNumberConfigHelper(),
+    description:
+      'The maximum possible size of the world state public data tree in KB. Overwrites the general worldStateDbMapSizeKb.',
   },
   worldStateDataDirectory: {
     env: 'WS_DATA_DIRECTORY',
     description: 'Optional directory for the world state database',
   },
-  worldStateBlockHistory: {
-    env: 'WS_NUM_HISTORIC_BLOCKS',
-    description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained',
+  worldStateCheckpointHistory: {
+    env: 'WS_NUM_HISTORIC_CHECKPOINTS',
+    description:
+      'The number of historic checkpoints worth of blocks to maintain. Values less than 1 mean all history is maintained',
+    fallback: ['WS_NUM_HISTORIC_BLOCKS'],
     ...numberConfigHelper(64),
   },
 };

package/src/synchronizer/errors.ts

@@ -0,0 +1,5 @@
+export class WorldStateSynchronizerError extends Error {
+  constructor(message: string, options?: ErrorOptions) {
+    super(message, options);
+  }
+}

package/src/synchronizer/factory.ts

@@ -1,7 +1,8 @@
-import type { DataStoreConfig } from '@aztec/kv-store/config';
+import type { LoggerBindings } from '@aztec/foundation/log';
 import type { L2BlockSource } from '@aztec/stdlib/block';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 import type { L1ToL2MessageSource } from '@aztec/stdlib/messaging';
-import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
+import { EMPTY_GENESIS_DATA, type GenesisData, isGenesisData } from '@aztec/stdlib/world-state';
 import { type TelemetryClient, getTelemetryClient } from '@aztec/telemetry-client';
 
 import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
@@ -9,44 +10,73 @@ import { NativeWorldStateService } from '../native/native_world_state.js';
 import type { WorldStateConfig } from './config.js';
 import { ServerWorldStateSynchronizer } from './server_world_state_synchronizer.js';
 
+export interface WorldStateTreeMapSizes {
+  archiveTreeMapSizeKb: number;
+  nullifierTreeMapSizeKb: number;
+  noteHashTreeMapSizeKb: number;
+  messageTreeMapSizeKb: number;
+  publicDataTreeMapSizeKb: number;
+}
+
 export async function createWorldStateSynchronizer(
   config: WorldStateConfig & DataStoreConfig,
   l2BlockSource: L2BlockSource & L1ToL2MessageSource,
-  prefilledPublicData: PublicDataTreeLeaf[] = [],
+  genesisOrNativeWorldState: GenesisData | NativeWorldStateService,
   client: TelemetryClient = getTelemetryClient(),
+  bindings?: LoggerBindings,
 ) {
   const instrumentation = new WorldStateInstrumentation(client);
-  const merkleTrees = await createWorldState(config, prefilledPublicData, instrumentation);
+  const merkleTrees = isGenesisData(genesisOrNativeWorldState)
+    ? await createWorldState(config, genesisOrNativeWorldState, instrumentation, bindings)
+    : genesisOrNativeWorldState;
   return new ServerWorldStateSynchronizer(merkleTrees, l2BlockSource, config, instrumentation);
 }
 
 export async function createWorldState(
-  config: WorldStateConfig & DataStoreConfig,
-  prefilledPublicData: PublicDataTreeLeaf[] = [],
+  config: Pick<
+    WorldStateConfig,
+    | 'worldStateDataDirectory'
+    | 'worldStateDbMapSizeKb'
+    | 'archiveTreeMapSizeKb'
+    | 'nullifierTreeMapSizeKb'
+    | 'noteHashTreeMapSizeKb'
+    | 'messageTreeMapSizeKb'
+    | 'publicDataTreeMapSizeKb'
+  > &
+    Pick<DataStoreConfig, 'dataDirectory' | 'dataStoreMapSizeKb' | 'rollupAddress'>,
+  genesis: GenesisData = EMPTY_GENESIS_DATA,
   instrumentation: WorldStateInstrumentation = new WorldStateInstrumentation(getTelemetryClient()),
+  bindings?: LoggerBindings,
 ) {
-  const newConfig = {
-    dataDirectory: config.worldStateDataDirectory ?? config.dataDirectory,
-    dataStoreMapSizeKB: config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKB,
-  } as DataStoreConfig;
+  const dataDirectory = config.worldStateDataDirectory ?? config.dataDirectory;
+  const dataStoreMapSizeKb = config.worldStateDbMapSizeKb ?? config.dataStoreMapSizeKb;
+  const wsTreeMapSizes: WorldStateTreeMapSizes = {
+    archiveTreeMapSizeKb: config.archiveTreeMapSizeKb ?? dataStoreMapSizeKb,
+    nullifierTreeMapSizeKb: config.nullifierTreeMapSizeKb ?? dataStoreMapSizeKb,
+    noteHashTreeMapSizeKb: config.noteHashTreeMapSizeKb ?? dataStoreMapSizeKb,
+    messageTreeMapSizeKb: config.messageTreeMapSizeKb ?? dataStoreMapSizeKb,
+    publicDataTreeMapSizeKb: config.publicDataTreeMapSizeKb ?? dataStoreMapSizeKb,
+  };
 
-  if (!config.l1Contracts?.rollupAddress) {
+  if (!config.rollupAddress) {
     throw new Error('Rollup address is required to create a world state synchronizer.');
   }
 
   // If a data directory is provided in config, then create a persistent store.
-  const merkleTrees = newConfig.dataDirectory
+  const merkleTrees = dataDirectory
     ? await NativeWorldStateService.new(
-        config.l1Contracts.rollupAddress,
-        newConfig.dataDirectory,
-        newConfig.dataStoreMapSizeKB,
-        prefilledPublicData,
+        config.rollupAddress,
+        dataDirectory,
+        wsTreeMapSizes,
+        genesis,
         instrumentation,
+        bindings,
       )
     : await NativeWorldStateService.tmp(
-        config.l1Contracts.rollupAddress,
         !['true', '1'].includes(process.env.DEBUG_WORLD_STATE!),
-        prefilledPublicData,
+        genesis,
+        instrumentation,
+        bindings,
       );
 
   return merkleTrees;