@aztec/world-state 0.0.0-test.0

package/dest/world-state-db/merkle_tree_db.d.ts

@@ -0,0 +1,68 @@
+import type { Fr } from '@aztec/foundation/fields';
+import type { IndexedTreeSnapshot, TreeSnapshot } from '@aztec/merkle-tree';
+import type { L2Block } from '@aztec/stdlib/block';
+import type { ForkMerkleTreeOperations, MerkleTreeReadOperations } from '@aztec/stdlib/interfaces/server';
+import type { MerkleTreeId } from '@aztec/stdlib/trees';
+import type { WorldStateStatusFull, WorldStateStatusSummary } from '../native/message.js';
+/**
+ *
+ * @remarks Short explanation:
+ *    The nullifier tree must be initially padded as the pre-populated 0 index prevents efficient subtree insertion.
+ *    Padding with some values solves this issue.
+ *
+ * @remarks Thorough explanation:
+ *    There needs to be an initial (0,0,0) leaf in the tree, so that when we insert the first 'proper' leaf, we can
+ *    prove that any value greater than 0 doesn't exist in the tree yet. We prefill/pad the tree with "the number of
+ *    leaves that are added by one block" so that the first 'proper' block can insert a full subtree.
+ *
+ *    Without this padding, there would be a leaf (0,0,0) at leaf index 0, making it really difficult to insert e.g.
+ *    1024 leaves for the first block, because there's only neat space for 1023 leaves after 0. By padding with 1023
+ *    more leaves, we can then insert the first block of 1024 leaves into indices 1024:2047.
+ */
+export declare const INITIAL_NULLIFIER_TREE_SIZE: number;
+export declare const INITIAL_PUBLIC_DATA_TREE_SIZE: number;
+export type TreeSnapshots = {
+    [MerkleTreeId.NULLIFIER_TREE]: IndexedTreeSnapshot;
+    [MerkleTreeId.NOTE_HASH_TREE]: TreeSnapshot<Fr>;
+    [MerkleTreeId.PUBLIC_DATA_TREE]: IndexedTreeSnapshot;
+    [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: TreeSnapshot<Fr>;
+    [MerkleTreeId.ARCHIVE]: TreeSnapshot<Fr>;
+};
+export interface MerkleTreeAdminDatabase extends ForkMerkleTreeOperations {
+    /**
+     * Handles a single L2 block (i.e. Inserts the new note hashes into the merkle tree).
+     * @param block - The L2 block to handle.
+     * @param l1ToL2Messages - The L1 to L2 messages for the block.
+     */
+    handleL2BlockAndMessages(block: L2Block, l1ToL2Messages: Fr[]): Promise<WorldStateStatusFull>;
+    /**
+     * Gets a handle that allows reading the latest committed state
+     */
+    getCommitted(): MerkleTreeReadOperations;
+    /**
+     * Removes all historical snapshots up to but not including the given block number
+     * @param toBlockNumber The block number of the new oldest historical block
+     * @returns The new WorldStateStatus
+     */
+    removeHistoricalBlocks(toBlockNumber: bigint): Promise<WorldStateStatusFull>;
+    /**
+     * Removes all pending blocks down to but not including the given block number
+     * @param toBlockNumber The block number of the new tip of the pending chain,
+     * @returns The new WorldStateStatus
+     */
+    unwindBlocks(toBlockNumber: bigint): Promise<WorldStateStatusFull>;
+    /**
+     * Advances the finalised block number to be the number provided
+     * @param toBlockNumber The block number that is now the tip of the finalised chain
+     * @returns The new WorldStateStatus
+     */
+    setFinalised(toBlockNumber: bigint): Promise<WorldStateStatusSummary>;
+    /**
+     * Gets the current status summary of the database.
+     * @returns The current WorldStateStatus.
+     */
+    getStatusSummary(): Promise<WorldStateStatusSummary>;
+    /** Stops the database */
+    close(): Promise<void>;
+}
+//# sourceMappingURL=merkle_tree_db.d.ts.map

package/dest/world-state-db/merkle_tree_db.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"merkle_tree_db.d.ts","sourceRoot":"","sources":["../../src/world-state-db/merkle_tree_db.ts"],"names":[],"mappings":"AACA,OAAO,KAAK,EAAE,EAAE,EAAE,MAAM,0BAA0B,CAAC;AACnD,OAAO,KAAK,EAAE,mBAAmB,EAAE,YAAY,EAAE,MAAM,oBAAoB,CAAC;AAC5E,OAAO,KAAK,EAAE,OAAO,EAAE,MAAM,qBAAqB,CAAC;AACnD,OAAO,KAAK,EAAE,wBAAwB,EAAE,wBAAwB,EAAE,MAAM,iCAAiC,CAAC;AAC1G,OAAO,KAAK,EAAE,YAAY,EAAE,MAAM,qBAAqB,CAAC;AAExD,OAAO,KAAK,EAAE,oBAAoB,EAAE,uBAAuB,EAAE,MAAM,sBAAsB,CAAC;AAE1F;;;;;;;;;;;;;;GAcG;AACH,eAAO,MAAM,2BAA2B,QAA4B,CAAC;AAErE,eAAO,MAAM,6BAA6B,QAAmD,CAAC;AAE9F,MAAM,MAAM,aAAa,GAAG;IAC1B,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,mBAAmB,CAAC;IACnD,CAAC,YAAY,CAAC,cAAc,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IAChD,CAAC,YAAY,CAAC,gBAAgB,CAAC,EAAE,mBAAmB,CAAC;IACrD,CAAC,YAAY,CAAC,qBAAqB,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;IACvD,CAAC,YAAY,CAAC,OAAO,CAAC,EAAE,YAAY,CAAC,EAAE,CAAC,CAAC;CAC1C,CAAC;AAEF,MAAM,WAAW,uBAAwB,SAAQ,wBAAwB;IACvE;;;;OAIG;IACH,wBAAwB,CAAC,KAAK,EAAE,OAAO,EAAE,cAAc,EAAE,EAAE,EAAE,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE9F;;OAEG;IACH,YAAY,IAAI,wBAAwB,CAAC;IAEzC;;;;OAIG;IACH,sBAAsB,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAE7E;;;;OAIG;IACH,YAAY,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,oBAAoB,CAAC,CAAC;IAEnE;;;;OAIG;IACH,YAAY,CAAC,aAAa,EAAE,MAAM,GAAG,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAEtE;;;OAGG;IACH,gBAAgB,IAAI,OAAO,CAAC,uBAAuB,CAAC,CAAC;IAErD,yBAAyB;IACzB,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC,CAAC;CACxB"}

package/dest/world-state-db/merkle_tree_db.js

@@ -0,0 +1,17 @@
+import { MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX } from '@aztec/constants';
+/**
+ *
+ * @remarks Short explanation:
+ *    The nullifier tree must be initially padded as the pre-populated 0 index prevents efficient subtree insertion.
+ *    Padding with some values solves this issue.
+ *
+ * @remarks Thorough explanation:
+ *    There needs to be an initial (0,0,0) leaf in the tree, so that when we insert the first 'proper' leaf, we can
+ *    prove that any value greater than 0 doesn't exist in the tree yet. We prefill/pad the tree with "the number of
+ *    leaves that are added by one block" so that the first 'proper' block can insert a full subtree.
+ *
+ *    Without this padding, there would be a leaf (0,0,0) at leaf index 0, making it really difficult to insert e.g.
+ *    1024 leaves for the first block, because there's only neat space for 1023 leaves after 0. By padding with 1023
+ *    more leaves, we can then insert the first block of 1024 leaves into indices 1024:2047.
+ */ export const INITIAL_NULLIFIER_TREE_SIZE = 2 * MAX_NULLIFIERS_PER_TX;
+export const INITIAL_PUBLIC_DATA_TREE_SIZE = 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX;

package/package.json

@@ -0,0 +1,98 @@
+{
+  "name": "@aztec/world-state",
+  "version": "0.0.0-test.0",
+  "type": "module",
+  "exports": {
+    ".": "./dest/index.js",
+    "./native": "./dest/native/index.js",
+    "./test": "./dest/test/index.js",
+    "./testing": "./dest/testing.js",
+    "./config": "./dest/synchronizer/config.js"
+  },
+  "typedocOptions": {
+    "entryPoints": [
+      "./src/index.ts",
+      "./src/testing.ts"
+    ],
+    "name": "World State",
+    "tsconfig": "./tsconfig.json"
+  },
+  "scripts": {
+    "build": "yarn clean && tsc -b",
+    "build:dev": "tsc -b --watch",
+    "clean": "rm -rf ./dest .tsbuildinfo",
+    "formatting": "run -T prettier --check ./src && run -T eslint ./src",
+    "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
+    "test": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
+  },
+  "inherits": [
+    "../package.common.json"
+  ],
+  "jest": {
+    "moduleNameMapper": {
+      "^(\\.{1,2}/.*)\\.[cm]?js$": "$1"
+    },
+    "testRegex": "./src/.*\\.test\\.(js|mjs|ts)$",
+    "rootDir": "./src",
+    "transform": {
+      "^.+\\.tsx?$": [
+        "@swc/jest",
+        {
+          "jsc": {
+            "parser": {
+              "syntax": "typescript",
+              "decorators": true
+            },
+            "transform": {
+              "decoratorVersion": "2022-03"
+            }
+          }
+        }
+      ]
+    },
+    "extensionsToTreatAsEsm": [
+      ".ts"
+    ],
+    "reporters": [
+      "default"
+    ],
+    "testTimeout": 120000,
+    "setupFiles": [
+      "../../foundation/src/jest/setup.mjs"
+    ]
+  },
+  "dependencies": {
+    "@aztec/constants": "0.0.0-test.0",
+    "@aztec/foundation": "0.0.0-test.0",
+    "@aztec/kv-store": "0.0.0-test.0",
+    "@aztec/merkle-tree": "0.0.0-test.0",
+    "@aztec/native": "0.0.0-test.0",
+    "@aztec/protocol-contracts": "0.0.0-test.0",
+    "@aztec/stdlib": "0.0.0-test.0",
+    "@aztec/telemetry-client": "0.0.0-test.0",
+    "tslib": "^2.4.0",
+    "zod": "^3.23.8"
+  },
+  "devDependencies": {
+    "@aztec/archiver": "0.0.0-test.0",
+    "@jest/globals": "^29.5.0",
+    "@types/jest": "^29.5.0",
+    "@types/levelup": "^5.1.2",
+    "@types/memdown": "^3.0.0",
+    "@types/node": "^18.7.23",
+    "jest": "^29.5.0",
+    "jest-mock-extended": "^3.0.5",
+    "memdown": "^6.1.1",
+    "ts-node": "^10.9.1",
+    "typescript": "^5.0.4"
+  },
+  "files": [
+    "dest",
+    "src",
+    "!*.test.*"
+  ],
+  "types": "./dest/index.d.ts",
+  "engines": {
+    "node": ">=18"
+  }
+}

package/src/index.ts

@@ -0,0 +1,4 @@
+export * from './synchronizer/index.js';
+export * from './world-state-db/index.js';
+export * from './synchronizer/config.js';
+export * from './native/index.js';

package/src/instrumentation/instrumentation.ts

@@ -0,0 +1,174 @@
+import { createLogger } from '@aztec/foundation/log';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
+import {
+  Attributes,
+  type Gauge,
+  type Histogram,
+  Metrics,
+  type TelemetryClient,
+  type UpDownCounter,
+  ValueType,
+} from '@aztec/telemetry-client';
+
+import {
+  type DBStats,
+  type TreeDBStats,
+  type TreeMeta,
+  WorldStateMessageType,
+  type WorldStateStatusFull,
+} from '../native/message.js';
+
+type DBTypeString = 'leaf_preimage' | 'leaf_indices' | 'nodes' | 'blocks' | 'block_indices';
+
+const durationTrackDenylist = new Set<WorldStateMessageType>([
+  WorldStateMessageType.GET_INITIAL_STATE_REFERENCE,
+  WorldStateMessageType.CLOSE,
+
+  // these aren't used anymore, should be removed from the API
+  WorldStateMessageType.COMMIT,
+  WorldStateMessageType.ROLLBACK,
+]);
+
+export class WorldStateInstrumentation {
+  private dbMapSize: Gauge;
+  private treeSize: Gauge;
+  private unfinalisedHeight: Gauge;
+  private finalisedHeight: Gauge;
+  private oldestBlock: Gauge;
+  private dbNumItems: Gauge;
+  private dbUsedSize: Gauge;
+  private requestHistogram: Histogram;
+  private criticalErrors: UpDownCounter;
+
+  constructor(public readonly telemetry: TelemetryClient, private log = createLogger('world-state:instrumentation')) {
+    const meter = telemetry.getMeter('World State');
+    this.dbMapSize = meter.createGauge(Metrics.WORLD_STATE_DB_MAP_SIZE, {
+      description: `The current configured map size for each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.treeSize = meter.createGauge(Metrics.WORLD_STATE_TREE_SIZE, {
+      description: `The current number of leaves in each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.unfinalisedHeight = meter.createGauge(Metrics.WORLD_STATE_UNFINALISED_HEIGHT, {
+      description: `The unfinalised block height of each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.finalisedHeight = meter.createGauge(Metrics.WORLD_STATE_FINALISED_HEIGHT, {
+      description: `The finalised block height of each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.oldestBlock = meter.createGauge(Metrics.WORLD_STATE_OLDEST_BLOCK, {
+      description: `The oldest historical block of each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.dbUsedSize = meter.createGauge(Metrics.WORLD_STATE_DB_USED_SIZE, {
+      description: `The current used database size for each db of each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.dbNumItems = meter.createGauge(Metrics.WORLD_STATE_DB_NUM_ITEMS, {
+      description: `The current number of items in each database of each merkle tree`,
+      valueType: ValueType.INT,
+    });
+
+    this.requestHistogram = meter.createHistogram(Metrics.WORLD_STATE_REQUEST_TIME, {
+      description: 'The round trip time of world state requests',
+      unit: 'us',
+      valueType: ValueType.INT,
+    });
+
+    this.criticalErrors = meter.createUpDownCounter(Metrics.WORLD_STATE_CRITICAL_ERROR_COUNT, {
+      description: 'The number of critical errors in the world state',
+      valueType: ValueType.INT,
+    });
+  }
+
+  private updateTreeStats(treeDbStats: TreeDBStats, treeMeta: TreeMeta, tree: MerkleTreeId) {
+    this.dbMapSize.record(Number(treeDbStats.mapSize), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.treeSize.record(Number(treeMeta.size), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.unfinalisedHeight.record(Number(treeMeta.unfinalisedBlockHeight), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.finalisedHeight.record(Number(treeMeta.finalisedBlockHeight), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.oldestBlock.record(Number(treeMeta.oldestHistoricBlock), {
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+
+    this.updateTreeDBStats(treeDbStats.blockIndicesDBStats, 'block_indices', tree);
+    this.updateTreeDBStats(treeDbStats.blocksDBStats, 'blocks', tree);
+    this.updateTreeDBStats(treeDbStats.leafIndicesDBStats, 'leaf_indices', tree);
+    this.updateTreeDBStats(treeDbStats.leafPreimagesDBStats, 'leaf_preimage', tree);
+    this.updateTreeDBStats(treeDbStats.nodesDBStats, 'nodes', tree);
+  }
+
+  private updateTreeDBStats(dbStats: DBStats, dbType: DBTypeString, tree: MerkleTreeId) {
+    this.dbNumItems.record(Number(dbStats.numDataItems), {
+      [Attributes.WS_DB_DATA_TYPE]: dbType,
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+    this.dbUsedSize.record(Number(dbStats.totalUsedSize), {
+      [Attributes.WS_DB_DATA_TYPE]: dbType,
+      [Attributes.MERKLE_TREE_NAME]: MerkleTreeId[tree],
+    });
+  }
+
+  public updateWorldStateMetrics(worldStateStatus: WorldStateStatusFull) {
+    this.updateTreeStats(
+      worldStateStatus.dbStats.archiveTreeStats,
+      worldStateStatus.meta.archiveTreeMeta,
+      MerkleTreeId.ARCHIVE,
+    );
+
+    this.updateTreeStats(
+      worldStateStatus.dbStats.messageTreeStats,
+      worldStateStatus.meta.messageTreeMeta,
+      MerkleTreeId.L1_TO_L2_MESSAGE_TREE,
+    );
+
+    this.updateTreeStats(
+      worldStateStatus.dbStats.noteHashTreeStats,
+      worldStateStatus.meta.noteHashTreeMeta,
+      MerkleTreeId.NOTE_HASH_TREE,
+    );
+
+    this.updateTreeStats(
+      worldStateStatus.dbStats.nullifierTreeStats,
+      worldStateStatus.meta.nullifierTreeMeta,
+      MerkleTreeId.NULLIFIER_TREE,
+    );
+
+    this.updateTreeStats(
+      worldStateStatus.dbStats.publicDataTreeStats,
+      worldStateStatus.meta.publicDataTreeMeta,
+      MerkleTreeId.PUBLIC_DATA_TREE,
+    );
+  }
+
+  public recordRoundTrip(timeUs: number, request: WorldStateMessageType) {
+    if (!durationTrackDenylist.has(request)) {
+      this.requestHistogram.record(Math.ceil(timeUs), {
+        [Attributes.WORLD_STATE_REQUEST_TYPE]: WorldStateMessageType[request],
+      });
+    }
+  }
+
+  public incCriticalErrors(
+    errorType: 'synch_pending_block' | 'finalize_block' | 'prune_pending_block' | 'prune_historical_block',
+  ) {
+    this.criticalErrors.add(1, {
+      [Attributes.ERROR_TYPE]: errorType,
+    });
+  }
+}

package/src/native/fork_checkpoint.ts

@@ -0,0 +1,30 @@
+import type { MerkleTreeCheckpointOperations } from '@aztec/stdlib/interfaces/server';
+
+export class ForkCheckpoint {
+  private completed = false;
+
+  private constructor(private readonly fork: MerkleTreeCheckpointOperations) {}
+
+  static async new(fork: MerkleTreeCheckpointOperations): Promise<ForkCheckpoint> {
+    await fork.createCheckpoint();
+    return new ForkCheckpoint(fork);
+  }
+
+  async commit(): Promise<void> {
+    if (this.completed) {
+      return;
+    }
+
+    await this.fork.commitCheckpoint();
+    this.completed = true;
+  }
+
+  async revert(): Promise<void> {
+    if (this.completed) {
+      return;
+    }
+
+    await this.fork.revertCheckpoint();
+    this.completed = true;
+  }
+}

package/src/native/index.ts

@@ -0,0 +1,2 @@
+export * from './native_world_state.js';
+export * from './fork_checkpoint.js';