@aztec/world-state 0.0.0-test.0

package/dest/native/native_world_state.js

@@ -0,0 +1,229 @@
+import { MAX_NOTE_HASHES_PER_TX, MAX_NULLIFIERS_PER_TX, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP } from '@aztec/constants';
+import { padArrayEnd } from '@aztec/foundation/collection';
+import { EthAddress } from '@aztec/foundation/eth-address';
+import { Fr } from '@aztec/foundation/fields';
+import { createLogger } from '@aztec/foundation/log';
+import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
+import { MerkleTreeId, NullifierLeaf, PublicDataTreeLeaf } from '@aztec/stdlib/trees';
+import { BlockHeader, PartialStateReference, StateReference } from '@aztec/stdlib/tx';
+import { getTelemetryClient } from '@aztec/telemetry-client';
+import assert from 'assert/strict';
+import { mkdtemp, rm } from 'fs/promises';
+import { tmpdir } from 'os';
+import { join } from 'path';
+import { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
+import { MerkleTreesFacade, MerkleTreesForkFacade, serializeLeaf } from './merkle_trees_facade.js';
+import { WorldStateMessageType, blockStateReference, sanitiseFullStatus, sanitiseSummary, treeStateReferenceToSnapshot, worldStateRevision } from './message.js';
+import { NativeWorldState } from './native_world_state_instance.js';
+// The current version of the world state database schema
+// Increment this when making incompatible changes to the database schema
+export const WORLD_STATE_DB_VERSION = 1; // The initial version
+export class NativeWorldStateService {
+    instance;
+    worldStateInstrumentation;
+    log;
+    cleanup;
+    initialHeader;
+    // This is read heavily and only changes when data is persisted, so we cache it
+    cachedStatusSummary;
+    constructor(instance, worldStateInstrumentation, log = createLogger('world-state:database'), cleanup = ()=>Promise.resolve()){
+        this.instance = instance;
+        this.worldStateInstrumentation = worldStateInstrumentation;
+        this.log = log;
+        this.cleanup = cleanup;
+    }
+    static async new(rollupAddress, dataDir, dbMapSizeKb, prefilledPublicData = [], instrumentation = new WorldStateInstrumentation(getTelemetryClient()), log = createLogger('world-state:database'), cleanup = ()=>Promise.resolve()) {
+        const worldStateDirectory = join(dataDir, 'world_state');
+        // Create a version manager to handle versioning
+        const versionManager = new DatabaseVersionManager(WORLD_STATE_DB_VERSION, rollupAddress, worldStateDirectory, (dir)=>{
+            return Promise.resolve(new NativeWorldState(dir, dbMapSizeKb, prefilledPublicData, instrumentation));
+        });
+        const [instance] = await versionManager.open();
+        const worldState = new this(instance, instrumentation, log, cleanup);
+        try {
+            await worldState.init();
+        } catch (e) {
+            log.error(`Error initialising world state: ${e}`);
+            throw e;
+        }
+        return worldState;
+    }
+    static async tmp(rollupAddress = EthAddress.ZERO, cleanupTmpDir = true, prefilledPublicData = [], instrumentation = new WorldStateInstrumentation(getTelemetryClient())) {
+        const log = createLogger('world-state:database');
+        const dataDir = await mkdtemp(join(tmpdir(), 'aztec-world-state-'));
+        const dbMapSizeKb = 10 * 1024 * 1024;
+        log.debug(`Created temporary world state database at: ${dataDir} with size: ${dbMapSizeKb}`);
+        // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
+        const cleanup = async ()=>{
+            if (cleanupTmpDir) {
+                await rm(dataDir, {
+                    recursive: true,
+                    force: true,
+                    maxRetries: 3
+                });
+                log.debug(`Deleted temporary world state database: ${dataDir}`);
+            } else {
+                log.debug(`Leaving temporary world state database: ${dataDir}`);
+            }
+        };
+        return this.new(rollupAddress, dataDir, dbMapSizeKb, prefilledPublicData, instrumentation, log, cleanup);
+    }
+    async init() {
+        const status = await this.getStatusSummary();
+        if (!status.treesAreSynched) {
+            throw new Error('World state trees are out of sync, please delete your data directory and re-sync');
+        }
+        this.initialHeader = await this.buildInitialHeader();
+        const committed = this.getCommitted();
+        // validate the initial state
+        const archive = await committed.getTreeInfo(MerkleTreeId.ARCHIVE);
+        if (archive.size === 0n) {
+            throw new Error("Archive tree can't be empty");
+        }
+        // the initial header _must_ be the first element in the archive tree
+        // if this assertion fails, check that the hashing done in Header in yarn-project matches the initial header hash done in world_state.cpp
+        const indices = await committed.findLeafIndices(MerkleTreeId.ARCHIVE, [
+            await this.initialHeader.hash()
+        ]);
+        const initialHeaderIndex = indices[0];
+        assert.strictEqual(initialHeaderIndex, 0n, 'Invalid initial archive state');
+    }
+    getCommitted() {
+        return new MerkleTreesFacade(this.instance, this.initialHeader, worldStateRevision(false, 0, 0));
+    }
+    getSnapshot(blockNumber) {
+        return new MerkleTreesFacade(this.instance, this.initialHeader, worldStateRevision(false, 0, blockNumber));
+    }
+    async fork(blockNumber) {
+        const resp = await this.instance.call(WorldStateMessageType.CREATE_FORK, {
+            latest: blockNumber === undefined,
+            blockNumber: blockNumber ?? 0,
+            canonical: true
+        });
+        return new MerkleTreesForkFacade(this.instance, this.initialHeader, worldStateRevision(true, resp.forkId, 0));
+    }
+    getInitialHeader() {
+        return this.initialHeader;
+    }
+    async handleL2BlockAndMessages(l2Block, l1ToL2Messages) {
+        // We have to pad both the values within tx effects because that's how the trees are built by circuits.
+        const paddedNoteHashes = l2Block.body.txEffects.flatMap((txEffect)=>padArrayEnd(txEffect.noteHashes, Fr.ZERO, MAX_NOTE_HASHES_PER_TX));
+        const paddedL1ToL2Messages = padArrayEnd(l1ToL2Messages, Fr.ZERO, NUMBER_OF_L1_L2_MESSAGES_PER_ROLLUP);
+        const paddedNullifiers = l2Block.body.txEffects.flatMap((txEffect)=>padArrayEnd(txEffect.nullifiers, Fr.ZERO, MAX_NULLIFIERS_PER_TX)).map((nullifier)=>new NullifierLeaf(nullifier));
+        const publicDataWrites = l2Block.body.txEffects.flatMap((txEffect)=>{
+            return txEffect.publicDataWrites.map((write)=>{
+                if (write.isEmpty()) {
+                    throw new Error('Public data write must not be empty when syncing');
+                }
+                return new PublicDataTreeLeaf(write.leafSlot, write.value);
+            });
+        });
+        try {
+            return await this.instance.call(WorldStateMessageType.SYNC_BLOCK, {
+                blockNumber: l2Block.number,
+                blockHeaderHash: await l2Block.header.hash(),
+                paddedL1ToL2Messages: paddedL1ToL2Messages.map(serializeLeaf),
+                paddedNoteHashes: paddedNoteHashes.map(serializeLeaf),
+                paddedNullifiers: paddedNullifiers.map(serializeLeaf),
+                publicDataWrites: publicDataWrites.map(serializeLeaf),
+                blockStateRef: blockStateReference(l2Block.header.state),
+                canonical: true
+            }, this.sanitiseAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this));
+        } catch (err) {
+            this.worldStateInstrumentation.incCriticalErrors('synch_pending_block');
+            throw err;
+        }
+    }
+    async close() {
+        await this.instance.close();
+        await this.cleanup();
+    }
+    async buildInitialHeader() {
+        const state = await this.getInitialStateReference();
+        return BlockHeader.empty({
+            state
+        });
+    }
+    sanitiseAndCacheSummaryFromFull(response) {
+        const sanitised = sanitiseFullStatus(response);
+        this.cachedStatusSummary = {
+            ...sanitised.summary
+        };
+        return sanitised;
+    }
+    sanitiseAndCacheSummary(response) {
+        const sanitised = sanitiseSummary(response);
+        this.cachedStatusSummary = {
+            ...sanitised
+        };
+        return sanitised;
+    }
+    deleteCachedSummary(_) {
+        this.cachedStatusSummary = undefined;
+    }
+    /**
+   * Advances the finalised block number to be the number provided
+   * @param toBlockNumber The block number that is now the tip of the finalised chain
+   * @returns The new WorldStateStatus
+   */ async setFinalised(toBlockNumber) {
+        try {
+            await this.instance.call(WorldStateMessageType.FINALISE_BLOCKS, {
+                toBlockNumber,
+                canonical: true
+            }, this.sanitiseAndCacheSummary.bind(this), this.deleteCachedSummary.bind(this));
+        } catch (err) {
+            this.worldStateInstrumentation.incCriticalErrors('finalize_block');
+            throw err;
+        }
+        return this.getStatusSummary();
+    }
+    /**
+   * Removes all historical snapshots up to but not including the given block number
+   * @param toBlockNumber The block number of the new oldest historical block
+   * @returns The new WorldStateStatus
+   */ async removeHistoricalBlocks(toBlockNumber) {
+        try {
+            return await this.instance.call(WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS, {
+                toBlockNumber,
+                canonical: true
+            }, this.sanitiseAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this));
+        } catch (err) {
+            this.worldStateInstrumentation.incCriticalErrors('prune_historical_block');
+            throw err;
+        }
+    }
+    /**
+   * Removes all pending blocks down to but not including the given block number
+   * @param toBlockNumber The block number of the new tip of the pending chain,
+   * @returns The new WorldStateStatus
+   */ async unwindBlocks(toBlockNumber) {
+        try {
+            return await this.instance.call(WorldStateMessageType.UNWIND_BLOCKS, {
+                toBlockNumber,
+                canonical: true
+            }, this.sanitiseAndCacheSummaryFromFull.bind(this), this.deleteCachedSummary.bind(this));
+        } catch (err) {
+            this.worldStateInstrumentation.incCriticalErrors('prune_pending_block');
+            throw err;
+        }
+    }
+    async getStatusSummary() {
+        if (this.cachedStatusSummary !== undefined) {
+            return {
+                ...this.cachedStatusSummary
+            };
+        }
+        return await this.instance.call(WorldStateMessageType.GET_STATUS, {
+            canonical: true
+        }, this.sanitiseAndCacheSummary.bind(this));
+    }
+    updateLeaf(_treeId, _leaf, _index) {
+        return Promise.reject(new Error('Method not implemented'));
+    }
+    async getInitialStateReference() {
+        const resp = await this.instance.call(WorldStateMessageType.GET_INITIAL_STATE_REFERENCE, {
+            canonical: true
+        });
+        return new StateReference(treeStateReferenceToSnapshot(resp.state[MerkleTreeId.L1_TO_L2_MESSAGE_TREE]), new PartialStateReference(treeStateReferenceToSnapshot(resp.state[MerkleTreeId.NOTE_HASH_TREE]), treeStateReferenceToSnapshot(resp.state[MerkleTreeId.NULLIFIER_TREE]), treeStateReferenceToSnapshot(resp.state[MerkleTreeId.PUBLIC_DATA_TREE])));
+    }
+}

package/dest/native/native_world_state_instance.d.ts

@@ -0,0 +1,33 @@
+import type { PublicDataTreeLeaf } from '@aztec/stdlib/trees';
+import type { WorldStateInstrumentation } from '../instrumentation/instrumentation.js';
+import { WorldStateMessageType, type WorldStateRequest, type WorldStateRequestCategories, type WorldStateResponse } from './message.js';
+export interface NativeWorldStateInstance {
+    call<T extends WorldStateMessageType>(messageType: T, body: WorldStateRequest[T] & WorldStateRequestCategories): Promise<WorldStateResponse[T]>;
+}
+/**
+ * Strongly-typed interface to access the WorldState class in the native world_state_napi module.
+ */
+export declare class NativeWorldState implements NativeWorldStateInstance {
+    private instrumentation;
+    private log;
+    private open;
+    private queues;
+    private instance;
+    /** Creates a new native WorldState instance */
+    constructor(dataDir: string, dbMapSizeKb: number, prefilledPublicData: PublicDataTreeLeaf[] | undefined, instrumentation: WorldStateInstrumentation, log?: import("@aztec/foundation/log").Logger);
+    /**
+     * Sends a message to the native instance and returns the response.
+     * @param messageType - The type of message to send
+     * @param body - The message body
+     * @param responseHandler - A callback accepting the response, executed on the job queue
+     * @param errorHandler - A callback called on request error, executed on the job queue
+     * @returns The response to the message
+     */
+    call<T extends WorldStateMessageType>(messageType: T, body: WorldStateRequest[T] & WorldStateRequestCategories, responseHandler?: (response: WorldStateResponse[T]) => WorldStateResponse[T], errorHandler?: (_: string) => void): Promise<WorldStateResponse[T]>;
+    /**
+     * Stops the native instance.
+     */
+    close(): Promise<void>;
+    private _sendMessage;
+}
+//# sourceMappingURL=native_world_state_instance.d.ts.map

package/dest/native/native_world_state_instance.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"native_world_state_instance.d.ts","sourceRoot":"","sources":["../../src/native/native_world_state_instance.ts"],"names":[],"mappings":"AAaA,OAAO,KAAK,EAAE,kBAAkB,EAAE,MAAM,qBAAqB,CAAC;AAK9D,OAAO,KAAK,EAAE,yBAAyB,EAAE,MAAM,uCAAuC,CAAC;AACvF,OAAO,EACL,qBAAqB,EACrB,KAAK,iBAAiB,EACtB,KAAK,2BAA2B,EAChC,KAAK,kBAAkB,EAIxB,MAAM,cAAc,CAAC;AAKtB,MAAM,WAAW,wBAAwB;IACvC,IAAI,CAAC,CAAC,SAAS,qBAAqB,EAClC,WAAW,EAAE,CAAC,EACd,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,2BAA2B,GACvD,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC,CAAC;CACnC;AAED;;GAEG;AACH,qBAAa,gBAAiB,YAAW,wBAAwB;IAa7D,OAAO,CAAC,eAAe;IACvB,OAAO,CAAC,GAAG;IAbb,OAAO,CAAC,IAAI,CAAQ;IAGpB,OAAO,CAAC,MAAM,CAAyC;IAEvD,OAAO,CAAC,QAAQ,CAA+E;IAE/F,+CAA+C;gBAE7C,OAAO,EAAE,MAAM,EACf,WAAW,EAAE,MAAM,EACnB,mBAAmB,kCAA2B,EACtC,eAAe,EAAE,yBAAyB,EAC1C,GAAG,yCAAuC;IA8BpD;;;;;;;OAOG;IACU,IAAI,CAAC,CAAC,SAAS,qBAAqB,EAC/C,WAAW,EAAE,CAAC,EACd,IAAI,EAAE,iBAAiB,CAAC,CAAC,CAAC,GAAG,2BAA2B,EAExD,eAAe,cAAc,kBAAkB,CAAC,CAAC,CAAC,KAAG,kBAAkB,CAAC,CAAC,CAAa,EACtF,YAAY,OAAO,MAAM,SAAO,GAC/B,OAAO,CAAC,kBAAkB,CAAC,CAAC,CAAC,CAAC;IAuDjC;;OAEG;IACU,KAAK,IAAI,OAAO,CAAC,IAAI,CAAC;YAiBrB,YAAY;CA8D3B"}

package/dest/native/native_world_state_instance.js

@@ -0,0 +1,164 @@
+import { ARCHIVE_HEIGHT, GeneratorIndex, L1_TO_L2_MSG_TREE_HEIGHT, MAX_NULLIFIERS_PER_TX, MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX, NOTE_HASH_TREE_HEIGHT, NULLIFIER_TREE_HEIGHT, PUBLIC_DATA_TREE_HEIGHT } from '@aztec/constants';
+import { createLogger } from '@aztec/foundation/log';
+import { NativeWorldState as BaseNativeWorldState, MsgpackChannel } from '@aztec/native';
+import { MerkleTreeId } from '@aztec/stdlib/trees';
+import assert from 'assert';
+import { cpus } from 'os';
+import { WorldStateMessageType, isWithCanonical, isWithForkId, isWithRevision } from './message.js';
+import { WorldStateOpsQueue } from './world_state_ops_queue.js';
+const MAX_WORLD_STATE_THREADS = +(process.env.HARDWARE_CONCURRENCY || '16');
+/**
+ * Strongly-typed interface to access the WorldState class in the native world_state_napi module.
+ */ export class NativeWorldState {
+    instrumentation;
+    log;
+    open;
+    // We maintain a map of queue to fork
+    queues;
+    instance;
+    /** Creates a new native WorldState instance */ constructor(dataDir, dbMapSizeKb, prefilledPublicData = [], instrumentation, log = createLogger('world-state:database')){
+        this.instrumentation = instrumentation;
+        this.log = log;
+        this.open = true;
+        this.queues = new Map();
+        const threads = Math.min(cpus().length, MAX_WORLD_STATE_THREADS);
+        log.info(`Creating world state data store at directory ${dataDir} with map size ${dbMapSizeKb} KB and ${threads} threads.`);
+        const prefilledPublicDataBufferArray = prefilledPublicData.map((d)=>[
+                d.slot.toBuffer(),
+                d.value.toBuffer()
+            ]);
+        const ws = new BaseNativeWorldState(dataDir, {
+            [MerkleTreeId.NULLIFIER_TREE]: NULLIFIER_TREE_HEIGHT,
+            [MerkleTreeId.NOTE_HASH_TREE]: NOTE_HASH_TREE_HEIGHT,
+            [MerkleTreeId.PUBLIC_DATA_TREE]: PUBLIC_DATA_TREE_HEIGHT,
+            [MerkleTreeId.L1_TO_L2_MESSAGE_TREE]: L1_TO_L2_MSG_TREE_HEIGHT,
+            [MerkleTreeId.ARCHIVE]: ARCHIVE_HEIGHT
+        }, {
+            [MerkleTreeId.NULLIFIER_TREE]: 2 * MAX_NULLIFIERS_PER_TX,
+            [MerkleTreeId.PUBLIC_DATA_TREE]: 2 * MAX_TOTAL_PUBLIC_DATA_UPDATE_REQUESTS_PER_TX
+        }, prefilledPublicDataBufferArray, GeneratorIndex.BLOCK_HASH, dbMapSizeKb, threads);
+        this.instance = new MsgpackChannel(ws);
+        // Manually create the queue for the canonical fork
+        this.queues.set(0, new WorldStateOpsQueue());
+    }
+    /**
+   * Sends a message to the native instance and returns the response.
+   * @param messageType - The type of message to send
+   * @param body - The message body
+   * @param responseHandler - A callback accepting the response, executed on the job queue
+   * @param errorHandler - A callback called on request error, executed on the job queue
+   * @returns The response to the message
+   */ async call(messageType, body, // allows for the pre-processing of responses on the job queue before being passed back
+    responseHandler = (response)=>response, errorHandler = (_)=>{}) {
+        // Here we determine which fork the request is being executed against and whether it requires uncommitted data
+        // We use the fork Id to select the appropriate request queue and the uncommitted data flag to pass to the queue
+        let forkId = -1;
+        // We assume it includes uncommitted unless explicitly told otherwise
+        let committedOnly = false;
+        // Canonical requests ALWAYS go against the canonical fork
+        // These include things like block syncs/unwinds etc
+        // These requests don't contain a fork ID
+        if (isWithCanonical(body)) {
+            forkId = 0;
+        } else if (isWithForkId(body)) {
+            forkId = body.forkId;
+        } else if (isWithRevision(body)) {
+            forkId = body.revision.forkId;
+            committedOnly = body.revision.includeUncommitted === false;
+        } else {
+            const _ = body;
+            throw new Error(`Unable to determine forkId for message=${WorldStateMessageType[messageType]}`);
+        }
+        // Get the queue or create a new one
+        let requestQueue = this.queues.get(forkId);
+        if (requestQueue === undefined) {
+            requestQueue = new WorldStateOpsQueue();
+            this.queues.set(forkId, requestQueue);
+        }
+        // Enqueue the request and wait for the response
+        const response = await requestQueue.execute(async ()=>{
+            assert.notEqual(messageType, WorldStateMessageType.CLOSE, 'Use close() to close the native instance');
+            assert.equal(this.open, true, 'Native instance is closed');
+            let response;
+            try {
+                response = await this._sendMessage(messageType, body);
+            } catch (error) {
+                errorHandler(error.message);
+                throw error;
+            }
+            return responseHandler(response);
+        }, messageType, committedOnly);
+        // If the request was to delete the fork then we clean it up here
+        if (messageType === WorldStateMessageType.DELETE_FORK) {
+            await requestQueue.stop();
+            this.queues.delete(forkId);
+        }
+        return response;
+    }
+    /**
+   * Stops the native instance.
+   */ async close() {
+        if (!this.open) {
+            return;
+        }
+        this.open = false;
+        const queue = this.queues.get(0);
+        await queue.execute(async ()=>{
+            await this._sendMessage(WorldStateMessageType.CLOSE, {
+                canonical: true
+            });
+        }, WorldStateMessageType.CLOSE, false);
+        await queue.stop();
+    }
+    async _sendMessage(messageType, body) {
+        let logMetadata = {};
+        if (body) {
+            if ('treeId' in body) {
+                logMetadata['treeId'] = MerkleTreeId[body.treeId];
+            }
+            if ('revision' in body) {
+                logMetadata = {
+                    ...logMetadata,
+                    ...body.revision
+                };
+            }
+            if ('forkId' in body) {
+                logMetadata['forkId'] = body.forkId;
+            }
+            if ('blockNumber' in body) {
+                logMetadata['blockNumber'] = body.blockNumber;
+            }
+            if ('toBlockNumber' in body) {
+                logMetadata['toBlockNumber'] = body.toBlockNumber;
+            }
+            if ('leafIndex' in body) {
+                logMetadata['leafIndex'] = body.leafIndex;
+            }
+            if ('blockHeaderHash' in body) {
+                logMetadata['blockHeaderHash'] = '0x' + body.blockHeaderHash.toString('hex');
+            }
+            if ('leaves' in body) {
+                logMetadata['leavesCount'] = body.leaves.length;
+            }
+            // sync operation
+            if ('paddedNoteHashes' in body) {
+                logMetadata['notesCount'] = body.paddedNoteHashes.length;
+                logMetadata['nullifiersCount'] = body.paddedNullifiers.length;
+                logMetadata['l1ToL2MessagesCount'] = body.paddedL1ToL2Messages.length;
+                logMetadata['publicDataWritesCount'] = body.publicDataWrites.length;
+            }
+        }
+        try {
+            const { duration, response } = await this.instance.sendMessage(messageType, body);
+            this.log.trace(`Call ${WorldStateMessageType[messageType]} took (ms)`, {
+                duration,
+                ...logMetadata
+            });
+            this.instrumentation.recordRoundTrip(duration.totalUs, messageType);
+            return response;
+        } catch (error) {
+            this.log.error(`Call ${WorldStateMessageType[messageType]} failed: ${error}`, error, logMetadata);
+            throw error;
+        }
+    }
+}

package/dest/native/world_state_ops_queue.d.ts

@@ -0,0 +1,19 @@
+import { WorldStateMessageType } from './message.js';
+export declare const MUTATING_MSG_TYPES: Set<WorldStateMessageType>;
+export declare class WorldStateOpsQueue {
+    private requests;
+    private inFlightMutatingCount;
+    private inFlightCount;
+    private stopPromise?;
+    private stopResolve?;
+    private requestId;
+    private ops;
+    execute(request: () => Promise<any>, messageType: WorldStateMessageType, committedOnly: boolean): Promise<any>;
+    private executeMutating;
+    private executeNonMutatingUncommitted;
+    private executeNonMutatingCommitted;
+    private checkAndEnqueue;
+    private sendEnqueuedRequest;
+    stop(): Promise<void>;
+}
+//# sourceMappingURL=world_state_ops_queue.d.ts.map

package/dest/native/world_state_ops_queue.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"world_state_ops_queue.d.ts","sourceRoot":"","sources":["../../src/native/world_state_ops_queue.ts"],"names":[],"mappings":"AAEA,OAAO,EAAE,qBAAqB,EAAE,MAAM,cAAc,CAAC;AAyBrD,eAAO,MAAM,kBAAkB,4BAgB7B,CAAC;AAGH,qBAAa,kBAAkB;IAC7B,OAAO,CAAC,QAAQ,CAAsB;IACtC,OAAO,CAAC,qBAAqB,CAAK;IAClC,OAAO,CAAC,aAAa,CAAK;IAC1B,OAAO,CAAC,WAAW,CAAC,CAAgB;IACpC,OAAO,CAAC,WAAW,CAAC,CAAa;IACjC,OAAO,CAAC,SAAS,CAAK;IACtB,OAAO,CAAC,GAAG,CAAwC;IAI5C,OAAO,CAAC,OAAO,EAAE,MAAM,OAAO,CAAC,GAAG,CAAC,EAAE,WAAW,EAAE,qBAAqB,EAAE,aAAa,EAAE,OAAO;IAyBtG,OAAO,CAAC,eAAe;IAWvB,OAAO,CAAC,6BAA6B;IAYrC,OAAO,CAAC,2BAA2B;IAUnC,OAAO,CAAC,eAAe;IAyCvB,OAAO,CAAC,mBAAmB;IAgBpB,IAAI;CAiBZ"}

package/dest/native/world_state_ops_queue.js

@@ -0,0 +1,146 @@
+import { promiseWithResolvers } from '@aztec/foundation/promise';
+import { WorldStateMessageType } from './message.js';
+// These are the set of message types that implement mutating operations
+// Messages of these types require exclusive access to their given forks
+export const MUTATING_MSG_TYPES = new Set([
+    WorldStateMessageType.APPEND_LEAVES,
+    WorldStateMessageType.BATCH_INSERT,
+    WorldStateMessageType.SEQUENTIAL_INSERT,
+    WorldStateMessageType.UPDATE_ARCHIVE,
+    WorldStateMessageType.COMMIT,
+    WorldStateMessageType.ROLLBACK,
+    WorldStateMessageType.SYNC_BLOCK,
+    WorldStateMessageType.CREATE_FORK,
+    WorldStateMessageType.DELETE_FORK,
+    WorldStateMessageType.FINALISE_BLOCKS,
+    WorldStateMessageType.UNWIND_BLOCKS,
+    WorldStateMessageType.REMOVE_HISTORICAL_BLOCKS,
+    WorldStateMessageType.CREATE_CHECKPOINT,
+    WorldStateMessageType.COMMIT_CHECKPOINT,
+    WorldStateMessageType.REVERT_CHECKPOINT
+]);
+// This class implements the per-fork operation queue
+export class WorldStateOpsQueue {
+    requests = [];
+    inFlightMutatingCount = 0;
+    inFlightCount = 0;
+    stopPromise;
+    stopResolve;
+    requestId = 0;
+    ops = new Map();
+    // The primary public api, this is where an operation is queued
+    // We return a promise that will ultimately be resolved/rejected with the response/error generated by the 'request' argument
+    execute(request, messageType, committedOnly) {
+        if (this.stopResolve !== undefined) {
+            throw new Error('Unable to send request to world state, queue already stopped');
+        }
+        const op = {
+            requestId: this.requestId++,
+            mutating: MUTATING_MSG_TYPES.has(messageType),
+            request,
+            promise: promiseWithResolvers()
+        };
+        this.ops.set(op.requestId, op);
+        // Perform the appropriate action based upon the queueing rules
+        if (op.mutating) {
+            this.executeMutating(op);
+        } else if (committedOnly === false) {
+            this.executeNonMutatingUncommitted(op);
+        } else {
+            this.executeNonMutatingCommitted(op);
+        }
+        return op.promise.promise;
+    }
+    // Mutating requests need exclusive access
+    executeMutating(op) {
+        // If nothing is in flight then we send the request immediately
+        // Otherwise add to the queue
+        if (this.inFlightCount === 0) {
+            this.sendEnqueuedRequest(op);
+        } else {
+            this.requests.push(op);
+        }
+    }
+    // Non mutating requests including uncommitted state
+    executeNonMutatingUncommitted(op) {
+        // If there are no mutating requests in flight and there is nothing queued
+        // then send the request immediately
+        // If a mutating request is in flight then we must wait
+        // If a mutating request is not in flight but something is queued then it must be a mutating request
+        if (this.inFlightMutatingCount == 0 && this.requests.length == 0) {
+            this.sendEnqueuedRequest(op);
+        } else {
+            this.requests.push(op);
+        }
+    }
+    executeNonMutatingCommitted(op) {
+        // This is a non-mutating request for committed data
+        // It can always be sent
+        op.request().then(op.promise.resolve, op.promise.reject).finally(()=>{
+            this.ops.delete(op.requestId);
+        });
+    }
+    checkAndEnqueue(completedOp) {
+        // As request has completed
+        // First we decrements the relevant in flight counters
+        if (completedOp.mutating) {
+            --this.inFlightMutatingCount;
+        }
+        --this.inFlightCount;
+        // If there are still requests in flight then do nothing further
+        if (this.inFlightCount != 0) {
+            return;
+        }
+        // No requests in flight, send next queued requests
+        // We loop and send:
+        // 1 mutating request if it is next in the queue
+        // As many non-mutating requests as we encounter until
+        // we exhaust the queue or we reach a mutating request
+        while(this.requests.length > 0){
+            const next = this.requests[0];
+            if (next.mutating) {
+                if (this.inFlightCount == 0) {
+                    // send the mutating request
+                    this.requests.shift();
+                    this.sendEnqueuedRequest(next);
+                }
+                break;
+            } else {
+                // not mutating, send and go round again
+                this.requests.shift();
+                this.sendEnqueuedRequest(next);
+            }
+        }
+        // If the queue is empty, there is nothing in flight and we have been told to stop, then resolve the stop promise
+        if (this.inFlightCount == 0 && this.stopResolve !== undefined) {
+            this.stopResolve();
+        }
+    }
+    sendEnqueuedRequest(op) {
+        // Here we increment the in flight counts before sending
+        ++this.inFlightCount;
+        if (op.mutating) {
+            ++this.inFlightMutatingCount;
+        }
+        // Make the request and pass the response/error through to the stored promise
+        op.request().then(op.promise.resolve, op.promise.reject).finally(()=>{
+            this.checkAndEnqueue(op);
+            this.ops.delete(op.requestId);
+        });
+    }
+    stop() {
+        // If there is already a stop promise then return it
+        if (this.stopPromise) {
+            return this.stopPromise;
+        }
+        // Otherwise create a new one and capture the resolve method
+        this.stopPromise = new Promise((resolve)=>{
+            this.stopResolve = resolve;
+        });
+        // If no outstanding requests then immediately resolve the promise
+        if (this.requests.length == 0 && this.inFlightCount == 0 && this.stopResolve !== undefined) {
+            this.stopResolve();
+        }
+        return this.stopPromise;
+    }
+}

package/dest/synchronizer/config.d.ts

@@ -0,0 +1,23 @@
+import { type ConfigMappingsType } from '@aztec/foundation/config';
+/** World State synchronizer configuration values. */
+export interface WorldStateConfig {
+    /** The frequency in which to check. */
+    worldStateBlockCheckIntervalMS: number;
+    /** Whether to follow only the proven chain. */
+    worldStateProvenBlocksOnly: boolean;
+    /** Size of the batch for each get-blocks request from the synchronizer to the archiver. */
+    worldStateBlockRequestBatchSize?: number;
+    /** The map size to be provided to LMDB for each world state tree DB, optional, will inherit from the general dataStoreMapSizeKB if not specified*/
+    worldStateDbMapSizeKb?: number;
+    /** Optional directory for the world state DB, if unspecified will default to the general data directory */
+    worldStateDataDirectory?: string;
+    /** The number of historic blocks to maintain */
+    worldStateBlockHistory: number;
+}
+export declare const worldStateConfigMappings: ConfigMappingsType<WorldStateConfig>;
+/**
+ * Returns the configuration values for the world state synchronizer.
+ * @returns The configuration values for the world state synchronizer.
+ */
+export declare function getWorldStateConfigFromEnv(): WorldStateConfig;
+//# sourceMappingURL=config.d.ts.map

package/dest/synchronizer/config.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../../src/synchronizer/config.ts"],"names":[],"mappings":"AAAA,OAAO,EACL,KAAK,kBAAkB,EAIxB,MAAM,0BAA0B,CAAC;AAElC,qDAAqD;AACrD,MAAM,WAAW,gBAAgB;IAC/B,uCAAuC;IACvC,8BAA8B,EAAE,MAAM,CAAC;IAEvC,+CAA+C;IAC/C,0BAA0B,EAAE,OAAO,CAAC;IAEpC,2FAA2F;IAC3F,+BAA+B,CAAC,EAAE,MAAM,CAAC;IAEzC,mJAAmJ;IACnJ,qBAAqB,CAAC,EAAE,MAAM,CAAC;IAE/B,2GAA2G;IAC3G,uBAAuB,CAAC,EAAE,MAAM,CAAC;IAEjC,gDAAgD;IAChD,sBAAsB,EAAE,MAAM,CAAC;CAChC;AAED,eAAO,MAAM,wBAAwB,EAAE,kBAAkB,CAAC,gBAAgB,CA+BzE,CAAC;AAEF;;;GAGG;AACH,wBAAgB,0BAA0B,IAAI,gBAAgB,CAE7D"}

package/dest/synchronizer/config.js

@@ -0,0 +1,39 @@
+import { booleanConfigHelper, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
+export const worldStateConfigMappings = {
+    worldStateBlockCheckIntervalMS: {
+        env: 'WS_BLOCK_CHECK_INTERVAL_MS',
+        parseEnv: (val)=>+val,
+        defaultValue: 100,
+        description: 'The frequency in which to check.'
+    },
+    worldStateProvenBlocksOnly: {
+        env: 'WS_PROVEN_BLOCKS_ONLY',
+        description: 'Whether to follow only the proven chain.',
+        ...booleanConfigHelper()
+    },
+    worldStateBlockRequestBatchSize: {
+        env: 'WS_BLOCK_REQUEST_BATCH_SIZE',
+        parseEnv: (val)=>val ? +val : undefined,
+        description: 'Size of the batch for each get-blocks request from the synchronizer to the archiver.'
+    },
+    worldStateDbMapSizeKb: {
+        env: 'WS_DB_MAP_SIZE_KB',
+        parseEnv: (val)=>val ? +val : undefined,
+        description: 'The maximum possible size of the world state DB'
+    },
+    worldStateDataDirectory: {
+        env: 'WS_DATA_DIRECTORY',
+        description: 'Optional directory for the world state database'
+    },
+    worldStateBlockHistory: {
+        env: 'WS_NUM_HISTORIC_BLOCKS',
+        description: 'The number of historic blocks to maintain. Values less than 1 mean all history is maintained',
+        ...numberConfigHelper(64)
+    }
+};
+/**
+ * Returns the configuration values for the world state synchronizer.
+ * @returns The configuration values for the world state synchronizer.
+ */ export function getWorldStateConfigFromEnv() {
+    return getConfigFromMappings(worldStateConfigMappings);
+}