@aztec/archiver 0.0.0-test.1 → 0.0.1-commit.0208eb9

package/dest/store/log_store.js

@@ -0,0 +1,436 @@
+import { INITIAL_L2_BLOCK_NUM } from '@aztec/constants';
+import { BlockNumber } from '@aztec/foundation/branded-types';
+import { filterAsync } from '@aztec/foundation/collection';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { createLogger } from '@aztec/foundation/log';
+import { BufferReader, numToUInt32BE } from '@aztec/foundation/serialize';
+import { BlockHash } from '@aztec/stdlib/block';
+import { MAX_LOGS_PER_TAG } from '@aztec/stdlib/interfaces/api-limit';
+import { ContractClassLog, ExtendedContractClassLog, ExtendedPublicLog, LogId, PublicLog, TxScopedL2Log } from '@aztec/stdlib/logs';
+/**
+ * A store for logs
+ */ export class LogStore {
+    db;
+    blockStore;
+    // `tag` --> private logs
+    #privateLogsByTag;
+    // `{contractAddress}_${tag}` --> public logs
+    #publicLogsByContractAndTag;
+    #privateLogKeysByBlock;
+    #publicLogKeysByBlock;
+    #publicLogsByBlock;
+    #contractClassLogsByBlock;
+    #logsMaxPageSize;
+    #log;
+    constructor(db, blockStore, logsMaxPageSize = 1000){
+        this.db = db;
+        this.blockStore = blockStore;
+        this.#log = createLogger('archiver:log_store');
+        this.#privateLogsByTag = db.openMap('archiver_private_tagged_logs_by_tag');
+        this.#publicLogsByContractAndTag = db.openMap('archiver_public_tagged_logs_by_tag');
+        this.#privateLogKeysByBlock = db.openMap('archiver_private_log_keys_by_block');
+        this.#publicLogKeysByBlock = db.openMap('archiver_public_log_keys_by_block');
+        this.#publicLogsByBlock = db.openMap('archiver_public_logs_by_block');
+        this.#contractClassLogsByBlock = db.openMap('archiver_contract_class_logs_by_block');
+        this.#logsMaxPageSize = logsMaxPageSize;
+    }
+    /**
+   * Extracts tagged logs from a single block, grouping them into private and public maps.
+   *
+   * @param block - The L2 block to extract logs from.
+   * @returns An object containing the private and public tagged logs for the block.
+   */ #extractTaggedLogsFromBlock(block) {
+        // SiloedTag (as string) -> array of log buffers.
+        const privateTaggedLogs = new Map();
+        // "{contractAddress}_{tag}" (as string) -> array of log buffers.
+        const publicTaggedLogs = new Map();
+        block.body.txEffects.forEach((txEffect)=>{
+            const txHash = txEffect.txHash;
+            txEffect.privateLogs.forEach((log)=>{
+                // Private logs use SiloedTag (already siloed by kernel)
+                const tag = log.fields[0];
+                this.#log.debug(`Found private log with tag ${tag.toString()} in block ${block.number}`);
+                const currentLogs = privateTaggedLogs.get(tag.toString()) ?? [];
+                currentLogs.push(new TxScopedL2Log(txHash, block.number, block.timestamp, log.getEmittedFields(), txEffect.noteHashes, txEffect.nullifiers[0]).toBuffer());
+                privateTaggedLogs.set(tag.toString(), currentLogs);
+            });
+            txEffect.publicLogs.forEach((log)=>{
+                // Public logs use Tag directly (not siloed) and are stored with contract address
+                const tag = log.fields[0];
+                const contractAddress = log.contractAddress;
+                const key = `${contractAddress.toString()}_${tag.toString()}`;
+                this.#log.debug(`Found public log with tag ${tag.toString()} from contract ${contractAddress.toString()} in block ${block.number}`);
+                const currentLogs = publicTaggedLogs.get(key) ?? [];
+                currentLogs.push(new TxScopedL2Log(txHash, block.number, block.timestamp, log.getEmittedFields(), txEffect.noteHashes, txEffect.nullifiers[0]).toBuffer());
+                publicTaggedLogs.set(key, currentLogs);
+            });
+        });
+        return {
+            privateTaggedLogs,
+            publicTaggedLogs
+        };
+    }
+    /**
+   * Extracts and aggregates tagged logs from a list of blocks.
+   * @param blocks - The blocks to extract logs from.
+   * @returns A map from tag (as string) to an array of serialized private logs belonging to that tag, and a map from
+   * "{contractAddress}_{tag}" (as string) to an array of serialized public logs belonging to that key.
+   */ #extractTaggedLogs(blocks) {
+        const taggedLogsInBlocks = blocks.map((block)=>this.#extractTaggedLogsFromBlock(block));
+        // Now we merge the maps from each block into a single map.
+        const privateTaggedLogs = taggedLogsInBlocks.reduce((acc, { privateTaggedLogs })=>{
+            for (const [tag, logs] of privateTaggedLogs.entries()){
+                const currentLogs = acc.get(tag) ?? [];
+                acc.set(tag, currentLogs.concat(logs));
+            }
+            return acc;
+        }, new Map());
+        const publicTaggedLogs = taggedLogsInBlocks.reduce((acc, { publicTaggedLogs })=>{
+            for (const [key, logs] of publicTaggedLogs.entries()){
+                const currentLogs = acc.get(key) ?? [];
+                acc.set(key, currentLogs.concat(logs));
+            }
+            return acc;
+        }, new Map());
+        return {
+            privateTaggedLogs,
+            publicTaggedLogs
+        };
+    }
+    async #addPrivateLogs(blocks) {
+        const newBlocks = await filterAsync(blocks, async (block)=>!await this.#privateLogKeysByBlock.hasAsync(block.number));
+        const { privateTaggedLogs } = this.#extractTaggedLogs(newBlocks);
+        const keysOfPrivateLogsToUpdate = Array.from(privateTaggedLogs.keys());
+        const currentPrivateTaggedLogs = await Promise.all(keysOfPrivateLogsToUpdate.map(async (key)=>({
+                tag: key,
+                logBuffers: await this.#privateLogsByTag.getAsync(key)
+            })));
+        for (const taggedLogBuffer of currentPrivateTaggedLogs){
+            if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
+                privateTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(privateTaggedLogs.get(taggedLogBuffer.tag)));
+            }
+        }
+        for (const block of newBlocks){
+            const privateTagsInBlock = [];
+            for (const [tag, logs] of privateTaggedLogs.entries()){
+                await this.#privateLogsByTag.set(tag, logs);
+                privateTagsInBlock.push(tag);
+            }
+            await this.#privateLogKeysByBlock.set(block.number, privateTagsInBlock);
+        }
+    }
+    async #addPublicLogs(blocks) {
+        const newBlocks = await filterAsync(blocks, async (block)=>!await this.#publicLogKeysByBlock.hasAsync(block.number));
+        const { publicTaggedLogs } = this.#extractTaggedLogs(newBlocks);
+        const keysOfPublicLogsToUpdate = Array.from(publicTaggedLogs.keys());
+        const currentPublicTaggedLogs = await Promise.all(keysOfPublicLogsToUpdate.map(async (key)=>({
+                tag: key,
+                logBuffers: await this.#publicLogsByContractAndTag.getAsync(key)
+            })));
+        for (const taggedLogBuffer of currentPublicTaggedLogs){
+            if (taggedLogBuffer.logBuffers && taggedLogBuffer.logBuffers.length > 0) {
+                publicTaggedLogs.set(taggedLogBuffer.tag, taggedLogBuffer.logBuffers.concat(publicTaggedLogs.get(taggedLogBuffer.tag)));
+            }
+        }
+        for (const block of newBlocks){
+            const blockHash = await block.hash();
+            const publicTagsInBlock = [];
+            for (const [tag, logs] of publicTaggedLogs.entries()){
+                await this.#publicLogsByContractAndTag.set(tag, logs);
+                publicTagsInBlock.push(tag);
+            }
+            await this.#publicLogKeysByBlock.set(block.number, publicTagsInBlock);
+            const publicLogsInBlock = block.body.txEffects.map((txEffect, txIndex)=>[
+                    numToUInt32BE(txIndex),
+                    numToUInt32BE(txEffect.publicLogs.length),
+                    txEffect.publicLogs.map((log)=>log.toBuffer())
+                ].flat()).flat();
+            await this.#publicLogsByBlock.set(block.number, this.#packWithBlockHash(blockHash, publicLogsInBlock));
+        }
+    }
+    async #addContractClassLogs(blocks) {
+        const newBlocks = await filterAsync(blocks, async (block)=>!await this.#contractClassLogsByBlock.hasAsync(block.number));
+        for (const block of newBlocks){
+            const blockHash = await block.hash();
+            const contractClassLogsInBlock = block.body.txEffects.map((txEffect, txIndex)=>[
+                    numToUInt32BE(txIndex),
+                    numToUInt32BE(txEffect.contractClassLogs.length),
+                    txEffect.contractClassLogs.map((log)=>log.toBuffer())
+                ].flat()).flat();
+            await this.#contractClassLogsByBlock.set(block.number, this.#packWithBlockHash(blockHash, contractClassLogsInBlock));
+        }
+    }
+    /**
+   * Append new logs to the store's list.
+   * @param blocks - The blocks for which to add the logs.
+   * @returns True if the operation is successful.
+   */ addLogs(blocks) {
+        return this.db.transactionAsync(async ()=>{
+            await Promise.all([
+                this.#addPrivateLogs(blocks),
+                this.#addPublicLogs(blocks),
+                this.#addContractClassLogs(blocks)
+            ]);
+            return true;
+        });
+    }
+    #packWithBlockHash(blockHash, data) {
+        return Buffer.concat([
+            blockHash.toBuffer(),
+            ...data
+        ]);
+    }
+    #unpackBlockHash(reader) {
+        const blockHash = reader.remainingBytes() > 0 ? reader.readObject(Fr) : undefined;
+        if (!blockHash) {
+            throw new Error('Failed to read block hash from log entry buffer');
+        }
+        return new BlockHash(blockHash);
+    }
+    deleteLogs(blocks) {
+        return this.db.transactionAsync(async ()=>{
+            await Promise.all(blocks.map(async (block)=>{
+                // Delete private logs
+                const privateKeys = await this.#privateLogKeysByBlock.getAsync(block.number) ?? [];
+                await Promise.all(privateKeys.map((tag)=>this.#privateLogsByTag.delete(tag)));
+                // Delete public logs
+                const publicKeys = await this.#publicLogKeysByBlock.getAsync(block.number) ?? [];
+                await Promise.all(publicKeys.map((key)=>this.#publicLogsByContractAndTag.delete(key)));
+            }));
+            await Promise.all(blocks.map((block)=>Promise.all([
+                    this.#publicLogsByBlock.delete(block.number),
+                    this.#privateLogKeysByBlock.delete(block.number),
+                    this.#publicLogKeysByBlock.delete(block.number),
+                    this.#contractClassLogsByBlock.delete(block.number)
+                ])));
+            return true;
+        });
+    }
+    /**
+   * Gets private logs that match any of the `tags`. For each tag, an array of matching logs is returned. An empty
+   * array implies no logs match that tag.
+   * @param tags - The tags to search for.
+   * @param page - The page number (0-indexed) for pagination.
+   * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
+   * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
+   */ async getPrivateLogsByTags(tags, page = 0) {
+        const logs = await Promise.all(tags.map((tag)=>this.#privateLogsByTag.getAsync(tag.toString())));
+        const start = page * MAX_LOGS_PER_TAG;
+        const end = start + MAX_LOGS_PER_TAG;
+        return logs.map((logBuffers)=>logBuffers?.slice(start, end).map((logBuffer)=>TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
+    }
+    /**
+   * Gets public logs that match any of the `tags` from the specified contract. For each tag, an array of matching
+   * logs is returned. An empty array implies no logs match that tag.
+   * @param contractAddress - The contract address to search logs for.
+   * @param tags - The tags to search for.
+   * @param page - The page number (0-indexed) for pagination.
+   * @returns An array of log arrays, one per tag. Returns at most MAX_LOGS_PER_TAG logs per tag per page. If
+   * MAX_LOGS_PER_TAG logs are returned for a tag, the caller should fetch the next page to check for more logs.
+   */ async getPublicLogsByTagsFromContract(contractAddress, tags, page = 0) {
+        const logs = await Promise.all(tags.map((tag)=>{
+            const key = `${contractAddress.toString()}_${tag.value.toString()}`;
+            return this.#publicLogsByContractAndTag.getAsync(key);
+        }));
+        const start = page * MAX_LOGS_PER_TAG;
+        const end = start + MAX_LOGS_PER_TAG;
+        return logs.map((logBuffers)=>logBuffers?.slice(start, end).map((logBuffer)=>TxScopedL2Log.fromBuffer(logBuffer)) ?? []);
+    }
+    /**
+   * Gets public logs based on the provided filter.
+   * @param filter - The filter to apply to the logs.
+   * @returns The requested logs.
+   */ getPublicLogs(filter) {
+        if (filter.afterLog) {
+            return this.#filterPublicLogsBetweenBlocks(filter);
+        } else if (filter.txHash) {
+            return this.#filterPublicLogsOfTx(filter);
+        } else {
+            return this.#filterPublicLogsBetweenBlocks(filter);
+        }
+    }
+    async #filterPublicLogsOfTx(filter) {
+        if (!filter.txHash) {
+            throw new Error('Missing txHash');
+        }
+        const [blockNumber, txIndex] = await this.blockStore.getTxLocation(filter.txHash) ?? [];
+        if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') {
+            return {
+                logs: [],
+                maxLogsHit: false
+            };
+        }
+        const buffer = await this.#publicLogsByBlock.getAsync(blockNumber) ?? Buffer.alloc(0);
+        const publicLogsInBlock = [
+            []
+        ];
+        const reader = new BufferReader(buffer);
+        const blockHash = this.#unpackBlockHash(reader);
+        while(reader.remainingBytes() > 0){
+            const indexOfTx = reader.readNumber();
+            const numLogsInTx = reader.readNumber();
+            publicLogsInBlock[indexOfTx] = [];
+            for(let i = 0; i < numLogsInTx; i++){
+                publicLogsInBlock[indexOfTx].push(reader.readObject(PublicLog));
+            }
+        }
+        const txLogs = publicLogsInBlock[txIndex];
+        const logs = [];
+        const maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+        return {
+            logs,
+            maxLogsHit
+        };
+    }
+    async #filterPublicLogsBetweenBlocks(filter) {
+        const start = filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM);
+        const end = filter.toBlock;
+        if (typeof end === 'number' && end < start) {
+            return {
+                logs: [],
+                maxLogsHit: true
+            };
+        }
+        const logs = [];
+        let maxLogsHit = false;
+        loopOverBlocks: for await (const [blockNumber, logBuffer] of this.#publicLogsByBlock.entriesAsync({
+            start,
+            end
+        })){
+            const publicLogsInBlock = [
+                []
+            ];
+            const reader = new BufferReader(logBuffer);
+            const blockHash = this.#unpackBlockHash(reader);
+            while(reader.remainingBytes() > 0){
+                const indexOfTx = reader.readNumber();
+                const numLogsInTx = reader.readNumber();
+                publicLogsInBlock[indexOfTx] = [];
+                for(let i = 0; i < numLogsInTx; i++){
+                    publicLogsInBlock[indexOfTx].push(reader.readObject(PublicLog));
+                }
+            }
+            for(let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < publicLogsInBlock.length; txIndex++){
+                const txLogs = publicLogsInBlock[txIndex];
+                maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+                if (maxLogsHit) {
+                    this.#log.debug(`Max logs hit at block ${blockNumber}`);
+                    break loopOverBlocks;
+                }
+            }
+        }
+        return {
+            logs,
+            maxLogsHit
+        };
+    }
+    /**
+   * Gets contract class logs based on the provided filter.
+   * @param filter - The filter to apply to the logs.
+   * @returns The requested logs.
+   */ getContractClassLogs(filter) {
+        if (filter.afterLog) {
+            return this.#filterContractClassLogsBetweenBlocks(filter);
+        } else if (filter.txHash) {
+            return this.#filterContractClassLogsOfTx(filter);
+        } else {
+            return this.#filterContractClassLogsBetweenBlocks(filter);
+        }
+    }
+    async #filterContractClassLogsOfTx(filter) {
+        if (!filter.txHash) {
+            throw new Error('Missing txHash');
+        }
+        const [blockNumber, txIndex] = await this.blockStore.getTxLocation(filter.txHash) ?? [];
+        if (typeof blockNumber !== 'number' || typeof txIndex !== 'number') {
+            return {
+                logs: [],
+                maxLogsHit: false
+            };
+        }
+        const contractClassLogsBuffer = await this.#contractClassLogsByBlock.getAsync(blockNumber) ?? Buffer.alloc(0);
+        const contractClassLogsInBlock = [
+            []
+        ];
+        const reader = new BufferReader(contractClassLogsBuffer);
+        const blockHash = this.#unpackBlockHash(reader);
+        while(reader.remainingBytes() > 0){
+            const indexOfTx = reader.readNumber();
+            const numLogsInTx = reader.readNumber();
+            contractClassLogsInBlock[indexOfTx] = [];
+            for(let i = 0; i < numLogsInTx; i++){
+                contractClassLogsInBlock[indexOfTx].push(reader.readObject(ContractClassLog));
+            }
+        }
+        const txLogs = contractClassLogsInBlock[txIndex];
+        const logs = [];
+        const maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+        return {
+            logs,
+            maxLogsHit
+        };
+    }
+    async #filterContractClassLogsBetweenBlocks(filter) {
+        const start = filter.afterLog?.blockNumber ?? Math.max(filter.fromBlock ?? INITIAL_L2_BLOCK_NUM, INITIAL_L2_BLOCK_NUM);
+        const end = filter.toBlock;
+        if (typeof end === 'number' && end < start) {
+            return {
+                logs: [],
+                maxLogsHit: true
+            };
+        }
+        const logs = [];
+        let maxLogsHit = false;
+        loopOverBlocks: for await (const [blockNumber, logBuffer] of this.#contractClassLogsByBlock.entriesAsync({
+            start,
+            end
+        })){
+            const contractClassLogsInBlock = [
+                []
+            ];
+            const reader = new BufferReader(logBuffer);
+            const blockHash = this.#unpackBlockHash(reader);
+            while(reader.remainingBytes() > 0){
+                const indexOfTx = reader.readNumber();
+                const numLogsInTx = reader.readNumber();
+                contractClassLogsInBlock[indexOfTx] = [];
+                for(let i = 0; i < numLogsInTx; i++){
+                    contractClassLogsInBlock[indexOfTx].push(reader.readObject(ContractClassLog));
+                }
+            }
+            for(let txIndex = filter.afterLog?.txIndex ?? 0; txIndex < contractClassLogsInBlock.length; txIndex++){
+                const txLogs = contractClassLogsInBlock[txIndex];
+                maxLogsHit = this.#accumulateLogs(logs, blockNumber, blockHash, txIndex, txLogs, filter);
+                if (maxLogsHit) {
+                    this.#log.debug(`Max logs hit at block ${blockNumber}`);
+                    break loopOverBlocks;
+                }
+            }
+        }
+        return {
+            logs,
+            maxLogsHit
+        };
+    }
+    #accumulateLogs(results, blockNumber, blockHash, txIndex, txLogs, filter = {}) {
+        let maxLogsHit = false;
+        let logIndex = typeof filter.afterLog?.logIndex === 'number' ? filter.afterLog.logIndex + 1 : 0;
+        for(; logIndex < txLogs.length; logIndex++){
+            const log = txLogs[logIndex];
+            if (!filter.contractAddress || log.contractAddress.equals(filter.contractAddress)) {
+                if (log instanceof ContractClassLog) {
+                    results.push(new ExtendedContractClassLog(new LogId(BlockNumber(blockNumber), blockHash, txIndex, logIndex), log));
+                } else if (log instanceof PublicLog) {
+                    results.push(new ExtendedPublicLog(new LogId(BlockNumber(blockNumber), blockHash, txIndex, logIndex), log));
+                } else {
+                    throw new Error('Unknown log type');
+                }
+                if (results.length >= this.#logsMaxPageSize) {
+                    maxLogsHit = true;
+                    break;
+                }
+            }
+        }
+        return maxLogsHit;
+    }
+}

package/dest/store/message_store.d.ts

@@ -0,0 +1,40 @@
+import type { L1BlockId } from '@aztec/ethereum/l1-types';
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { Fr } from '@aztec/foundation/curves/bn254';
+import { type AztecAsyncKVStore, type CustomRange } from '@aztec/kv-store';
+import { type InboxMessage } from '../structs/inbox_message.js';
+export declare class MessageStoreError extends Error {
+    readonly inboxMessage: InboxMessage;
+    constructor(message: string, inboxMessage: InboxMessage);
+}
+export declare class MessageStore {
+    #private;
+    private db;
+    constructor(db: AztecAsyncKVStore);
+    getTotalL1ToL2MessageCount(): Promise<bigint>;
+    /** Gets the last L1 block synced. */
+    getSynchedL1Block(): Promise<L1BlockId | undefined>;
+    /** Sets the last L1 block synced */
+    setSynchedL1Block(l1Block: L1BlockId): Promise<void>;
+    /**
+     * Append L1 to L2 messages to the store.
+     * Requires new messages to be in order and strictly after the last message added.
+     * Throws if out of order messages are added or if the rolling hash is invalid.
+     */
+    addL1ToL2Messages(messages: InboxMessage[]): Promise<void>;
+    /**
+     * Gets the L1 to L2 message index in the L1 to L2 message tree.
+     * @param l1ToL2Message - The L1 to L2 message.
+     * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
+     */
+    getL1ToL2MessageIndex(l1ToL2Message: Fr): Promise<bigint | undefined>;
+    getLastMessage(): Promise<InboxMessage | undefined>;
+    getL1ToL2Messages(checkpointNumber: CheckpointNumber): Promise<Fr[]>;
+    iterateL1ToL2Messages(range?: CustomRange<bigint>): AsyncIterableIterator<InboxMessage>;
+    removeL1ToL2Messages(startIndex: bigint): Promise<void>;
+    rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber: CheckpointNumber): Promise<void>;
+    private indexToKey;
+    private leafToIndexKey;
+    private increaseTotalMessageCount;
+}
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoibWVzc2FnZV9zdG9yZS5kLnRzIiwic291cmNlUm9vdCI6IiIsInNvdXJjZXMiOlsiLi4vLi4vc3JjL3N0b3JlL21lc3NhZ2Vfc3RvcmUudHMiXSwibmFtZXMiOltdLCJtYXBwaW5ncyI6IkFBQUEsT0FBTyxLQUFLLEVBQUUsU0FBUyxFQUFFLE1BQU0sMEJBQTBCLENBQUM7QUFDMUQsT0FBTyxFQUFFLGdCQUFnQixFQUFFLE1BQU0saUNBQWlDLENBQUM7QUFFbkUsT0FBTyxFQUFFLEVBQUUsRUFBRSxNQUFNLGdDQUFnQyxDQUFDO0FBSXBELE9BQU8sRUFDTCxLQUFLLGlCQUFpQixFQUd0QixLQUFLLFdBQVcsRUFFakIsTUFBTSxpQkFBaUIsQ0FBQztBQUd6QixPQUFPLEVBQ0wsS0FBSyxZQUFZLEVBSWxCLE1BQU0sNkJBQTZCLENBQUM7QUFFckMscUJBQWEsaUJBQWtCLFNBQVEsS0FBSzthQUd4QixZQUFZLEVBQUUsWUFBWTtJQUY1QyxZQUNFLE9BQU8sRUFBRSxNQUFNLEVBQ0MsWUFBWSxFQUFFLFlBQVksRUFJM0M7Q0FDRjtBQUVELHFCQUFhLFlBQVk7O0lBWVgsT0FBTyxDQUFDLEVBQUU7SUFBdEIsWUFBb0IsRUFBRSxFQUFFLGlCQUFpQixFQUt4QztJQUVZLDBCQUEwQixJQUFJLE9BQU8sQ0FBQyxNQUFNLENBQUMsQ0FFekQ7SUFFRCxxQ0FBcUM7SUFDeEIsaUJBQWlCLElBQUksT0FBTyxDQUFDLFNBQVMsR0FBRyxTQUFTLENBQUMsQ0FRL0Q7SUFFRCxvQ0FBb0M7SUFDdkIsaUJBQWlCLENBQUMsT0FBTyxFQUFFLFNBQVMsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBR2hFO0lBRUQ7Ozs7T0FJRztJQUNJLGlCQUFpQixDQUFDLFFBQVEsRUFBRSxZQUFZLEVBQUUsR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBNkZoRTtJQUVEOzs7O09BSUc7SUFDSSxxQkFBcUIsQ0FBQyxhQUFhLEVBQUUsRUFBRSxHQUFHLE9BQU8sQ0FBQyxNQUFNLEdBQUcsU0FBUyxDQUFDLENBRTNFO0lBRVksY0FBYyxJQUFJLE9BQU8sQ0FBQyxZQUFZLEdBQUcsU0FBUyxDQUFDLENBRy9EO0lBRVksaUJBQWlCLENBQUMsZ0JBQWdCLEVBQUUsZ0JBQWdCLEdBQUcsT0FBTyxDQUFDLEVBQUUsRUFBRSxDQUFDLENBdUJoRjtJQUVhLHFCQUFxQixDQUFDLEtBQUssR0FBRSxXQUFXLENBQUMsTUFBTSxDQUFNLEdBQUcscUJBQXFCLENBQUMsWUFBWSxDQUFDLENBS3hHO0lBRU0sb0JBQW9CLENBQUMsVUFBVSxFQUFFLE1BQU0sR0FBRyxPQUFPLENBQUMsSUFBSSxDQUFDLENBZ0I3RDtJQUVNLGtDQUFrQyxDQUFDLHNCQUFzQixFQUFFLGdCQUFnQixHQUFHLE9BQU8sQ0FBQyxJQUFJLENBQUMsQ0FJakc7SUFFRCxPQUFPLENBQUMsVUFBVTtJQUlsQixPQUFPLENBQUMsY0FBYztZQUlSLHlCQUF5QjtDQVN4QyJ9

package/dest/store/message_store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"message_store.d.ts","sourceRoot":"","sources":["../../src/store/message_store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAAE,SAAS,EAAE,MAAM,0BAA0B,CAAC;AAC1D,OAAO,EAAE,gBAAgB,EAAE,MAAM,iCAAiC,CAAC;AAEnE,OAAO,EAAE,EAAE,EAAE,MAAM,gCAAgC,CAAC;AAIpD,OAAO,EACL,KAAK,iBAAiB,EAGtB,KAAK,WAAW,EAEjB,MAAM,iBAAiB,CAAC;AAGzB,OAAO,EACL,KAAK,YAAY,EAIlB,MAAM,6BAA6B,CAAC;AAErC,qBAAa,iBAAkB,SAAQ,KAAK;aAGxB,YAAY,EAAE,YAAY;IAF5C,YACE,OAAO,EAAE,MAAM,EACC,YAAY,EAAE,YAAY,EAI3C;CACF;AAED,qBAAa,YAAY;;IAYX,OAAO,CAAC,EAAE;IAAtB,YAAoB,EAAE,EAAE,iBAAiB,EAKxC;IAEY,0BAA0B,IAAI,OAAO,CAAC,MAAM,CAAC,CAEzD;IAED,qCAAqC;IACxB,iBAAiB,IAAI,OAAO,CAAC,SAAS,GAAG,SAAS,CAAC,CAQ/D;IAED,oCAAoC;IACvB,iBAAiB,CAAC,OAAO,EAAE,SAAS,GAAG,OAAO,CAAC,IAAI,CAAC,CAGhE;IAED;;;;OAIG;IACI,iBAAiB,CAAC,QAAQ,EAAE,YAAY,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC,CA6FhE;IAED;;;;OAIG;IACI,qBAAqB,CAAC,aAAa,EAAE,EAAE,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC,CAE3E;IAEY,cAAc,IAAI,OAAO,CAAC,YAAY,GAAG,SAAS,CAAC,CAG/D;IAEY,iBAAiB,CAAC,gBAAgB,EAAE,gBAAgB,GAAG,OAAO,CAAC,EAAE,EAAE,CAAC,CAuBhF;IAEa,qBAAqB,CAAC,KAAK,GAAE,WAAW,CAAC,MAAM,CAAM,GAAG,qBAAqB,CAAC,YAAY,CAAC,CAKxG;IAEM,oBAAoB,CAAC,UAAU,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC,CAgB7D;IAEM,kCAAkC,CAAC,sBAAsB,EAAE,gBAAgB,GAAG,OAAO,CAAC,IAAI,CAAC,CAIjG;IAED,OAAO,CAAC,UAAU;IAIlB,OAAO,CAAC,cAAc;YAIR,yBAAyB;CASxC"}

package/dest/store/message_store.js

@@ -0,0 +1,188 @@
+import { CheckpointNumber } from '@aztec/foundation/branded-types';
+import { Buffer16, Buffer32 } from '@aztec/foundation/buffer';
+import { toArray } from '@aztec/foundation/iterable';
+import { createLogger } from '@aztec/foundation/log';
+import { BufferReader, serializeToBuffer } from '@aztec/foundation/serialize';
+import { mapRange } from '@aztec/kv-store';
+import { InboxLeaf } from '@aztec/stdlib/messaging';
+import { deserializeInboxMessage, serializeInboxMessage, updateRollingHash } from '../structs/inbox_message.js';
+export class MessageStoreError extends Error {
+    inboxMessage;
+    constructor(message, inboxMessage){
+        super(message), this.inboxMessage = inboxMessage;
+        this.name = 'MessageStoreError';
+    }
+}
+export class MessageStore {
+    db;
+    /** Maps from message index to serialized InboxMessage */ #l1ToL2Messages;
+    /** Maps from hex-stringified message leaf to its index */ #l1ToL2MessageIndices;
+    /** Stores L1 block number and hash of the L1 synchpoint */ #lastSynchedL1Block;
+    /** Stores total messages stored */ #totalMessageCount;
+    #log;
+    constructor(db){
+        this.db = db;
+        this.#log = createLogger('archiver:message_store');
+        this.#l1ToL2Messages = db.openMap('archiver_l1_to_l2_messages');
+        this.#l1ToL2MessageIndices = db.openMap('archiver_l1_to_l2_message_indices');
+        this.#lastSynchedL1Block = db.openSingleton('archiver_last_l1_block_id');
+        this.#totalMessageCount = db.openSingleton('archiver_l1_to_l2_message_count');
+    }
+    async getTotalL1ToL2MessageCount() {
+        return await this.#totalMessageCount.getAsync() ?? 0n;
+    }
+    /** Gets the last L1 block synced. */ async getSynchedL1Block() {
+        const buffer = await this.#lastSynchedL1Block.getAsync();
+        if (!buffer) {
+            return undefined;
+        }
+        const reader = BufferReader.asReader(buffer);
+        return {
+            l1BlockNumber: reader.readUInt256(),
+            l1BlockHash: Buffer32.fromBuffer(reader.readBytes(Buffer32.SIZE))
+        };
+    }
+    /** Sets the last L1 block synced */ async setSynchedL1Block(l1Block) {
+        const buffer = serializeToBuffer([
+            l1Block.l1BlockNumber,
+            l1Block.l1BlockHash
+        ]);
+        await this.#lastSynchedL1Block.set(buffer);
+    }
+    /**
+   * Append L1 to L2 messages to the store.
+   * Requires new messages to be in order and strictly after the last message added.
+   * Throws if out of order messages are added or if the rolling hash is invalid.
+   */ addL1ToL2Messages(messages) {
+        if (messages.length === 0) {
+            return Promise.resolve();
+        }
+        return this.db.transactionAsync(async ()=>{
+            let lastMessage = await this.getLastMessage();
+            let messageCount = 0;
+            for (const message of messages){
+                // Check messages are inserted in increasing order, but allow reinserting messages.
+                if (lastMessage && message.index <= lastMessage.index) {
+                    const existing = await this.#l1ToL2Messages.getAsync(this.indexToKey(message.index));
+                    if (existing && deserializeInboxMessage(existing).rollingHash.equals(message.rollingHash)) {
+                        // We reinsert instead of skipping in case the message was re-orged and got added in a different L1 block.
+                        this.#log.trace(`Reinserting message with index ${message.index} in the store`);
+                        await this.#l1ToL2Messages.set(this.indexToKey(message.index), serializeInboxMessage(message));
+                        continue;
+                    }
+                    throw new MessageStoreError(`Cannot insert L1 to L2 message with index ${message.index} before last message with index ${lastMessage.index}`, message);
+                }
+                // Check rolling hash is valid.
+                const previousRollingHash = lastMessage?.rollingHash ?? Buffer16.ZERO;
+                const expectedRollingHash = updateRollingHash(previousRollingHash, message.leaf);
+                if (!expectedRollingHash.equals(message.rollingHash)) {
+                    throw new MessageStoreError(`Invalid rolling hash for incoming L1 to L2 message ${message.leaf.toString()} ` + `with index ${message.index} ` + `(expected ${expectedRollingHash.toString()} from previous hash ${previousRollingHash} but got ${message.rollingHash.toString()})`, message);
+                }
+                // Check index corresponds to the checkpoint number.
+                const [expectedStart, expectedEnd] = InboxLeaf.indexRangeForCheckpoint(message.checkpointNumber);
+                if (message.index < expectedStart || message.index >= expectedEnd) {
+                    throw new MessageStoreError(`Invalid index ${message.index} for incoming L1 to L2 message ${message.leaf.toString()} ` + `at checkpoint ${message.checkpointNumber} (expected value in range [${expectedStart}, ${expectedEnd}))`, message);
+                }
+                // Check there are no gaps in the indices within the same checkpoint.
+                if (lastMessage && message.checkpointNumber === lastMessage.checkpointNumber && message.index !== lastMessage.index + 1n) {
+                    throw new MessageStoreError(`Missing prior message for incoming L1 to L2 message ${message.leaf.toString()} ` + `with index ${message.index}`, message);
+                }
+                // Check the first message in a block has the correct index.
+                if ((!lastMessage || message.checkpointNumber > lastMessage.checkpointNumber) && message.index !== expectedStart) {
+                    throw new MessageStoreError(`Message ${message.leaf.toString()} for checkpoint ${message.checkpointNumber} has wrong index ` + `${message.index} (expected ${expectedStart})`, message);
+                }
+                // Perform the insertions.
+                await this.#l1ToL2Messages.set(this.indexToKey(message.index), serializeInboxMessage(message));
+                await this.#l1ToL2MessageIndices.set(this.leafToIndexKey(message.leaf), message.index);
+                messageCount++;
+                this.#log.trace(`Inserted L1 to L2 message ${message.leaf} with index ${message.index} into the store`);
+                lastMessage = message;
+            }
+            // Update the L1 sync point to that of the last message added.
+            const currentSyncPoint = await this.getSynchedL1Block();
+            if (!currentSyncPoint || currentSyncPoint.l1BlockNumber < lastMessage.l1BlockNumber) {
+                await this.setSynchedL1Block({
+                    l1BlockNumber: lastMessage.l1BlockNumber,
+                    l1BlockHash: lastMessage.l1BlockHash
+                });
+            }
+            // Update total message count with the number of inserted messages.
+            await this.increaseTotalMessageCount(messageCount);
+        });
+    }
+    /**
+   * Gets the L1 to L2 message index in the L1 to L2 message tree.
+   * @param l1ToL2Message - The L1 to L2 message.
+   * @returns The index of the L1 to L2 message in the L1 to L2 message tree (undefined if not found).
+   */ getL1ToL2MessageIndex(l1ToL2Message) {
+        return this.#l1ToL2MessageIndices.getAsync(this.leafToIndexKey(l1ToL2Message));
+    }
+    async getLastMessage() {
+        const [msg] = await toArray(this.#l1ToL2Messages.valuesAsync({
+            reverse: true,
+            limit: 1
+        }));
+        return msg ? deserializeInboxMessage(msg) : undefined;
+    }
+    async getL1ToL2Messages(checkpointNumber) {
+        const messages = [];
+        const [startIndex, endIndex] = InboxLeaf.indexRangeForCheckpoint(checkpointNumber);
+        let lastIndex = startIndex - 1n;
+        for await (const msgBuffer of this.#l1ToL2Messages.valuesAsync({
+            start: this.indexToKey(startIndex),
+            end: this.indexToKey(endIndex)
+        })){
+            const msg = deserializeInboxMessage(msgBuffer);
+            if (msg.checkpointNumber !== checkpointNumber) {
+                throw new Error(`L1 to L2 message with index ${msg.index} has invalid checkpoint number ${msg.checkpointNumber}`);
+            } else if (msg.index !== lastIndex + 1n) {
+                throw new Error(`Expected L1 to L2 message with index ${lastIndex + 1n} but got ${msg.index}`);
+            }
+            lastIndex = msg.index;
+            messages.push(msg.leaf);
+        }
+        return messages;
+    }
+    async *iterateL1ToL2Messages(range = {}) {
+        const entriesRange = mapRange(range, this.indexToKey);
+        for await (const msgBuffer of this.#l1ToL2Messages.valuesAsync(entriesRange)){
+            yield deserializeInboxMessage(msgBuffer);
+        }
+    }
+    removeL1ToL2Messages(startIndex) {
+        this.#log.debug(`Deleting L1 to L2 messages from index ${startIndex}`);
+        let deleteCount = 0;
+        return this.db.transactionAsync(async ()=>{
+            for await (const [key, msgBuffer] of this.#l1ToL2Messages.entriesAsync({
+                start: this.indexToKey(startIndex)
+            })){
+                this.#log.trace(`Deleting L1 to L2 message with index ${key - 1} from the store`);
+                await this.#l1ToL2Messages.delete(key);
+                await this.#l1ToL2MessageIndices.delete(this.leafToIndexKey(deserializeInboxMessage(msgBuffer).leaf));
+                deleteCount++;
+            }
+            await this.increaseTotalMessageCount(-deleteCount);
+            this.#log.warn(`Deleted ${deleteCount} L1 to L2 messages from index ${startIndex} from the store`);
+        });
+    }
+    rollbackL1ToL2MessagesToCheckpoint(targetCheckpointNumber) {
+        this.#log.debug(`Deleting L1 to L2 messages up to target checkpoint ${targetCheckpointNumber}`);
+        const startIndex = InboxLeaf.smallestIndexForCheckpoint(CheckpointNumber(targetCheckpointNumber + 1));
+        return this.removeL1ToL2Messages(startIndex);
+    }
+    indexToKey(index) {
+        return Number(index);
+    }
+    leafToIndexKey(leaf) {
+        return leaf.toString();
+    }
+    async increaseTotalMessageCount(count) {
+        if (count === 0) {
+            return;
+        }
+        return await this.db.transactionAsync(async ()=>{
+            const lastTotalMessageCount = await this.getTotalL1ToL2MessageCount();
+            await this.#totalMessageCount.set(lastTotalMessageCount + BigInt(count));
+        });
+    }
+}