@aztec/kv-store 0.0.0-test.0

package/dest/lmdb-v2/store.js

@@ -0,0 +1,181 @@
+import { createLogger } from '@aztec/foundation/log';
+import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
+import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
+import { AsyncLocalStorage } from 'async_hooks';
+import { rm } from 'fs/promises';
+import { LMDBArray } from './array.js';
+import { LMDBMap, LMDBMultiMap } from './map.js';
+import { Database, LMDBMessageType } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+import { LMDBSingleValue } from './singleton.js';
+import { WriteTransaction } from './write_transaction.js';
+export class AztecLMDBStoreV2 {
+    dataDir;
+    log;
+    cleanup;
+    open;
+    channel;
+    writerCtx;
+    writerQueue;
+    availableCursors;
+    constructor(dataDir, mapSize, maxReaders, log, cleanup){
+        this.dataDir = dataDir;
+        this.log = log;
+        this.cleanup = cleanup;
+        this.open = false;
+        this.writerCtx = new AsyncLocalStorage();
+        this.writerQueue = new SerialQueue();
+        this.log.info(`Starting data store with maxReaders ${maxReaders}`);
+        this.channel = new MsgpackChannel(new NativeLMDBStore(dataDir, mapSize, maxReaders));
+        // leave one reader to always be available for regular, atomic, reads
+        this.availableCursors = new Semaphore(maxReaders - 1);
+    }
+    get dataDirectory() {
+        return this.dataDir;
+    }
+    async start() {
+        this.writerQueue.start();
+        await this.channel.sendMessage(LMDBMessageType.OPEN_DATABASE, {
+            db: Database.DATA,
+            uniqueKeys: true
+        });
+        await this.channel.sendMessage(LMDBMessageType.OPEN_DATABASE, {
+            db: Database.INDEX,
+            uniqueKeys: false
+        });
+        this.open = true;
+    }
+    static async new(dataDir, dbMapSizeKb = 10 * 1024 * 1024, maxReaders = 16, cleanup, log = createLogger('kv-store:lmdb-v2')) {
+        const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup);
+        await db.start();
+        return db;
+    }
+    getReadTx() {
+        if (!this.open) {
+            throw new Error('Store is closed');
+        }
+        return new ReadTransaction(this);
+    }
+    getCurrentWriteTx() {
+        if (!this.open) {
+            throw new Error('Store is closed');
+        }
+        const currentWrite = this.writerCtx.getStore();
+        return currentWrite;
+    }
+    openMap(name) {
+        return new LMDBMap(this, name);
+    }
+    openMultiMap(name) {
+        return new LMDBMultiMap(this, name);
+    }
+    openSingleton(name) {
+        return new LMDBSingleValue(this, name);
+    }
+    openArray(name) {
+        return new LMDBArray(this, name);
+    }
+    openSet(_name) {
+        throw new Error('Not implemented');
+    }
+    openCounter(_name) {
+        throw new Error('Not implemented');
+    }
+    async transactionAsync(callback) {
+        if (!this.open) {
+            throw new Error('Store is closed');
+        }
+        // transactionAsync might be called recursively
+        // send any writes to the parent tx, but don't close it
+        // if the callback throws then the parent tx will rollback automatically
+        const currentTx = this.getCurrentWriteTx();
+        if (currentTx) {
+            return await callback(currentTx);
+        }
+        return this.writerQueue.put(async ()=>{
+            const tx = new WriteTransaction(this);
+            try {
+                const res = await this.writerCtx.run(tx, callback, tx);
+                await tx.commit();
+                return res;
+            } catch (err) {
+                this.log.error(`Failed to commit transaction`, err);
+                throw err;
+            } finally{
+                tx.close();
+            }
+        });
+    }
+    clear() {
+        return Promise.resolve();
+    }
+    fork() {
+        throw new Error('Not implemented');
+    }
+    async delete() {
+        await this.close();
+        await rm(this.dataDir, {
+            recursive: true,
+            force: true,
+            maxRetries: 3
+        });
+        this.log.verbose(`Deleted database files at ${this.dataDir}`);
+        await this.cleanup?.();
+    }
+    async close() {
+        if (!this.open) {
+            // already closed
+            return;
+        }
+        this.open = false;
+        await this.writerQueue.cancel();
+        await this.channel.sendMessage(LMDBMessageType.CLOSE, undefined);
+    }
+    async sendMessage(msgType, body) {
+        if (!this.open) {
+            throw new Error('Store is closed');
+        }
+        if (msgType === LMDBMessageType.START_CURSOR) {
+            await this.availableCursors.acquire();
+        }
+        let response = undefined;
+        try {
+            ({ response } = await this.channel.sendMessage(msgType, body));
+            return response;
+        } finally{
+            if (msgType === LMDBMessageType.START_CURSOR && response === undefined || msgType === LMDBMessageType.CLOSE_CURSOR || // it's possible for a START_CURSOR command to not return a cursor (e.g. db is empty)
+            msgType === LMDBMessageType.START_CURSOR && typeof response.cursor !== 'number') {
+                this.availableCursors.release();
+            }
+        }
+    }
+    async estimateSize() {
+        const resp = await this.sendMessage(LMDBMessageType.STATS, undefined);
+        return {
+            mappingSize: Number(resp.dbMapSizeBytes),
+            actualSize: resp.stats.reduce((s, db)=>Number(db.totalUsedSize) + s, 0),
+            numItems: resp.stats.reduce((s, db)=>Number(db.numDataItems) + s, 0)
+        };
+    }
+}
+export function execInWriteTx(store, fn) {
+    const currentWrite = store.getCurrentWriteTx();
+    if (currentWrite) {
+        return fn(currentWrite);
+    } else {
+        return store.transactionAsync(fn);
+    }
+}
+export async function execInReadTx(store, fn) {
+    const currentWrite = store.getCurrentWriteTx();
+    if (currentWrite) {
+        return await fn(currentWrite);
+    } else {
+        const tx = store.getReadTx();
+        try {
+            return await fn(tx);
+        } finally{
+            tx.close();
+        }
+    }
+}

package/dest/lmdb-v2/utils.d.ts

@@ -0,0 +1,19 @@
+/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" resolution-mode="require"/>
+import type { Key } from '../interfaces/common.js';
+type Cmp<T> = (a: T, b: T) => -1 | 0 | 1;
+export declare function dedupeSortedArray<T>(arr: T[], cmp: Cmp<T>): void;
+export declare function insertIntoSortedArray<T>(arr: T[], item: T, cmp: (a: T, b: T) => number): void;
+export declare function findIndexInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): number;
+export declare function findInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): T | undefined;
+export declare function removeAnyOf<T, N>(arr: T[], vals: N[], cmp: (a: T, b: N) => -1 | 0 | 1): void;
+export declare function removeFromSortedArray<T, N>(arr: T[], val: N, cmp: (a: T, b: N) => -1 | 0 | 1): void;
+export declare function merge<T>(arr: T[], toInsert: T[], cmp: (a: T, b: T) => -1 | 0 | 1): void;
+export declare function keyCmp(a: [Uint8Array, Uint8Array[] | null], b: [Uint8Array, Uint8Array[] | null]): -1 | 0 | 1;
+export declare function singleKeyCmp(a: [Uint8Array, Uint8Array[] | null], b: Uint8Array): -1 | 0 | 1;
+export declare function minKey(prefix: string): Buffer;
+export declare function maxKey(prefix: string): Buffer;
+export declare function serializeKey(prefix: string, key: Key): Buffer;
+export declare function deserializeKey<K extends Key>(prefix: string, key: Uint8Array): K | false;
+export {};
+//# sourceMappingURL=utils.d.ts.map

package/dest/lmdb-v2/utils.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"utils.d.ts","sourceRoot":"","sources":["../../src/lmdb-v2/utils.ts"],"names":[],"mappings":";;AAEA,OAAO,KAAK,EAAE,GAAG,EAAE,MAAM,yBAAyB,CAAC;AAEnD,KAAK,GAAG,CAAC,CAAC,IAAI,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAAC;AAEzC,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,GAAG,CAAC,CAAC,CAAC,GAAG,IAAI,CAkBhE;AAED,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,GAAG,IAAI,CAgB7F;AAED,wBAAgB,sBAAsB,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,GAAG,MAAM,CAiBxG;AAED,wBAAgB,iBAAiB,CAAC,CAAC,EAAE,CAAC,EAAE,MAAM,EAAE,CAAC,EAAE,EAAE,MAAM,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,MAAM,GAAG,CAAC,GAAG,SAAS,CAG1G;AAED,wBAAgB,WAAW,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,IAAI,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAsB5F;AAED,wBAAgB,qBAAqB,CAAC,CAAC,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,QAK5F;AAED,wBAAgB,KAAK,CAAC,CAAC,EAAE,GAAG,EAAE,CAAC,EAAE,EAAE,QAAQ,EAAE,CAAC,EAAE,EAAE,GAAG,EAAE,CAAC,CAAC,EAAE,CAAC,EAAE,CAAC,EAAE,CAAC,KAAK,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,GAAG,IAAI,CAqBvF;AAED,wBAAgB,MAAM,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAE7G;AAED,wBAAgB,YAAY,CAAC,CAAC,EAAE,CAAC,UAAU,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC,EAAE,CAAC,EAAE,UAAU,GAAG,CAAC,CAAC,GAAG,CAAC,GAAG,CAAC,CAE5F;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM,UAEpC;AAED,wBAAgB,MAAM,CAAC,MAAM,EAAE,MAAM,UAEpC;AAED,wBAAgB,YAAY,CAAC,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,GAAG,GAAG,MAAM,CAE7D;AAED,wBAAgB,cAAc,CAAC,CAAC,SAAS,GAAG,EAAE,MAAM,EAAE,MAAM,EAAE,GAAG,EAAE,UAAU,GAAG,CAAC,GAAG,KAAK,CAOxF"}

package/dest/lmdb-v2/utils.js

@@ -0,0 +1,126 @@
+import { MAXIMUM_KEY, fromBufferKey, toBufferKey } from 'ordered-binary';
+export function dedupeSortedArray(arr, cmp) {
+    for(let i = 0; i < arr.length; i++){
+        let j = i + 1;
+        for(; j < arr.length; j++){
+            const res = cmp(arr[i], arr[j]);
+            if (res === 0) {
+                continue;
+            } else if (res < 0) {
+                break;
+            } else {
+                throw new Error('Array not sorted');
+            }
+        }
+        if (j - i > 1) {
+            arr.splice(i + 1, j - i - 1);
+        }
+    }
+}
+export function insertIntoSortedArray(arr, item, cmp) {
+    let left = 0;
+    let right = arr.length;
+    while(left < right){
+        const mid = left + right >> 1;
+        const comparison = cmp(arr[mid], item);
+        if (comparison < 0) {
+            left = mid + 1;
+        } else {
+            right = mid;
+        }
+    }
+    arr.splice(left, 0, item);
+}
+export function findIndexInSortedArray(values, needle, cmp) {
+    let start = 0;
+    let end = values.length - 1;
+    while(start <= end){
+        const mid = start + ((end - start) / 2 | 0);
+        const res = cmp(values[mid], needle);
+        if (res === 0) {
+            return mid;
+        } else if (res > 0) {
+            end = mid - 1;
+        } else {
+            start = mid + 1;
+        }
+    }
+    return -1;
+}
+export function findInSortedArray(values, needle, cmp) {
+    const idx = findIndexInSortedArray(values, needle, cmp);
+    return idx > -1 ? values[idx] : undefined;
+}
+export function removeAnyOf(arr, vals, cmp) {
+    let writeIdx = 0;
+    let readIdx = 0;
+    let valIdx = 0;
+    while(readIdx < arr.length && valIdx < vals.length){
+        const comparison = cmp(arr[readIdx], vals[valIdx]);
+        if (comparison < 0) {
+            arr[writeIdx++] = arr[readIdx++];
+        } else if (comparison > 0) {
+            valIdx++;
+        } else {
+            readIdx++;
+        }
+    }
+    while(readIdx < arr.length){
+        arr[writeIdx++] = arr[readIdx++];
+    }
+    arr.length = writeIdx;
+}
+export function removeFromSortedArray(arr, val, cmp) {
+    const idx = findIndexInSortedArray(arr, val, cmp);
+    if (idx > -1) {
+        arr.splice(idx, 1);
+    }
+}
+export function merge(arr, toInsert, cmp) {
+    const result = new Array(arr.length + toInsert.length);
+    let i = 0, j = 0, k = 0;
+    while(i < arr.length && j < toInsert.length){
+        result[k++] = cmp(arr[i], toInsert[j]) <= 0 ? arr[i++] : toInsert[j++];
+    }
+    while(i < arr.length){
+        result[k++] = arr[i++];
+    }
+    while(j < toInsert.length){
+        result[k++] = toInsert[j++];
+    }
+    for(i = 0; i < result.length; i++){
+        arr[i] = result[i];
+    }
+    arr.length = result.length;
+}
+export function keyCmp(a, b) {
+    return Buffer.compare(a[0], b[0]);
+}
+export function singleKeyCmp(a, b) {
+    return Buffer.compare(a[0], b);
+}
+export function minKey(prefix) {
+    return toBufferKey([
+        prefix
+    ]);
+}
+export function maxKey(prefix) {
+    return toBufferKey([
+        prefix,
+        MAXIMUM_KEY
+    ]);
+}
+export function serializeKey(prefix, key) {
+    return toBufferKey([
+        prefix,
+        key
+    ]);
+}
+export function deserializeKey(prefix, key) {
+    const buf = Buffer.from(key);
+    const parsed = fromBufferKey(buf);
+    if (!Array.isArray(parsed) || parsed[0] !== prefix) {
+        return false;
+    }
+    return parsed[1];
+}

package/dest/lmdb-v2/write_transaction.d.ts

@@ -0,0 +1,19 @@
+/// <reference types="node" resolution-mode="require"/>
+/// <reference types="node" resolution-mode="require"/>
+import { type Batch } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+export declare class WriteTransaction extends ReadTransaction {
+    #private;
+    readonly dataBatch: Batch;
+    readonly indexBatch: Batch;
+    set(key: Uint8Array, value: Uint8Array): Promise<void>;
+    remove(key: Uint8Array): Promise<void>;
+    get(key: Buffer): Promise<Uint8Array | undefined>;
+    setIndex(key: Buffer, ...values: Buffer[]): Promise<void>;
+    removeIndex(key: Buffer, ...values: Buffer[]): Promise<void>;
+    getIndex(key: Buffer): Promise<Uint8Array[]>;
+    iterate(startKey: Uint8Array, endKey?: Uint8Array | undefined, reverse?: boolean, limit?: number): AsyncIterable<[Uint8Array, Uint8Array]>;
+    iterateIndex(startKey: Uint8Array, endKey?: Uint8Array | undefined, reverse?: boolean, limit?: number): AsyncIterable<[Uint8Array, Uint8Array[]]>;
+    commit(): Promise<void>;
+}
+//# sourceMappingURL=write_transaction.d.ts.map

package/dest/lmdb-v2/write_transaction.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"write_transaction.d.ts","sourceRoot":"","sources":["../../src/lmdb-v2/write_transaction.ts"],"names":[],"mappings":";;AAAA,OAAO,EAAE,KAAK,KAAK,EAA6B,MAAM,cAAc,CAAC;AACrE,OAAO,EAAE,eAAe,EAAE,MAAM,uBAAuB,CAAC;AAaxD,qBAAa,gBAAiB,SAAQ,eAAe;;IAEnD,SAAgB,SAAS,EAAE,KAAK,CAG9B;IACF,SAAgB,UAAU,EAAE,KAAK,CAG/B;IAEF,GAAG,CAAC,GAAG,EAAE,UAAU,EAAE,KAAK,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAkBtD,MAAM,CAAC,GAAG,EAAE,UAAU,GAAG,OAAO,CAAC,IAAI,CAAC;IAchB,GAAG,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,GAAG,SAAS,CAAC;IAevE,QAAQ,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IA6BzD,WAAW,CAAC,GAAG,EAAE,MAAM,EAAE,GAAG,MAAM,EAAE,MAAM,EAAE,GAAG,OAAO,CAAC,IAAI,CAAC;IAuCtC,QAAQ,CAAC,GAAG,EAAE,MAAM,GAAG,OAAO,CAAC,UAAU,EAAE,CAAC;IAuB3C,OAAO,CAC5B,QAAQ,EAAE,UAAU,EACpB,MAAM,CAAC,EAAE,UAAU,GAAG,SAAS,EAC/B,OAAO,CAAC,EAAE,OAAO,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,aAAa,CAAC,CAAC,UAAU,EAAE,UAAU,CAAC,CAAC;IAanB,YAAY,CACjC,QAAQ,EAAE,UAAU,EACpB,MAAM,CAAC,EAAE,UAAU,GAAG,SAAS,EAC/B,OAAO,CAAC,EAAE,OAAO,EACjB,KAAK,CAAC,EAAE,MAAM,GACb,aAAa,CAAC,CAAC,UAAU,EAAE,UAAU,EAAE,CAAC,CAAC;IAqH/B,MAAM;CAUpB"}

package/dest/lmdb-v2/write_transaction.js

@@ -0,0 +1,254 @@
+import { Database, LMDBMessageType } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+import { dedupeSortedArray, findInSortedArray, findIndexInSortedArray, insertIntoSortedArray, keyCmp, merge, removeAnyOf, removeFromSortedArray, singleKeyCmp } from './utils.js';
+export class WriteTransaction extends ReadTransaction {
+    // exposed for tests
+    dataBatch = {
+        addEntries: [],
+        removeEntries: []
+    };
+    indexBatch = {
+        addEntries: [],
+        removeEntries: []
+    };
+    set(key, value) {
+        this.assertIsOpen();
+        const addEntry = findInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+        if (!addEntry) {
+            insertIntoSortedArray(this.dataBatch.addEntries, [
+                key,
+                [
+                    value
+                ]
+            ], keyCmp);
+        } else {
+            addEntry[1] = [
+                value
+            ];
+        }
+        const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+        if (removeEntryIndex > -1) {
+            this.dataBatch.removeEntries.splice(removeEntryIndex, 1);
+        }
+        return Promise.resolve();
+    }
+    remove(key) {
+        const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+        if (removeEntryIndex === -1) {
+            this.dataBatch.removeEntries.push([
+                key,
+                null
+            ]);
+        }
+        const addEntryIndex = findIndexInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+        if (addEntryIndex > -1) {
+            this.dataBatch.addEntries.splice(addEntryIndex, 1);
+        }
+        return Promise.resolve();
+    }
+    async get(key) {
+        this.assertIsOpen();
+        const addEntry = findInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+        if (addEntry) {
+            return addEntry[1][0];
+        }
+        const removeEntryIdx = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+        if (removeEntryIdx > -1) {
+            return undefined;
+        }
+        return await super.get(key);
+    }
+    setIndex(key, ...values) {
+        this.assertIsOpen();
+        const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+        const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+        if (removeEntries) {
+            if (removeEntries[1]) {
+                // check if we were deleting these values and update
+                removeAnyOf(removeEntries[1], values, Buffer.compare);
+            }
+            if (!removeEntries[1] || removeEntries[1].length === 0) {
+                // either we were deleting the entire key previously
+                // or after cleaning up duplicates, we don't have anything else to delete
+                removeFromSortedArray(this.indexBatch.removeEntries, removeEntries, keyCmp);
+            }
+        }
+        if (addEntries) {
+            merge(addEntries[1], values, Buffer.compare);
+            dedupeSortedArray(addEntries[1], Buffer.compare);
+        } else {
+            insertIntoSortedArray(this.indexBatch.addEntries, [
+                key,
+                values
+            ], keyCmp);
+        }
+        return Promise.resolve();
+    }
+    removeIndex(key, ...values) {
+        this.assertIsOpen();
+        const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+        const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+        if (values.length === 0) {
+            // special case, we're deleting the entire key
+            if (addEntries) {
+                removeFromSortedArray(this.indexBatch.addEntries, addEntries, keyCmp);
+            }
+            if (removeEntries) {
+                removeEntries[1] = null;
+            } else {
+                insertIntoSortedArray(this.indexBatch.removeEntries, [
+                    key,
+                    null
+                ], keyCmp);
+            }
+            return Promise.resolve();
+        }
+        if (addEntries) {
+            removeAnyOf(addEntries[1], values, Buffer.compare);
+            if (addEntries[1].length === 0) {
+                removeFromSortedArray(this.indexBatch.addEntries, addEntries, keyCmp);
+            }
+        }
+        if (removeEntries) {
+            removeEntries[1] ??= [];
+            merge(removeEntries[1], values, Buffer.compare);
+            dedupeSortedArray(removeEntries[1], Buffer.compare);
+        } else {
+            insertIntoSortedArray(this.indexBatch.removeEntries, [
+                key,
+                values
+            ], keyCmp);
+        }
+        return Promise.resolve();
+    }
+    async getIndex(key) {
+        this.assertIsOpen();
+        const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+        if (removeEntries && removeEntries[1] === null) {
+            return [];
+        }
+        const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+        const results = await super.getIndex(key);
+        if (addEntries) {
+            merge(results, addEntries[1], Buffer.compare);
+            dedupeSortedArray(results, Buffer.compare);
+        }
+        if (removeEntries && Array.isArray(removeEntries[1])) {
+            removeAnyOf(results, removeEntries[1], Buffer.compare);
+        }
+        return results;
+    }
+    async *iterate(startKey, endKey, reverse, limit) {
+        yield* this.#iterate(super.iterate(startKey, endKey, reverse), this.dataBatch, startKey, endKey, reverse, limit, (committed, toAdd)=>toAdd.length > 0 ? toAdd[0] : committed, (vals)=>vals[0]);
+    }
+    async *iterateIndex(startKey, endKey, reverse, limit) {
+        yield* this.#iterate(super.iterateIndex(startKey, endKey, reverse), this.indexBatch, startKey, endKey, reverse, limit, (committed, toAdd, toRemove)=>{
+            if (toAdd.length > 0) {
+                merge(committed, toAdd, Buffer.compare);
+                dedupeSortedArray(committed, Buffer.compare);
+            }
+            if (toRemove.length > 0) {
+                removeAnyOf(committed, toRemove, Buffer.compare);
+            }
+            return committed;
+        }, (vals)=>vals);
+    }
+    async *#iterate(iterator, batch, startKey, endKey, reverse = false, limit, merge, map) {
+        this.assertIsOpen();
+        // make a copy of this in case we're running in reverse
+        const uncommittedEntries = [
+            ...batch.addEntries
+        ];
+        // used to check we're in the right order when comparing between a key and uncommittedEntries
+        let cmpDirection = -1;
+        if (reverse) {
+            cmpDirection = 1;
+            uncommittedEntries.reverse();
+        }
+        let uncommittedEntriesIdx = 0;
+        while(uncommittedEntriesIdx < uncommittedEntries.length){
+            const entry = uncommittedEntries[uncommittedEntriesIdx];
+            // go to the first key in our cache that would be captured by the iterator
+            if (Buffer.compare(entry[0], startKey) !== cmpDirection) {
+                break;
+            }
+            uncommittedEntriesIdx++;
+        }
+        let count = 0;
+        // helper to early return if we've reached our limit
+        const checkLimit = typeof limit === 'number' ? ()=>count < limit : ()=>true;
+        for await (const [key, values] of iterator){
+            // yield every key that we have cached that's captured by the iterator
+            while(uncommittedEntriesIdx < uncommittedEntries.length && checkLimit()){
+                const entry = uncommittedEntries[uncommittedEntriesIdx];
+                if (endKey && Buffer.compare(entry[0], endKey) !== cmpDirection) {
+                    break;
+                }
+                if (Buffer.compare(entry[0], key) === cmpDirection) {
+                    count++;
+                    yield [
+                        entry[0],
+                        map(entry[1])
+                    ];
+                } else {
+                    break;
+                }
+                uncommittedEntriesIdx++;
+            }
+            if (!checkLimit()) {
+                break;
+            }
+            const toRemove = findInSortedArray(batch.removeEntries, key, singleKeyCmp);
+            // at this point we've either exhausted all uncommitted entries,
+            // we reached a key strictly greater/smaller than `key`
+            // or we found the key itself
+            // check if it's the key and use the uncommitted value
+            let toAdd = [];
+            if (uncommittedEntriesIdx < uncommittedEntries.length && Buffer.compare(uncommittedEntries[uncommittedEntriesIdx][0], key) === 0) {
+                toAdd = uncommittedEntries[uncommittedEntriesIdx][1];
+                uncommittedEntriesIdx++;
+            }
+            if (toRemove && !toRemove[1]) {
+                continue;
+            } else {
+                const mergedValues = merge(values, toAdd, toRemove?.[1] ?? []);
+                if (mergedValues) {
+                    count++;
+                    yield [
+                        key,
+                        mergedValues
+                    ];
+                }
+            }
+        }
+        // emit all the uncommitted data that would be captured by this iterator
+        while(uncommittedEntriesIdx < uncommittedEntries.length && checkLimit()){
+            const entry = uncommittedEntries[uncommittedEntriesIdx];
+            if (endKey && Buffer.compare(entry[0], endKey) !== cmpDirection) {
+                break;
+            }
+            count++;
+            yield [
+                entry[0],
+                map(entry[1])
+            ];
+            uncommittedEntriesIdx++;
+        }
+    }
+    async commit() {
+        this.assertIsOpen();
+        this.close();
+        await this.channel.sendMessage(LMDBMessageType.BATCH, {
+            batches: new Map([
+                [
+                    Database.DATA,
+                    this.dataBatch
+                ],
+                [
+                    Database.INDEX,
+                    this.indexBatch
+                ]
+            ])
+        });
+    }
+}

package/dest/stores/index.d.ts

@@ -0,0 +1,2 @@
+export * from './l2_tips_store.js';
+//# sourceMappingURL=index.d.ts.map

package/dest/stores/index.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"index.d.ts","sourceRoot":"","sources":["../../src/stores/index.ts"],"names":[],"mappings":"AAAA,cAAc,oBAAoB,CAAC"}

package/dest/stores/index.js

@@ -0,0 +1 @@
+export * from './l2_tips_store.js';

package/dest/stores/l2_tips_store.d.ts

@@ -0,0 +1,13 @@
+import type { L2BlockStreamEvent, L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider, L2Tips } from '@aztec/stdlib/block';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+/** Stores currently synced L2 tips and unfinalized block hashes. */
+export declare class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
+    private readonly l2TipsStore;
+    private readonly l2BlockHashesStore;
+    constructor(store: AztecAsyncKVStore, namespace: string);
+    getL2BlockHash(number: number): Promise<string | undefined>;
+    getL2Tips(): Promise<L2Tips>;
+    private getL2Tip;
+    handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void>;
+}
+//# sourceMappingURL=l2_tips_store.d.ts.map

package/dest/stores/l2_tips_store.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"l2_tips_store.d.ts","sourceRoot":"","sources":["../../src/stores/l2_tips_store.ts"],"names":[],"mappings":"AAAA,OAAO,KAAK,EAEV,kBAAkB,EAClB,yBAAyB,EACzB,8BAA8B,EAE9B,MAAM,EACP,MAAM,qBAAqB,CAAC;AAG7B,OAAO,KAAK,EAAE,iBAAiB,EAAE,MAAM,wBAAwB,CAAC;AAEhE,oEAAoE;AACpE,qBAAa,WAAY,YAAW,yBAAyB,EAAE,8BAA8B;IAC3F,OAAO,CAAC,QAAQ,CAAC,WAAW,CAAoC;IAChE,OAAO,CAAC,QAAQ,CAAC,kBAAkB,CAAgC;gBAEvD,KAAK,EAAE,iBAAiB,EAAE,SAAS,EAAE,MAAM;IAKhD,cAAc,CAAC,MAAM,EAAE,MAAM,GAAG,OAAO,CAAC,MAAM,GAAG,SAAS,CAAC;IAIrD,SAAS,IAAI,OAAO,CAAC,MAAM,CAAC;YAQ3B,QAAQ;IAaT,sBAAsB,CAAC,KAAK,EAAE,kBAAkB,GAAG,OAAO,CAAC,IAAI,CAAC;CAsB9E"}