@aztec/kv-store 0.0.1-commit.8c0b8ff → 0.0.1-commit.8cb2d04d8

package/dest/sqlite-opfs/store.js

@@ -0,0 +1,242 @@
+import { SerialQueue } from '@aztec/foundation/queue';
+import { SQLiteOPFSAztecArray } from './array.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
+import { SQLiteOPFSAztecSet } from './set.js';
+import { SQLiteOPFSAztecSingleton } from './singleton.js';
+/**
+ * Main-thread handle for a SQLite database persisted to OPFS via the `opfs-sahpool`
+ * VFS. Owns a dedicated Web Worker (the SAH Pool VFS requires Worker context) and
+ * routes every SQL op through it via typed postMessage RPC.
+ *
+ * Transaction ordering is guaranteed by a `SerialQueue` on the main thread combined
+ * with an `#inTx` flag: outside a `transactionAsync` block, each op acquires the
+ * queue for its own auto-commit; inside a block, the outer call holds the queue and
+ * nested ops bypass it to avoid deadlock.
+ */ export class AztecSQLiteOPFSStore {
+    isEphemeral;
+    #worker;
+    #pending;
+    #txQueue;
+    #name;
+    #log;
+    #nextId;
+    #inTx;
+    #closed;
+    constructor(worker, name, log, isEphemeral){
+        this.isEphemeral = isEphemeral;
+        this.#pending = new Map();
+        this.#txQueue = new SerialQueue();
+        this.#nextId = 0;
+        this.#inTx = false;
+        this.#closed = false;
+        this.#worker = worker;
+        this.#name = name;
+        this.#log = log;
+        this.#worker.onmessage = (ev)=>{
+            const { id } = ev.data;
+            const handler = this.#pending.get(id);
+            if (!handler) {
+                this.#log.warn(`SQLite worker: no pending handler for id ${id}`);
+                return;
+            }
+            this.#pending.delete(id);
+            handler.resolve(ev.data);
+        };
+        this.#worker.onerror = (ev)=>{
+            this.#log.error(`SQLite worker crashed: ${ev.message}`);
+            this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
+        };
+        this.#txQueue.start();
+    }
+    /**
+   * Opens (or creates) a SQLite database stored in the OPFS SAH Pool. When `ephemeral`
+   * is true the database lives only in memory and is lost when the worker terminates.
+   * Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
+   * required when multiple stores coexist in the same tab, because the SAH Pool holds
+   * an exclusive lock on its directory.
+   */ static async open(log, name, ephemeral = false, poolDirectory) {
+        const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
+        log.debug(`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}database ${dbName}`);
+        const worker = new Worker(new URL('./worker.js', import.meta.url), {
+            type: 'module'
+        });
+        const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
+        await store.#sendRequest({
+            type: 'init',
+            id: store.#allocId(),
+            dbName,
+            ephemeral,
+            poolDirectory
+        });
+        return store;
+    }
+    openMap(name) {
+        return new SQLiteOPFSAztecMap(this, name);
+    }
+    openSet(name) {
+        return new SQLiteOPFSAztecSet(this, name);
+    }
+    openMultiMap(name) {
+        return new SQLiteOPFSAztecMultiMap(this, name);
+    }
+    openCounter(_name) {
+        throw new Error('Method not implemented.');
+    }
+    openArray(name) {
+        return new SQLiteOPFSAztecArray(this, name);
+    }
+    openSingleton(name) {
+        return new SQLiteOPFSAztecSingleton(this, name);
+    }
+    transactionAsync(callback) {
+        // Nested calls join the outer transaction — SQLite does not support nested BEGIN,
+        // and re-acquiring the SerialQueue while the outer call holds it would deadlock.
+        // Errors in the nested callback propagate to the outer catch, which rolls back the
+        // whole thing (the standard "nested tx = savepoint-free join" semantic).
+        if (this.#inTx) {
+            return callback();
+        }
+        return this.#txQueue.put(async ()=>{
+            this.#inTx = true;
+            await this.#sendRequest({
+                type: 'begin',
+                id: this.#allocId()
+            });
+            try {
+                const result = await callback();
+                await this.#sendRequest({
+                    type: 'commit',
+                    id: this.#allocId()
+                });
+                return result;
+            } catch (err) {
+                await this.#sendRequest({
+                    type: 'rollback',
+                    id: this.#allocId()
+                }).catch((rollbackErr)=>this.#log.warn(`SQLite ROLLBACK failed: ${rollbackErr instanceof Error ? rollbackErr.message : rollbackErr}`));
+                throw err;
+            } finally{
+                this.#inTx = false;
+            }
+        });
+    }
+    async clear() {
+        await this.runAsync('DELETE FROM data');
+    }
+    async delete() {
+        if (this.#closed) {
+            return;
+        }
+        this.#closed = true;
+        await this.#txQueue.end();
+        await this.#sendRequest({
+            type: 'deleteDb',
+            id: this.#allocId(),
+            dbName: this.#name
+        }).catch((err)=>this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`));
+        this.#worker.terminate();
+        this.#rejectPending('SQLite store deleted');
+    }
+    /**
+   * Placeholder — returns zeros to mirror the IndexedDB backend. SQLite exposes real
+   * numbers cheaply via `PRAGMA page_count` / `page_size` / `freelist_count` and
+   * `SELECT COUNT(*) FROM data`, which would populate `physicalFileSize`, `actualSize`,
+   * and `numItems` meaningfully (`mappingSize` stays 0 — it's an LMDB mmap concept).
+   * Upgrade when any caller actually consumes these values; all current consumers
+   * tolerate zeros.
+   */ estimateSize() {
+        return Promise.resolve({
+            mappingSize: 0,
+            physicalFileSize: 0,
+            actualSize: 0,
+            numItems: 0
+        });
+    }
+    async close() {
+        if (this.#closed) {
+            return;
+        }
+        this.#closed = true;
+        await this.#txQueue.end();
+        await this.#sendRequest({
+            type: 'close',
+            id: this.#allocId()
+        }).catch(()=>{});
+        this.#worker.terminate();
+        this.#rejectPending('SQLite store closed');
+    }
+    backupTo(_dstPath, _compact) {
+        throw new Error('Method not implemented.');
+    }
+    /**
+   * Returns a raw SQLite image (bytes suitable for writing as a `.sqlite` file and
+   * opening in any SQLite tool). Works only for non-ephemeral DBs because the OPFS
+   * SAH Pool has to be initialized. Useful for inspection/debugging.
+   */ async exportDb() {
+        const resp = await this.#sendRequest({
+            type: 'export',
+            id: this.#allocId()
+        });
+        if (!('bytes' in resp) || !resp.bytes) {
+            throw new Error('exportDb: worker returned no bytes');
+        }
+        return resp.bytes;
+    }
+    /**
+   * Runs a write statement (INSERT/UPDATE/DELETE/DDL). If called inside a
+   * `transactionAsync` block, bypasses the queue; otherwise acquires it so the
+   * op runs in its own auto-commit.
+   */ runAsync(sql, bind) {
+        const send = ()=>this.#sendRequest({
+                type: 'run',
+                id: this.#allocId(),
+                sql,
+                bind
+            }).then((r)=>({
+                    changes: 'changes' in r ? r.changes ?? 0 : 0
+                }));
+        return this.#inTx ? send() : this.#txQueue.put(send);
+    }
+    /** Runs a SELECT statement and returns rows in array row-mode. */ allAsync(sql, bind) {
+        const send = ()=>this.#sendRequest({
+                type: 'all',
+                id: this.#allocId(),
+                sql,
+                bind
+            }).then((r)=>'rows' in r ? r.rows ?? [] : []);
+        return this.#inTx ? send() : this.#txQueue.put(send);
+    }
+    #allocId() {
+        return ++this.#nextId;
+    }
+    /**
+   * Reject any in-flight requests with `reason`, so callers awaiting a response to a
+   * request sent to a now-terminated worker don't hang forever. Called from
+   * close()/delete() and from the worker.onerror handler.
+   */ #rejectPending(reason) {
+        if (this.#pending.size === 0) {
+            return;
+        }
+        const err = new Error(reason);
+        for (const { reject } of this.#pending.values()){
+            reject(err);
+        }
+        this.#pending.clear();
+    }
+    #sendRequest(req) {
+        return new Promise((resolve, reject)=>{
+            this.#pending.set(req.id, {
+                resolve: (resp)=>{
+                    if (resp.type === 'err') {
+                        reject(new Error(resp.message));
+                    } else {
+                        resolve(resp);
+                    }
+                },
+                reject
+            });
+            this.#worker.postMessage(req);
+        });
+    }
+}

package/dest/sqlite-opfs/worker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoid29ya2VyLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi8uLi9zcmMvc3FsaXRlLW9wZnMvd29ya2VyLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiIifQ==

package/dest/sqlite-opfs/worker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"worker.d.ts","sourceRoot":"","sources":["../../src/sqlite-opfs/worker.ts"],"names":[],"mappings":""}

package/dest/sqlite-opfs/worker.js

@@ -0,0 +1,189 @@
+/// <reference lib="webworker" />
+import sqlite3InitModule from '@sqlite.org/sqlite-wasm';
+const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS data (
+    slot TEXT NOT NULL PRIMARY KEY,
+    container TEXT NOT NULL,
+    key BLOB NOT NULL,
+    key_count INTEGER NOT NULL,
+    hash TEXT NOT NULL,
+    value BLOB
+  ) WITHOUT ROWID;
+
+  CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
+`;
+const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
+const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
+let sqlite3;
+let pool;
+let poolDirectory;
+let db;
+let dbPath;
+async function ensurePool(directory) {
+    sqlite3 ??= await sqlite3InitModule();
+    if (!pool) {
+        poolDirectory = directory;
+        pool = await sqlite3.installOpfsSAHPoolVfs({
+            name: SAH_POOL_VFS_NAME,
+            directory,
+            initialCapacity: 8
+        });
+    }
+    return pool;
+}
+async function handleInit(dbName, ephemeral, directory) {
+    sqlite3 ??= await sqlite3InitModule();
+    if (ephemeral) {
+        db = new sqlite3.oo1.DB(':memory:', 'c');
+    } else {
+        const p = await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
+        dbPath = normalizeDbPath(dbName);
+        db = new p.OpfsSAHPoolDb(dbPath);
+    }
+    runSql(SCHEMA_SQL);
+}
+function handleClose() {
+    db?.close();
+    db = undefined;
+    dbPath = undefined;
+}
+async function handleExport() {
+    if (!db || !dbPath) {
+        throw new Error('SQLite worker: no database open to export');
+    }
+    if (!pool) {
+        throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
+    }
+    return await pool.exportFile(dbPath);
+}
+async function handleDeleteDb(dbName) {
+    const path = normalizeDbPath(dbName);
+    if (db && dbPath === path) {
+        db.close();
+        db = undefined;
+        dbPath = undefined;
+    }
+    const p = await ensurePool(poolDirectory ?? DEFAULT_SAH_POOL_DIRECTORY);
+    try {
+        p.unlink(path);
+    } catch  {
+    // File may not exist; ignore.
+    }
+}
+function requireDb() {
+    if (!db) {
+        throw new Error('SQLite worker: no database open');
+    }
+    return db;
+}
+function runSql(sql, bind) {
+    const conn = requireDb();
+    conn.exec({
+        sql,
+        bind
+    });
+    return {
+        changes: conn.changes()
+    };
+}
+function selectAll(sql, bind) {
+    const conn = requireDb();
+    const rows = [];
+    conn.exec({
+        sql,
+        bind,
+        rowMode: 'array',
+        resultRows: rows
+    });
+    return rows;
+}
+function normalizeDbPath(dbName) {
+    return dbName.startsWith('/') ? dbName : `/${dbName}`;
+}
+function respond(msg) {
+    self.postMessage(msg);
+}
+self.onmessage = async (ev)=>{
+    const req = ev.data;
+    try {
+        switch(req.type){
+            case 'init':
+                await handleInit(req.dbName, req.ephemeral, req.poolDirectory);
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'close':
+                handleClose();
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'deleteDb':
+                await handleDeleteDb(req.dbName);
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'run':
+                {
+                    const { changes } = runSql(req.sql, req.bind);
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        changes
+                    });
+                }
+            case 'all':
+                {
+                    const rows = selectAll(req.sql, req.bind);
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        rows
+                    });
+                }
+            case 'export':
+                {
+                    const bytes = await handleExport();
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        bytes
+                    });
+                }
+            case 'begin':
+                runSql('BEGIN');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'commit':
+                runSql('COMMIT');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'rollback':
+                runSql('ROLLBACK');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            default:
+                {
+                    const _exhaustive = req;
+                    throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
+                }
+        }
+    } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        respond({
+            type: 'err',
+            id: req.id,
+            message
+        });
+    }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/kv-store",
-  "version": "0.0.1-commit.8c0b8ff",
+  "version": "0.0.1-commit.8cb2d04d8",
   "type": "module",
   "exports": {
     ".": "./dest/interfaces/index.js",
@@ -8,8 +8,8 @@
     "./lmdb": "./dest/lmdb/index.js",
     "./lmdb-v2": "./dest/lmdb-v2/index.js",
     "./indexeddb": "./dest/indexeddb/index.js",
-    "./stores": "./dest/stores/index.js",
-    "./config": "./dest/config.js"
+    "./sqlite-opfs": "./dest/sqlite-opfs/index.js",
+    "./stores": "./dest/stores/index.js"
   },
   "scripts": {
     "build": "yarn clean && ../scripts/tsc.sh",
@@ -17,6 +17,7 @@
     "clean": "rm -rf ./dest .tsbuildinfo",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
     "test:browser": "vitest run --config ./vitest.config.ts",
+    "bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
     "test": "yarn test:node && yarn test:browser",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
@@ -25,11 +26,12 @@
     "./package.local.json"
   ],
   "dependencies": {
-    "@aztec/constants": "0.0.1-commit.8c0b8ff",
-    "@aztec/ethereum": "0.0.1-commit.8c0b8ff",
-    "@aztec/foundation": "0.0.1-commit.8c0b8ff",
-    "@aztec/native": "0.0.1-commit.8c0b8ff",
-    "@aztec/stdlib": "0.0.1-commit.8c0b8ff",
+    "@aztec/constants": "0.0.1-commit.8cb2d04d8",
+    "@aztec/ethereum": "0.0.1-commit.8cb2d04d8",
+    "@aztec/foundation": "0.0.1-commit.8cb2d04d8",
+    "@aztec/native": "0.0.1-commit.8cb2d04d8",
+    "@aztec/stdlib": "0.0.1-commit.8cb2d04d8",
+    "@sqlite.org/sqlite-wasm": "3.50.4-build1",
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",

package/src/bench/shared_map_bench.ts

@@ -0,0 +1,111 @@
+import type { Logger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+
+import type { Key } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+
+/** One benchmark measurement. */
+export type BenchResult = {
+  /** Benchmark name (includes the backend prefix for disambiguation). */
+  name: string;
+  value: number;
+  unit: 'ms' | 'us';
+};
+
+export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
+
+/**
+ * Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
+ * populates `results`, and calls `reporter` in `afterAll`.
+ *
+ * Kept free of Node-only deps (`fs`, `path`) so the same runner works under
+ * vitest-browser for IndexedDB and SQLite-OPFS.
+ */
+export function describeAztecMapBench(
+  backendPrefix: string,
+  getStore: () => Promise<AztecAsyncKVStore>,
+  logger: Logger,
+  reporter: BenchReporter,
+) {
+  describe(`${backendPrefix} Map benchmarks`, () => {
+    let store: AztecAsyncKVStore;
+    let map: AztecAsyncMap<Key, string>;
+
+    const results: BenchResult[] = [];
+
+    const generateKeyValuePairs = (count: number, offset = 0) => {
+      const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
+      const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
+      return keys.map((key, i) => ({ key, value: values[i] }));
+    };
+
+    const record = (name: string, value: number, unit: BenchResult['unit']) => {
+      results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
+    };
+
+    beforeEach(async () => {
+      store = await getStore();
+      map = store.openMap<Key, string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    afterAll(async () => {
+      const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
+      logger.info(`\n${pretty}\n`);
+      await reporter(results);
+    });
+
+    it('adds individual values', async () => {
+      const pairs = generateKeyValuePairs(1000);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.set(pair.key, pair.value);
+      }
+      record('Individual insertion', timer.ms() / pairs.length, 'ms');
+    });
+
+    it('adds batched values', async () => {
+      const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
+      const timer = new Timer();
+      for (const batch of batches) {
+        await map.setMany(batch);
+      }
+      record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
+    });
+
+    it('reads individual values', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.getAsync(pair.key);
+      }
+      record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads via a cursor', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for await (const _ of map.entriesAsync()) {
+        // consume
+      }
+      record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads the size of the map', async () => {
+      const numIterations = 1000;
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (let i = 0; i < numIterations; i++) {
+        await map.sizeAsync();
+      }
+      record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
+    });
+  });
+}

package/src/indexeddb/index.ts

@@ -1,6 +1,6 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
-import type { DataStoreConfig } from '../config.js';
 import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecIndexedDBStore } from './store.js';
 

package/src/lmdb/index.ts

@@ -1,8 +1,8 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
 import { join } from 'path';
 
-import type { DataStoreConfig } from '../config.js';
 import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecLmdbStore } from './store.js';
 

package/src/lmdb/store.ts

@@ -147,21 +147,25 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
   }
 
   /**
-   * Clears all entries in the store & sub DBs.
+   * Clears all entries in the store & sub DBs atomically within a single transaction.
    */
   async clear() {
-    await this.#data.clearAsync();
-    await this.#multiMapData.clearAsync();
-    await this.#rootDb.clearAsync();
+    await this.#rootDb.transaction(async () => {
+      await this.#data.clearAsync();
+      await this.#multiMapData.clearAsync();
+      await this.#rootDb.clearAsync();
+    });
   }
 
   /**
-   * Drops the database & sub DBs.
+   * Drops the database & sub DBs atomically within a single transaction.
    */
   async drop() {
-    await this.#data.drop();
-    await this.#multiMapData.drop();
-    await this.#rootDb.drop();
+    await this.#rootDb.transaction(async () => {
+      await this.#data.drop();
+      await this.#multiMapData.drop();
+      await this.#rootDb.drop();
+    });
   }
 
   /**

package/src/lmdb-v2/factory.ts

@@ -1,12 +1,12 @@
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
 import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
-import type { DataStoreConfig } from '../config.js';
 import { AztecLMDBStoreV2 } from './store.js';
 
 const MAX_READERS = 16;