@aztec/kv-store 0.0.1-commit.e588bc7e5 → 0.0.1-commit.e5a3663dd

package/dest/sqlite-opfs/worker.js

@@ -0,0 +1,225 @@
+/// <reference lib="webworker" />
+import sqlite3InitModule from '@aztec/sqlite3mc-wasm';
+const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS data (
+    slot TEXT NOT NULL PRIMARY KEY,
+    container TEXT NOT NULL,
+    key BLOB NOT NULL,
+    key_count INTEGER NOT NULL,
+    hash TEXT NOT NULL,
+    value BLOB
+  ) WITHOUT ROWID;
+
+  CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
+`;
+const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
+const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
+const MC_SAH_POOL_VFS_NAME = `multipleciphers-${SAH_POOL_VFS_NAME}`;
+let sqlite3;
+let pool;
+let db;
+let dbPath;
+async function ensurePool(directory) {
+    sqlite3 ??= await sqlite3InitModule();
+    const s = sqlite3;
+    if (!pool) {
+        pool = await s.installOpfsSAHPoolVfs({
+            name: SAH_POOL_VFS_NAME,
+            directory,
+            initialCapacity: 8
+        });
+        // Register a sqlite3mc-wrapped VFS pointing at our SAH Pool VFS.
+        // Encrypted DBs must be opened through this wrapper so sqlite3mc can
+        // intercept file I/O; plain DBs continue using the SAH Pool VFS directly.
+        // The wrapper name is `multipleciphers-<underlying>`.
+        s.capi.sqlite3mc_vfs_create(SAH_POOL_VFS_NAME, 0);
+    }
+    return pool;
+}
+/**
+ * Applies sqlite3mc's ChaCha20 page cipher using a pre-derived 32-byte key.
+ * The PRAGMAs must run before any schema DDL so sqlite3mc can decrypt existing
+ * pages and encrypt new ones. Zeroes the caller-held key array after the PRAGMA
+ * completes to minimize residency of the raw key bytes outside sqlite3mc's heap.
+ */ function applyEncryptionKey(conn, key) {
+    const hex = Array.from(key, (b)=>b.toString(16).padStart(2, '0')).join('');
+    conn.exec(`PRAGMA cipher = 'chacha20'`);
+    conn.exec(`PRAGMA key = "x'${hex}'"`);
+    key.fill(0);
+}
+async function handleInit(dbName, ephemeral, directory, encryptionKey) {
+    sqlite3 ??= await sqlite3InitModule();
+    const s = sqlite3;
+    if (encryptionKey !== undefined && ephemeral) {
+        throw new Error('encryptionKey is not supported for ephemeral (:memory:) stores');
+    }
+    if (ephemeral) {
+        db = new s.oo1.DB(':memory:', 'c');
+    } else {
+        await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
+        dbPath = normalizeDbPath(dbName);
+        if (encryptionKey !== undefined) {
+            db = new s.oo1.DB({
+                filename: dbPath,
+                flags: 'c',
+                vfs: MC_SAH_POOL_VFS_NAME
+            });
+            applyEncryptionKey(db, encryptionKey);
+        } else {
+            db = new pool.OpfsSAHPoolDb(dbPath);
+        }
+    }
+    runSql(SCHEMA_SQL);
+}
+function handleClose() {
+    db?.close();
+    db = undefined;
+    dbPath = undefined;
+}
+async function handleExport() {
+    if (!db || !dbPath) {
+        throw new Error('SQLite worker: no database open to export');
+    }
+    if (!pool) {
+        throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
+    }
+    return await pool.exportFile(dbPath);
+}
+function handleDeleteDb(dbName) {
+    const path = normalizeDbPath(dbName);
+    if (db && dbPath === path) {
+        db.close();
+        db = undefined;
+        dbPath = undefined;
+    }
+    // Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
+    // nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
+    // directory, and under heavy test churn that can contend with workers from
+    // previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
+    // and then the whole test run.
+    if (!pool) {
+        return;
+    }
+    try {
+        pool.unlink(path);
+    } catch  {
+    // File may not exist; ignore.
+    }
+}
+function requireDb() {
+    if (!db) {
+        throw new Error('SQLite worker: no database open');
+    }
+    return db;
+}
+function runSql(sql, bind) {
+    const conn = requireDb();
+    conn.exec({
+        sql,
+        bind
+    });
+    return {
+        changes: conn.changes()
+    };
+}
+function selectAll(sql, bind) {
+    const conn = requireDb();
+    const rows = [];
+    conn.exec({
+        sql,
+        bind,
+        rowMode: 'array',
+        resultRows: rows
+    });
+    return rows;
+}
+function normalizeDbPath(dbName) {
+    return dbName.startsWith('/') ? dbName : `/${dbName}`;
+}
+function respond(msg) {
+    self.postMessage(msg);
+}
+self.onmessage = async (ev)=>{
+    const req = ev.data;
+    try {
+        switch(req.type){
+            case 'init':
+                await handleInit(req.dbName, req.ephemeral, req.poolDirectory, req.encryptionKey);
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'close':
+                handleClose();
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'deleteDb':
+                handleDeleteDb(req.dbName);
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'run':
+                {
+                    const { changes } = runSql(req.sql, req.bind);
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        changes
+                    });
+                }
+            case 'all':
+                {
+                    const rows = selectAll(req.sql, req.bind);
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        rows
+                    });
+                }
+            case 'export':
+                {
+                    const bytes = await handleExport();
+                    return respond({
+                        type: 'ok',
+                        id: req.id,
+                        bytes
+                    });
+                }
+            case 'begin':
+                runSql('BEGIN');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'commit':
+                runSql('COMMIT');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            case 'rollback':
+                runSql('ROLLBACK');
+                return respond({
+                    type: 'ok',
+                    id: req.id
+                });
+            default:
+                {
+                    const _exhaustive = req;
+                    throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
+                }
+        }
+    } catch (err) {
+        const message = err instanceof Error ? err.message : String(err);
+        respond({
+            type: 'err',
+            id: req.id,
+            message
+        });
+    }
+};

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "@aztec/kv-store",
-  "version": "0.0.1-commit.e588bc7e5",
+  "version": "0.0.1-commit.e5a3663dd",
   "type": "module",
   "exports": {
     ".": "./dest/interfaces/index.js",
@@ -8,6 +8,7 @@
     "./lmdb": "./dest/lmdb/index.js",
     "./lmdb-v2": "./dest/lmdb-v2/index.js",
     "./indexeddb": "./dest/indexeddb/index.js",
+    "./sqlite-opfs": "./dest/sqlite-opfs/index.js",
     "./stores": "./dest/stores/index.js"
   },
   "scripts": {
@@ -15,7 +16,8 @@
     "build:dev": "../scripts/tsc.sh --watch",
     "clean": "rm -rf ./dest .tsbuildinfo",
     "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
-    "test:browser": "vitest run --config ./vitest.config.ts",
+    "test:browser": "bash scripts/run-browser-tests.sh",
+    "bench:browser": "VITE_BENCH=1 vitest run --config ./vitest.config.ts src/bench",
     "test": "yarn test:node && yarn test:browser",
     "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
   },
@@ -24,11 +26,12 @@
     "./package.local.json"
   ],
   "dependencies": {
-    "@aztec/constants": "0.0.1-commit.e588bc7e5",
-    "@aztec/ethereum": "0.0.1-commit.e588bc7e5",
-    "@aztec/foundation": "0.0.1-commit.e588bc7e5",
-    "@aztec/native": "0.0.1-commit.e588bc7e5",
-    "@aztec/stdlib": "0.0.1-commit.e588bc7e5",
+    "@aztec/constants": "0.0.1-commit.e5a3663dd",
+    "@aztec/ethereum": "0.0.1-commit.e5a3663dd",
+    "@aztec/foundation": "0.0.1-commit.e5a3663dd",
+    "@aztec/native": "0.0.1-commit.e5a3663dd",
+    "@aztec/sqlite3mc-wasm": "0.0.1-commit.e5a3663dd",
+    "@aztec/stdlib": "0.0.1-commit.e5a3663dd",
     "idb": "^8.0.0",
     "lmdb": "^3.2.0",
     "msgpackr": "^1.11.2",

package/src/bench/shared_map_bench.ts

@@ -0,0 +1,111 @@
+import type { Logger } from '@aztec/foundation/log';
+import { Timer } from '@aztec/foundation/timer';
+
+import type { Key } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+
+/** One benchmark measurement. */
+export type BenchResult = {
+  /** Benchmark name (includes the backend prefix for disambiguation). */
+  name: string;
+  value: number;
+  unit: 'ms' | 'us';
+};
+
+export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
+
+/**
+ * Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
+ * populates `results`, and calls `reporter` in `afterAll`.
+ *
+ * Kept free of Node-only deps (`fs`, `path`) so the same runner works under
+ * vitest-browser for IndexedDB and SQLite-OPFS.
+ */
+export function describeAztecMapBench(
+  backendPrefix: string,
+  getStore: () => Promise<AztecAsyncKVStore>,
+  logger: Logger,
+  reporter: BenchReporter,
+) {
+  describe(`${backendPrefix} Map benchmarks`, () => {
+    let store: AztecAsyncKVStore;
+    let map: AztecAsyncMap<Key, string>;
+
+    const results: BenchResult[] = [];
+
+    const generateKeyValuePairs = (count: number, offset = 0) => {
+      const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
+      const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
+      return keys.map((key, i) => ({ key, value: values[i] }));
+    };
+
+    const record = (name: string, value: number, unit: BenchResult['unit']) => {
+      results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
+    };
+
+    beforeEach(async () => {
+      store = await getStore();
+      map = store.openMap<Key, string>('test');
+    });
+
+    afterEach(async () => {
+      await store.delete();
+    });
+
+    afterAll(async () => {
+      const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
+      logger.info(`\n${pretty}\n`);
+      await reporter(results);
+    });
+
+    it('adds individual values', async () => {
+      const pairs = generateKeyValuePairs(1000);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.set(pair.key, pair.value);
+      }
+      record('Individual insertion', timer.ms() / pairs.length, 'ms');
+    });
+
+    it('adds batched values', async () => {
+      const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
+      const timer = new Timer();
+      for (const batch of batches) {
+        await map.setMany(batch);
+      }
+      record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
+    });
+
+    it('reads individual values', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (const pair of pairs) {
+        await map.getAsync(pair.key);
+      }
+      record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads via a cursor', async () => {
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for await (const _ of map.entriesAsync()) {
+        // consume
+      }
+      record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
+    });
+
+    it('reads the size of the map', async () => {
+      const numIterations = 1000;
+      const pairs = generateKeyValuePairs(10000);
+      await map.setMany(pairs);
+      const timer = new Timer();
+      for (let i = 0; i < numIterations; i++) {
+        await map.sizeAsync();
+      }
+      record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
+    });
+  });
+}

package/src/lmdb-v2/read_transaction.ts

@@ -66,20 +66,20 @@ export class ReadTransaction {
   ): AsyncIterable<[Uint8Array, T]> {
     this.assertIsOpen();
 
-    const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
-      key: startKey,
-      reverse,
-      count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
-      onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
-      db,
-    });
-
-    const cursor = response.cursor;
-    let entries = response.entries;
-    let done = typeof cursor !== 'number';
-    let count = 0;
-
+    let cursor: number | undefined;
     try {
+      const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
+        key: startKey,
+        reverse,
+        count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
+        onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
+        db,
+      });
+
+      cursor = response.cursor ?? undefined;
+      let entries = response.entries;
+      let done = typeof cursor !== 'number';
+      let count = 0;
       // emit the first page and any subsequent pages in a while loop
       // NB: end contition is in the middle of the while loop
       while (entries.length > 0) {
@@ -125,17 +125,17 @@ export class ReadTransaction {
   async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
     this.assertIsOpen();
 
-    const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
-      key: startKey,
-      reverse,
-      count: 0,
-      onePage: false,
-      db,
-    });
-
-    const cursor = response.cursor;
-
+    let cursor: number | undefined;
     try {
+      const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
+        key: startKey,
+        reverse,
+        count: 0,
+        onePage: false,
+        db,
+      });
+
+      cursor = response.cursor ?? undefined;
       if (!cursor) {
         return 0;
       }

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { toBufferKey } from 'ordered-binary';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,37 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  const { dataDirectory } = config;
+  log.info(
+    dataDirectory
+      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts?.rollupAddress, log);
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}