@aztec/kv-store 0.0.1-commit.9372f48 → 0.0.1-commit.949a33fd8

package/src/sqlite-opfs/store.ts

@@ -0,0 +1,248 @@
+import type { Logger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
+import type { AztecAsyncCounter } from '../interfaces/counter.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+import { SQLiteOPFSAztecArray } from './array.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
+import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
+import { SQLiteOPFSAztecSet } from './set.js';
+import { SQLiteOPFSAztecSingleton } from './singleton.js';
+
+/**
+ * Main-thread handle for a SQLite database persisted to OPFS via the `opfs-sahpool`
+ * VFS. Owns a dedicated Web Worker (the SAH Pool VFS requires Worker context) and
+ * routes every SQL op through it via typed postMessage RPC.
+ *
+ * Transaction ordering is guaranteed by a `SerialQueue` on the main thread combined
+ * with an `#inTx` flag: outside a `transactionAsync` block, each op acquires the
+ * queue for its own auto-commit; inside a block, the outer call holds the queue and
+ * nested ops bypass it to avoid deadlock.
+ */
+export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
+  readonly #worker: Worker;
+  readonly #pending = new Map<number, { resolve: (r: WorkerResponse) => void; reject: (err: Error) => void }>();
+  readonly #txQueue = new SerialQueue();
+  readonly #name: string;
+  readonly #log: Logger;
+  #nextId = 0;
+  #inTx = false;
+  #closed = false;
+
+  private constructor(
+    worker: Worker,
+    name: string,
+    log: Logger,
+    public readonly isEphemeral: boolean,
+  ) {
+    this.#worker = worker;
+    this.#name = name;
+    this.#log = log;
+    this.#worker.onmessage = (ev: MessageEvent<WorkerResponse>) => {
+      const { id } = ev.data;
+      const handler = this.#pending.get(id);
+      if (!handler) {
+        this.#log.warn(`SQLite worker: no pending handler for id ${id}`);
+        return;
+      }
+      this.#pending.delete(id);
+      handler.resolve(ev.data);
+    };
+    this.#worker.onerror = ev => {
+      this.#log.error(`SQLite worker crashed: ${ev.message}`);
+      this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
+    };
+    this.#txQueue.start();
+  }
+
+  /**
+   * Opens (or creates) a SQLite database stored in the OPFS SAH Pool. When `ephemeral`
+   * is true the database lives only in memory and is lost when the worker terminates.
+   * Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
+   * required when multiple stores coexist in the same tab, because the SAH Pool holds
+   * an exclusive lock on its directory.
+   */
+  static async open(
+    log: Logger,
+    name?: string,
+    ephemeral: boolean = false,
+    poolDirectory?: string,
+  ): Promise<AztecSQLiteOPFSStore> {
+    const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
+    log.debug(`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}database ${dbName}`);
+    const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
+    const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
+    await store.#sendRequest({ type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory });
+    return store;
+  }
+
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
+    return new SQLiteOPFSAztecMap<K, V>(this, name);
+  }
+
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    return new SQLiteOPFSAztecSet<K>(this, name);
+  }
+
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
+    return new SQLiteOPFSAztecMultiMap<K, V>(this, name);
+  }
+
+  openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
+    throw new Error('Method not implemented.');
+  }
+
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
+    return new SQLiteOPFSAztecArray<T>(this, name);
+  }
+
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
+    return new SQLiteOPFSAztecSingleton<T>(this, name);
+  }
+
+  transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    // Nested calls join the outer transaction — SQLite does not support nested BEGIN,
+    // and re-acquiring the SerialQueue while the outer call holds it would deadlock.
+    // Errors in the nested callback propagate to the outer catch, which rolls back the
+    // whole thing (the standard "nested tx = savepoint-free join" semantic).
+    if (this.#inTx) {
+      return callback();
+    }
+    return this.#txQueue.put(async () => {
+      this.#inTx = true;
+      await this.#sendRequest({ type: 'begin', id: this.#allocId() });
+      try {
+        const result = await callback();
+        await this.#sendRequest({ type: 'commit', id: this.#allocId() });
+        return result;
+      } catch (err) {
+        await this.#sendRequest({ type: 'rollback', id: this.#allocId() }).catch(rollbackErr =>
+          this.#log.warn(`SQLite ROLLBACK failed: ${rollbackErr instanceof Error ? rollbackErr.message : rollbackErr}`),
+        );
+        throw err;
+      } finally {
+        this.#inTx = false;
+      }
+    });
+  }
+
+  async clear(): Promise<void> {
+    await this.runAsync('DELETE FROM data');
+  }
+
+  async delete(): Promise<void> {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    await this.#txQueue.end();
+    await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
+      this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
+    );
+    this.#worker.terminate();
+    this.#rejectPending('SQLite store deleted');
+  }
+
+  /**
+   * Placeholder — returns zeros to mirror the IndexedDB backend. SQLite exposes real
+   * numbers cheaply via `PRAGMA page_count` / `page_size` / `freelist_count` and
+   * `SELECT COUNT(*) FROM data`, which would populate `physicalFileSize`, `actualSize`,
+   * and `numItems` meaningfully (`mappingSize` stays 0 — it's an LMDB mmap concept).
+   * Upgrade when any caller actually consumes these values; all current consumers
+   * tolerate zeros.
+   */
+  estimateSize(): Promise<StoreSize> {
+    return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    await this.#txQueue.end();
+    await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
+    this.#worker.terminate();
+    this.#rejectPending('SQLite store closed');
+  }
+
+  backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+
+  /**
+   * Returns a raw SQLite image (bytes suitable for writing as a `.sqlite` file and
+   * opening in any SQLite tool). Works only for non-ephemeral DBs because the OPFS
+   * SAH Pool has to be initialized. Useful for inspection/debugging.
+   */
+  async exportDb(): Promise<Uint8Array> {
+    const resp = await this.#sendRequest({ type: 'export', id: this.#allocId() });
+    if (!('bytes' in resp) || !resp.bytes) {
+      throw new Error('exportDb: worker returned no bytes');
+    }
+    return resp.bytes;
+  }
+
+  /**
+   * Runs a write statement (INSERT/UPDATE/DELETE/DDL). If called inside a
+   * `transactionAsync` block, bypasses the queue; otherwise acquires it so the
+   * op runs in its own auto-commit.
+   */
+  runAsync(sql: string, bind?: SqlValue[]): Promise<{ changes: number }> {
+    const send = () =>
+      this.#sendRequest({ type: 'run', id: this.#allocId(), sql, bind }).then(r => ({
+        changes: 'changes' in r ? (r.changes ?? 0) : 0,
+      }));
+    return this.#inTx ? send() : this.#txQueue.put(send);
+  }
+
+  /** Runs a SELECT statement and returns rows in array row-mode. */
+  allAsync(sql: string, bind?: SqlValue[]): Promise<ResultRow[]> {
+    const send = () =>
+      this.#sendRequest({ type: 'all', id: this.#allocId(), sql, bind }).then(r => ('rows' in r ? (r.rows ?? []) : []));
+    return this.#inTx ? send() : this.#txQueue.put(send);
+  }
+
+  #allocId(): number {
+    return ++this.#nextId;
+  }
+
+  /**
+   * Reject any in-flight requests with `reason`, so callers awaiting a response to a
+   * request sent to a now-terminated worker don't hang forever. Called from
+   * close()/delete() and from the worker.onerror handler.
+   */
+  #rejectPending(reason: string): void {
+    if (this.#pending.size === 0) {
+      return;
+    }
+    const err = new Error(reason);
+    for (const { reject } of this.#pending.values()) {
+      reject(err);
+    }
+    this.#pending.clear();
+  }
+
+  #sendRequest(req: WorkerRequest): Promise<WorkerResponse> {
+    return new Promise<WorkerResponse>((resolve, reject) => {
+      this.#pending.set(req.id, {
+        resolve: resp => {
+          if (resp.type === 'err') {
+            reject(new Error(resp.message));
+          } else {
+            resolve(resp);
+          }
+        },
+        reject,
+      });
+      this.#worker.postMessage(req);
+    });
+  }
+}

package/src/sqlite-opfs/worker.ts

@@ -0,0 +1,162 @@
+/// <reference lib="webworker" />
+import sqlite3InitModule, { type Database, type SAHPoolUtil, type Sqlite3Static } from '@sqlite.org/sqlite-wasm';
+
+import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
+
+const SCHEMA_SQL = `
+  CREATE TABLE IF NOT EXISTS data (
+    slot TEXT NOT NULL PRIMARY KEY,
+    container TEXT NOT NULL,
+    key BLOB NOT NULL,
+    key_count INTEGER NOT NULL,
+    hash TEXT NOT NULL,
+    value BLOB
+  ) WITHOUT ROWID;
+
+  CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
+  CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
+`;
+
+const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
+const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
+
+let sqlite3: Sqlite3Static | undefined;
+let pool: SAHPoolUtil | undefined;
+let db: Database | undefined;
+let dbPath: string | undefined;
+
+async function ensurePool(directory: string): Promise<SAHPoolUtil> {
+  sqlite3 ??= await sqlite3InitModule();
+  if (!pool) {
+    pool = await sqlite3.installOpfsSAHPoolVfs({
+      name: SAH_POOL_VFS_NAME,
+      directory,
+      initialCapacity: 8,
+    });
+  }
+  return pool;
+}
+
+async function handleInit(dbName: string, ephemeral: boolean, directory?: string): Promise<void> {
+  sqlite3 ??= await sqlite3InitModule();
+  if (ephemeral) {
+    db = new sqlite3.oo1.DB(':memory:', 'c');
+  } else {
+    const p = await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
+    dbPath = normalizeDbPath(dbName);
+    db = new p.OpfsSAHPoolDb(dbPath);
+  }
+  runSql(SCHEMA_SQL);
+}
+
+function handleClose(): void {
+  db?.close();
+  db = undefined;
+  dbPath = undefined;
+}
+
+async function handleExport(): Promise<Uint8Array> {
+  if (!db || !dbPath) {
+    throw new Error('SQLite worker: no database open to export');
+  }
+  if (!pool) {
+    throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
+  }
+  return await pool.exportFile(dbPath);
+}
+
+function handleDeleteDb(dbName: string): void {
+  const path = normalizeDbPath(dbName);
+  if (db && dbPath === path) {
+    db.close();
+    db = undefined;
+    dbPath = undefined;
+  }
+  // Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
+  // nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
+  // directory, and under heavy test churn that can contend with workers from
+  // previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
+  // and then the whole test run.
+  if (!pool) {
+    return;
+  }
+  try {
+    pool.unlink(path);
+  } catch {
+    // File may not exist; ignore.
+  }
+}
+
+function requireDb(): Database {
+  if (!db) {
+    throw new Error('SQLite worker: no database open');
+  }
+  return db;
+}
+
+function runSql(sql: string, bind?: SqlValue[]): { changes: number } {
+  const conn = requireDb();
+  conn.exec({ sql, bind });
+  return { changes: conn.changes() };
+}
+
+function selectAll(sql: string, bind?: SqlValue[]): ResultRow[] {
+  const conn = requireDb();
+  const rows: ResultRow[] = [];
+  conn.exec({ sql, bind, rowMode: 'array', resultRows: rows });
+  return rows;
+}
+
+function normalizeDbPath(dbName: string): string {
+  return dbName.startsWith('/') ? dbName : `/${dbName}`;
+}
+
+function respond(msg: WorkerResponse): void {
+  (self as DedicatedWorkerGlobalScope).postMessage(msg);
+}
+
+(self as DedicatedWorkerGlobalScope).onmessage = async (ev: MessageEvent<WorkerRequest>) => {
+  const req = ev.data;
+  try {
+    switch (req.type) {
+      case 'init':
+        await handleInit(req.dbName, req.ephemeral, req.poolDirectory);
+        return respond({ type: 'ok', id: req.id });
+      case 'close':
+        handleClose();
+        return respond({ type: 'ok', id: req.id });
+      case 'deleteDb':
+        handleDeleteDb(req.dbName);
+        return respond({ type: 'ok', id: req.id });
+      case 'run': {
+        const { changes } = runSql(req.sql, req.bind);
+        return respond({ type: 'ok', id: req.id, changes });
+      }
+      case 'all': {
+        const rows = selectAll(req.sql, req.bind);
+        return respond({ type: 'ok', id: req.id, rows });
+      }
+      case 'export': {
+        const bytes = await handleExport();
+        return respond({ type: 'ok', id: req.id, bytes });
+      }
+      case 'begin':
+        runSql('BEGIN');
+        return respond({ type: 'ok', id: req.id });
+      case 'commit':
+        runSql('COMMIT');
+        return respond({ type: 'ok', id: req.id });
+      case 'rollback':
+        runSql('ROLLBACK');
+        return respond({ type: 'ok', id: req.id });
+      default: {
+        const _exhaustive: never = req;
+        throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
+      }
+    }
+  } catch (err) {
+    const message = err instanceof Error ? err.message : String(err);
+    respond({ type: 'err', id: req.id, message });
+  }
+};

package/src/utils.ts

@@ -33,7 +33,14 @@ export async function initStoreForRollupAndSchemaVersion<T extends AztecKVStore
     ? (dbVersion as AztecSingleton<string>).get()
     : await (dbVersion as AztecAsyncSingleton<string>).getAsync();
 
+  // Check if this is an old format store (has rollupAddress singleton but no dbVersion)
+  const oldRollupSingleton = store.openSingleton<string>('rollupAddress');
+  const hasOldFormat = isSyncStore(store)
+    ? !storedDatabaseVersion && !!(oldRollupSingleton as AztecSingleton<string>).get()
+    : !storedDatabaseVersion && !!(await (oldRollupSingleton as AztecAsyncSingleton<string>).getAsync());
+
   if (
+    hasOldFormat ||
     doesStoreNeedToBeCleared(
       targetDatabaseVersion,
       storedDatabaseVersion,
@@ -42,6 +49,9 @@ export async function initStoreForRollupAndSchemaVersion<T extends AztecKVStore
       log,
     )
   ) {
+    if (hasOldFormat) {
+      log?.warn('Detected old store format without dbVersion, clearing database');
+    }
     await store.clear();
   }
 

package/dest/config.d.ts

@@ -1,17 +0,0 @@
-import { type ConfigMappingsType } from '@aztec/foundation/config';
-import type { EthAddress } from '@aztec/foundation/eth-address';
-export type DataStoreConfig = {
-    dataDirectory: string | undefined;
-    dataStoreMapSizeKb: number;
-    l1Contracts?: {
-        rollupAddress: EthAddress;
-    };
-};
-export declare const dataConfigMappings: ConfigMappingsType<DataStoreConfig>;
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */
-export declare function getDataConfigFromEnv(): DataStoreConfig;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUE2QyxNQUFNLDBCQUEwQixDQUFDO0FBQzlHLE9BQU8sS0FBSyxFQUFFLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBRWhFLE1BQU0sTUFBTSxlQUFlLEdBQUc7SUFDNUIsYUFBYSxFQUFFLE1BQU0sR0FBRyxTQUFTLENBQUM7SUFDbEMsa0JBQWtCLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFO1FBQUUsYUFBYSxFQUFFLFVBQVUsQ0FBQTtLQUFFLENBQUM7Q0FDN0MsQ0FBQztBQUVGLGVBQU8sTUFBTSxrQkFBa0IsRUFBRSxrQkFBa0IsQ0FBQyxlQUFlLENBZ0JsRSxDQUFDO0FBRUY7Ozs7R0FJRztBQUNILHdCQUFnQixvQkFBb0IsSUFBSSxlQUFlLENBRXREIn0=

package/dest/config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,kBAAkB,EAA6C,MAAM,0BAA0B,CAAC;AAC9G,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE;QAAE,aAAa,EAAE,UAAU,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,kBAAkB,CAAC,eAAe,CAgBlE,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,eAAe,CAEtD"}

package/dest/config.js

@@ -1,26 +0,0 @@
-import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
-import { getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
-export const dataConfigMappings = {
-    dataDirectory: {
-        env: 'DATA_DIRECTORY',
-        description: 'Optional dir to store data. If omitted will store in memory.'
-    },
-    dataStoreMapSizeKb: {
-        env: 'DATA_STORE_MAP_SIZE_KB',
-        description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
-        ...numberConfigHelper(128 * 1_024 * 1_024)
-    },
-    l1Contracts: {
-        description: 'The deployed L1 contract addresses',
-        nested: {
-            rollupAddress: l1ContractAddressesMapping.rollupAddress
-        }
-    }
-};
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */ export function getDataConfigFromEnv() {
-    return getConfigFromMappings(dataConfigMappings);
-}

package/src/config.ts

@@ -1,36 +0,0 @@
-import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
-import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
-import type { EthAddress } from '@aztec/foundation/eth-address';
-
-export type DataStoreConfig = {
-  dataDirectory: string | undefined;
-  dataStoreMapSizeKb: number;
-  l1Contracts?: { rollupAddress: EthAddress };
-};
-
-export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
-  dataDirectory: {
-    env: 'DATA_DIRECTORY',
-    description: 'Optional dir to store data. If omitted will store in memory.',
-  },
-  dataStoreMapSizeKb: {
-    env: 'DATA_STORE_MAP_SIZE_KB',
-    description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
-    ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
-  },
-  l1Contracts: {
-    description: 'The deployed L1 contract addresses',
-    nested: {
-      rollupAddress: l1ContractAddressesMapping.rollupAddress,
-    },
-  },
-};
-
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */
-export function getDataConfigFromEnv(): DataStoreConfig {
-  return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
-}