@aztec/kv-store 0.0.1-commit.936cb2cae → 0.0.1-commit.949a33fd8

package/src/sqlite-opfs/map.ts

@@ -0,0 +1,163 @@
+import { Buffer } from 'buffer';
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { fromBufferKey, toBufferKey } from 'ordered-binary';
+
+import type { Key, Range, Value } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { SqlValue } from './messages.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** A map backed by SQLite in OPFS. Mirrors `IndexedDBAztecMap`. */
+export class SQLiteOPFSAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
+  protected readonly name: string;
+  protected readonly container: string;
+  protected readonly encoder = new Encoder();
+
+  constructor(
+    protected readonly store: AztecSQLiteOPFSStore,
+    mapName: string,
+  ) {
+    this.name = mapName;
+    this.container = `map:${mapName}`;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.slot(key)]);
+    if (rows.length === 0) {
+      return undefined;
+    }
+    const raw = rows[0][0];
+    return raw == null ? undefined : this.decodeValue(raw);
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const rows = await this.store.allAsync('SELECT 1 FROM data WHERE slot = ? LIMIT 1', [this.slot(key)]);
+    return rows.length > 0;
+  }
+
+  async sizeAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ?', [this.container]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    await this.store.runAsync(
+      `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [this.slot(key), this.container, this.encodedKey(key), 1, hash(val), this.encoder.pack(val)],
+    );
+  }
+
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    if (entries.length === 0) {
+      return;
+    }
+    await this.store.transactionAsync(async () => {
+      for (const { key, value } of entries) {
+        await this.set(key, value);
+      }
+    });
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      if (await this.hasAsync(key)) {
+        return false;
+      }
+      await this.set(key, val);
+      return true;
+    });
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE slot = ?', [this.slot(key)]);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const rows = await this.rangeQuery(range);
+    for (const row of rows) {
+      const [_slot, keyBlob, value] = row;
+      if (keyBlob == null || value == null) {
+        continue;
+      }
+      yield [this.decodeKey(keyBlob), this.decodeValue(value)];
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  protected async rangeQuery(range: Range<K>): Promise<Array<[string, Uint8Array, Uint8Array | null]>> {
+    // Inclusivity flips with direction to match the IndexedDB backend:
+    //   forward: [start, end)     reverse: (start, end]
+    // That asymmetry is load-bearing — tests pin the exact inclusivity at boundaries.
+    const reverse = !!range.reverse;
+    const parts: string[] = ['container = ?'];
+    const bind: SqlValue[] = [this.container];
+    if (range.start !== undefined) {
+      parts.push(reverse ? 'key > ?' : 'key >= ?');
+      bind.push(this.encodedKey(range.start));
+    }
+    if (range.end !== undefined) {
+      parts.push(reverse ? 'key <= ?' : 'key < ?');
+      bind.push(this.encodedKey(range.end));
+    }
+    const order = reverse ? 'DESC' : 'ASC';
+    let sql = `SELECT slot, key, value FROM data WHERE ${parts.join(' AND ')} ORDER BY key ${order}, key_count ${order}`;
+    if (range.limit !== undefined) {
+      sql += ' LIMIT ?';
+      bind.push(range.limit);
+    }
+    const rows = await this.store.allAsync(sql, bind);
+    return rows.map(r => [String(r[0]), r[1] as Uint8Array, r[2] as Uint8Array | null]);
+  }
+
+  protected decodeValue(val: SqlValue): V {
+    if (!(val instanceof Uint8Array)) {
+      return val as V;
+    }
+    const unpacked = this.encoder.unpack(val);
+    // msgpackr returns plain Uint8Array in browsers for packed Buffers. Callers that
+    // stored Buffers (walletDB uses Buffer.from(...).toString('utf8') round-trips)
+    // rely on Buffer-flavored behavior — re-wrap at the storage boundary, mirroring
+    // IndexedDBAztecMap.restoreBuffers.
+    if (unpacked instanceof Uint8Array && !Buffer.isBuffer(unpacked)) {
+      return Buffer.from(unpacked) as V;
+    }
+    return unpacked as V;
+  }
+
+  protected decodeKey(raw: Uint8Array): K {
+    const parsed = fromBufferKey(Buffer.from(raw));
+    if (Array.isArray(parsed)) {
+      return (parsed.length > 1 ? parsed : parsed[0]) as K;
+    }
+    return parsed as K;
+  }
+
+  protected encodedKey(key: K): Buffer {
+    return toBufferKey(this.normalizeKey(key));
+  }
+
+  protected normalizeKey(key: K): (string | number | Uint8Array)[] {
+    return Array.isArray(key) ? key : [key];
+  }
+
+  protected slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
+  }
+}

package/src/sqlite-opfs/messages.ts

@@ -0,0 +1,28 @@
+/**
+ * RPC protocol between the main thread and the SQLite worker.
+ * All requests carry a unique `id`; responses echo the same id so the main thread
+ * can resolve the right pending promise.
+ */
+
+/** Matches `@sqlite.org/sqlite-wasm`'s internal SqlValue type. Boolean is not a native SQLite type. */
+export type SqlValue = string | number | bigint | null | Uint8Array;
+
+/** A row returned in 'array' rowMode — one value per column, in select order. */
+export type ResultRow = SqlValue[];
+
+export type WorkerRequest =
+  | { type: 'init'; id: number; dbName: string; ephemeral: boolean; poolDirectory?: string }
+  | { type: 'close'; id: number }
+  | { type: 'deleteDb'; id: number; dbName: string }
+  | { type: 'run'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'all'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'export'; id: number }
+  | { type: 'begin'; id: number }
+  | { type: 'commit'; id: number }
+  | { type: 'rollback'; id: number };
+
+export type WorkerResponse =
+  | { type: 'ok'; id: number; rows?: ResultRow[]; changes?: number; bytes?: Uint8Array }
+  | { type: 'err'; id: number; message: string };
+
+export type WorkerRequestType = WorkerRequest['type'];

package/src/sqlite-opfs/multi_map.ts

@@ -0,0 +1,74 @@
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+
+/**
+ * Multi-map backed by SQLite. Extends the base map with always-incrementing
+ * `key_count` per-key so the same slot is never reused across deletions — this
+ * matches the IndexedDB backend's sparse-multi-map semantics.
+ */
+export class SQLiteOPFSAztecMultiMap<K extends Key, V extends Value>
+  extends SQLiteOPFSAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    const valueHash = hash(val);
+    await this.store.transactionAsync(async () => {
+      const exists = await this.store.allAsync(
+        'SELECT 1 FROM data WHERE container = ? AND key = ? AND hash = ? LIMIT 1',
+        [this.container, this.encodedKey(key), valueHash],
+      );
+      if (exists.length > 0) {
+        return;
+      }
+      const maxRow = await this.store.allAsync('SELECT MAX(key_count) FROM data WHERE container = ? AND key = ?', [
+        this.container,
+        this.encodedKey(key),
+      ]);
+      const count = Number(maxRow[0]?.[0] ?? 0);
+      await this.store.runAsync(
+        `INSERT INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.slot(key, count), this.container, this.encodedKey(key), count + 1, valueHash, this.encoder.pack(val)],
+      );
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const rows = await this.store.allAsync(
+      'SELECT value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.container, this.encodedKey(key)],
+    );
+    for (const row of rows) {
+      const raw = row[0];
+      if (raw instanceof Uint8Array) {
+        yield this.decodeValue(raw);
+      }
+    }
+  }
+
+  async getValueCountAsync(key: K): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ? AND hash = ?', [
+      this.container,
+      this.encodedKey(key),
+      hash(val),
+    ]);
+  }
+
+  override async delete(key: K): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+  }
+}

package/src/sqlite-opfs/set.ts

@@ -0,0 +1,29 @@
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Set backed by SQLite. Composes a Map<K, true>. */
+export class SQLiteOPFSAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  readonly #map: SQLiteOPFSAztecMap<K, boolean>;
+
+  constructor(store: AztecSQLiteOPFSStore, name: string) {
+    this.#map = new SQLiteOPFSAztecMap<K, boolean>(store, name);
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.#map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.#map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.#map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.#map.keysAsync(range);
+  }
+}

package/src/sqlite-opfs/singleton.ts

@@ -0,0 +1,48 @@
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { toBufferKey } from 'ordered-binary';
+
+import type { Value } from '../interfaces/common.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Stores a single value identified by `name`. */
+export class SQLiteOPFSAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
+  readonly #container: string;
+  readonly #slot: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot]);
+    if (rows.length === 0) {
+      return undefined;
+    }
+    const raw = rows[0][0];
+    if (raw instanceof Uint8Array) {
+      return this.#encoder.unpack(raw) as T;
+    }
+    return undefined;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const { changes } = await this.store.runAsync(
+      `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [this.#slot, this.#container, toBufferKey([this.#slot]), 1, hash(val), this.#encoder.pack(val)],
+    );
+    return changes > 0;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.store.runAsync('DELETE FROM data WHERE slot = ?', [this.#slot]);
+    return true;
+  }
+}

package/src/sqlite-opfs/store.ts

@@ -0,0 +1,248 @@
+import type { Logger } from '@aztec/foundation/log';
+import { SerialQueue } from '@aztec/foundation/queue';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
+import type { AztecAsyncCounter } from '../interfaces/counter.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
+import { SQLiteOPFSAztecArray } from './array.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
+import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
+import { SQLiteOPFSAztecSet } from './set.js';
+import { SQLiteOPFSAztecSingleton } from './singleton.js';
+
+/**
+ * Main-thread handle for a SQLite database persisted to OPFS via the `opfs-sahpool`
+ * VFS. Owns a dedicated Web Worker (the SAH Pool VFS requires Worker context) and
+ * routes every SQL op through it via typed postMessage RPC.
+ *
+ * Transaction ordering is guaranteed by a `SerialQueue` on the main thread combined
+ * with an `#inTx` flag: outside a `transactionAsync` block, each op acquires the
+ * queue for its own auto-commit; inside a block, the outer call holds the queue and
+ * nested ops bypass it to avoid deadlock.
+ */
+export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
+  readonly #worker: Worker;
+  readonly #pending = new Map<number, { resolve: (r: WorkerResponse) => void; reject: (err: Error) => void }>();
+  readonly #txQueue = new SerialQueue();
+  readonly #name: string;
+  readonly #log: Logger;
+  #nextId = 0;
+  #inTx = false;
+  #closed = false;
+
+  private constructor(
+    worker: Worker,
+    name: string,
+    log: Logger,
+    public readonly isEphemeral: boolean,
+  ) {
+    this.#worker = worker;
+    this.#name = name;
+    this.#log = log;
+    this.#worker.onmessage = (ev: MessageEvent<WorkerResponse>) => {
+      const { id } = ev.data;
+      const handler = this.#pending.get(id);
+      if (!handler) {
+        this.#log.warn(`SQLite worker: no pending handler for id ${id}`);
+        return;
+      }
+      this.#pending.delete(id);
+      handler.resolve(ev.data);
+    };
+    this.#worker.onerror = ev => {
+      this.#log.error(`SQLite worker crashed: ${ev.message}`);
+      this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
+    };
+    this.#txQueue.start();
+  }
+
+  /**
+   * Opens (or creates) a SQLite database stored in the OPFS SAH Pool. When `ephemeral`
+   * is true the database lives only in memory and is lost when the worker terminates.
+   * Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
+   * required when multiple stores coexist in the same tab, because the SAH Pool holds
+   * an exclusive lock on its directory.
+   */
+  static async open(
+    log: Logger,
+    name?: string,
+    ephemeral: boolean = false,
+    poolDirectory?: string,
+  ): Promise<AztecSQLiteOPFSStore> {
+    const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
+    log.debug(`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}database ${dbName}`);
+    const worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
+    const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral);
+    await store.#sendRequest({ type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory });
+    return store;
+  }
+
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
+    return new SQLiteOPFSAztecMap<K, V>(this, name);
+  }
+
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    return new SQLiteOPFSAztecSet<K>(this, name);
+  }
+
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
+    return new SQLiteOPFSAztecMultiMap<K, V>(this, name);
+  }
+
+  openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
+    throw new Error('Method not implemented.');
+  }
+
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
+    return new SQLiteOPFSAztecArray<T>(this, name);
+  }
+
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
+    return new SQLiteOPFSAztecSingleton<T>(this, name);
+  }
+
+  transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
+    // Nested calls join the outer transaction — SQLite does not support nested BEGIN,
+    // and re-acquiring the SerialQueue while the outer call holds it would deadlock.
+    // Errors in the nested callback propagate to the outer catch, which rolls back the
+    // whole thing (the standard "nested tx = savepoint-free join" semantic).
+    if (this.#inTx) {
+      return callback();
+    }
+    return this.#txQueue.put(async () => {
+      this.#inTx = true;
+      await this.#sendRequest({ type: 'begin', id: this.#allocId() });
+      try {
+        const result = await callback();
+        await this.#sendRequest({ type: 'commit', id: this.#allocId() });
+        return result;
+      } catch (err) {
+        await this.#sendRequest({ type: 'rollback', id: this.#allocId() }).catch(rollbackErr =>
+          this.#log.warn(`SQLite ROLLBACK failed: ${rollbackErr instanceof Error ? rollbackErr.message : rollbackErr}`),
+        );
+        throw err;
+      } finally {
+        this.#inTx = false;
+      }
+    });
+  }
+
+  async clear(): Promise<void> {
+    await this.runAsync('DELETE FROM data');
+  }
+
+  async delete(): Promise<void> {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    await this.#txQueue.end();
+    await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
+      this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
+    );
+    this.#worker.terminate();
+    this.#rejectPending('SQLite store deleted');
+  }
+
+  /**
+   * Placeholder — returns zeros to mirror the IndexedDB backend. SQLite exposes real
+   * numbers cheaply via `PRAGMA page_count` / `page_size` / `freelist_count` and
+   * `SELECT COUNT(*) FROM data`, which would populate `physicalFileSize`, `actualSize`,
+   * and `numItems` meaningfully (`mappingSize` stays 0 — it's an LMDB mmap concept).
+   * Upgrade when any caller actually consumes these values; all current consumers
+   * tolerate zeros.
+   */
+  estimateSize(): Promise<StoreSize> {
+    return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
+  }
+
+  async close(): Promise<void> {
+    if (this.#closed) {
+      return;
+    }
+    this.#closed = true;
+    await this.#txQueue.end();
+    await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
+    this.#worker.terminate();
+    this.#rejectPending('SQLite store closed');
+  }
+
+  backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
+    throw new Error('Method not implemented.');
+  }
+
+  /**
+   * Returns a raw SQLite image (bytes suitable for writing as a `.sqlite` file and
+   * opening in any SQLite tool). Works only for non-ephemeral DBs because the OPFS
+   * SAH Pool has to be initialized. Useful for inspection/debugging.
+   */
+  async exportDb(): Promise<Uint8Array> {
+    const resp = await this.#sendRequest({ type: 'export', id: this.#allocId() });
+    if (!('bytes' in resp) || !resp.bytes) {
+      throw new Error('exportDb: worker returned no bytes');
+    }
+    return resp.bytes;
+  }
+
+  /**
+   * Runs a write statement (INSERT/UPDATE/DELETE/DDL). If called inside a
+   * `transactionAsync` block, bypasses the queue; otherwise acquires it so the
+   * op runs in its own auto-commit.
+   */
+  runAsync(sql: string, bind?: SqlValue[]): Promise<{ changes: number }> {
+    const send = () =>
+      this.#sendRequest({ type: 'run', id: this.#allocId(), sql, bind }).then(r => ({
+        changes: 'changes' in r ? (r.changes ?? 0) : 0,
+      }));
+    return this.#inTx ? send() : this.#txQueue.put(send);
+  }
+
+  /** Runs a SELECT statement and returns rows in array row-mode. */
+  allAsync(sql: string, bind?: SqlValue[]): Promise<ResultRow[]> {
+    const send = () =>
+      this.#sendRequest({ type: 'all', id: this.#allocId(), sql, bind }).then(r => ('rows' in r ? (r.rows ?? []) : []));
+    return this.#inTx ? send() : this.#txQueue.put(send);
+  }
+
+  #allocId(): number {
+    return ++this.#nextId;
+  }
+
+  /**
+   * Reject any in-flight requests with `reason`, so callers awaiting a response to a
+   * request sent to a now-terminated worker don't hang forever. Called from
+   * close()/delete() and from the worker.onerror handler.
+   */
+  #rejectPending(reason: string): void {
+    if (this.#pending.size === 0) {
+      return;
+    }
+    const err = new Error(reason);
+    for (const { reject } of this.#pending.values()) {
+      reject(err);
+    }
+    this.#pending.clear();
+  }
+
+  #sendRequest(req: WorkerRequest): Promise<WorkerResponse> {
+    return new Promise<WorkerResponse>((resolve, reject) => {
+      this.#pending.set(req.id, {
+        resolve: resp => {
+          if (resp.type === 'err') {
+            reject(new Error(resp.message));
+          } else {
+            resolve(resp);
+          }
+        },
+        reject,
+      });
+      this.#worker.postMessage(req);
+    });
+  }
+}