@aztec/kv-store 0.0.1-commit.8c0b8ff → 0.0.1-commit.8cb2d04d8

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { toBufferKey } from 'ordered-binary';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,27 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  const { dataDirectory } = config;
+  log.info(
+    dataDirectory
+      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts?.rollupAddress, log);
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}

package/src/sqlite-opfs/map.ts

@@ -0,0 +1,163 @@
+import { Buffer } from 'buffer';
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { fromBufferKey, toBufferKey } from 'ordered-binary';
+
+import type { Key, Range, Value } from '../interfaces/common.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { SqlValue } from './messages.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** A map backed by SQLite in OPFS. Mirrors `IndexedDBAztecMap`. */
+export class SQLiteOPFSAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
+  protected readonly name: string;
+  protected readonly container: string;
+  protected readonly encoder = new Encoder();
+
+  constructor(
+    protected readonly store: AztecSQLiteOPFSStore,
+    mapName: string,
+  ) {
+    this.name = mapName;
+    this.container = `map:${mapName}`;
+  }
+
+  async getAsync(key: K): Promise<V | undefined> {
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.slot(key)]);
+    if (rows.length === 0) {
+      return undefined;
+    }
+    const raw = rows[0][0];
+    return raw == null ? undefined : this.decodeValue(raw);
+  }
+
+  async hasAsync(key: K): Promise<boolean> {
+    const rows = await this.store.allAsync('SELECT 1 FROM data WHERE slot = ? LIMIT 1', [this.slot(key)]);
+    return rows.length > 0;
+  }
+
+  async sizeAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ?', [this.container]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async set(key: K, val: V): Promise<void> {
+    await this.store.runAsync(
+      `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [this.slot(key), this.container, this.encodedKey(key), 1, hash(val), this.encoder.pack(val)],
+    );
+  }
+
+  async setMany(entries: { key: K; value: V }[]): Promise<void> {
+    if (entries.length === 0) {
+      return;
+    }
+    await this.store.transactionAsync(async () => {
+      for (const { key, value } of entries) {
+        await this.set(key, value);
+      }
+    });
+  }
+
+  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
+    throw new Error('Not implemented');
+  }
+
+  async setIfNotExists(key: K, val: V): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      if (await this.hasAsync(key)) {
+        return false;
+      }
+      await this.set(key, val);
+      return true;
+    });
+  }
+
+  async delete(key: K): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE slot = ?', [this.slot(key)]);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
+    const rows = await this.rangeQuery(range);
+    for (const row of rows) {
+      const [_slot, keyBlob, value] = row;
+      if (keyBlob == null || value == null) {
+        continue;
+      }
+      yield [this.decodeKey(keyBlob), this.decodeValue(value)];
+    }
+  }
+
+  async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
+    for await (const [, value] of this.entriesAsync(range)) {
+      yield value;
+    }
+  }
+
+  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const [key] of this.entriesAsync(range)) {
+      yield key;
+    }
+  }
+
+  protected async rangeQuery(range: Range<K>): Promise<Array<[string, Uint8Array, Uint8Array | null]>> {
+    // Inclusivity flips with direction to match the IndexedDB backend:
+    //   forward: [start, end)     reverse: (start, end]
+    // That asymmetry is load-bearing — tests pin the exact inclusivity at boundaries.
+    const reverse = !!range.reverse;
+    const parts: string[] = ['container = ?'];
+    const bind: SqlValue[] = [this.container];
+    if (range.start !== undefined) {
+      parts.push(reverse ? 'key > ?' : 'key >= ?');
+      bind.push(this.encodedKey(range.start));
+    }
+    if (range.end !== undefined) {
+      parts.push(reverse ? 'key <= ?' : 'key < ?');
+      bind.push(this.encodedKey(range.end));
+    }
+    const order = reverse ? 'DESC' : 'ASC';
+    let sql = `SELECT slot, key, value FROM data WHERE ${parts.join(' AND ')} ORDER BY key ${order}, key_count ${order}`;
+    if (range.limit !== undefined) {
+      sql += ' LIMIT ?';
+      bind.push(range.limit);
+    }
+    const rows = await this.store.allAsync(sql, bind);
+    return rows.map(r => [String(r[0]), r[1] as Uint8Array, r[2] as Uint8Array | null]);
+  }
+
+  protected decodeValue(val: SqlValue): V {
+    if (!(val instanceof Uint8Array)) {
+      return val as V;
+    }
+    const unpacked = this.encoder.unpack(val);
+    // msgpackr returns plain Uint8Array in browsers for packed Buffers. Callers that
+    // stored Buffers (walletDB uses Buffer.from(...).toString('utf8') round-trips)
+    // rely on Buffer-flavored behavior — re-wrap at the storage boundary, mirroring
+    // IndexedDBAztecMap.restoreBuffers.
+    if (unpacked instanceof Uint8Array && !Buffer.isBuffer(unpacked)) {
+      return Buffer.from(unpacked) as V;
+    }
+    return unpacked as V;
+  }
+
+  protected decodeKey(raw: Uint8Array): K {
+    const parsed = fromBufferKey(Buffer.from(raw));
+    if (Array.isArray(parsed)) {
+      return (parsed.length > 1 ? parsed : parsed[0]) as K;
+    }
+    return parsed as K;
+  }
+
+  protected encodedKey(key: K): Buffer {
+    return toBufferKey(this.normalizeKey(key));
+  }
+
+  protected normalizeKey(key: K): (string | number | Uint8Array)[] {
+    return Array.isArray(key) ? key : [key];
+  }
+
+  protected slot(key: K, index: number = 0): string {
+    return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
+  }
+}

package/src/sqlite-opfs/messages.ts

@@ -0,0 +1,28 @@
+/**
+ * RPC protocol between the main thread and the SQLite worker.
+ * All requests carry a unique `id`; responses echo the same id so the main thread
+ * can resolve the right pending promise.
+ */
+
+/** Matches `@sqlite.org/sqlite-wasm`'s internal SqlValue type. Boolean is not a native SQLite type. */
+export type SqlValue = string | number | bigint | null | Uint8Array;
+
+/** A row returned in 'array' rowMode — one value per column, in select order. */
+export type ResultRow = SqlValue[];
+
+export type WorkerRequest =
+  | { type: 'init'; id: number; dbName: string; ephemeral: boolean; poolDirectory?: string }
+  | { type: 'close'; id: number }
+  | { type: 'deleteDb'; id: number; dbName: string }
+  | { type: 'run'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'all'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'export'; id: number }
+  | { type: 'begin'; id: number }
+  | { type: 'commit'; id: number }
+  | { type: 'rollback'; id: number };
+
+export type WorkerResponse =
+  | { type: 'ok'; id: number; rows?: ResultRow[]; changes?: number; bytes?: Uint8Array }
+  | { type: 'err'; id: number; message: string };
+
+export type WorkerRequestType = WorkerRequest['type'];

package/src/sqlite-opfs/multi_map.ts

@@ -0,0 +1,74 @@
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+
+/**
+ * Multi-map backed by SQLite. Extends the base map with always-incrementing
+ * `key_count` per-key so the same slot is never reused across deletions — this
+ * matches the IndexedDB backend's sparse-multi-map semantics.
+ */
+export class SQLiteOPFSAztecMultiMap<K extends Key, V extends Value>
+  extends SQLiteOPFSAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    const valueHash = hash(val);
+    await this.store.transactionAsync(async () => {
+      const exists = await this.store.allAsync(
+        'SELECT 1 FROM data WHERE container = ? AND key = ? AND hash = ? LIMIT 1',
+        [this.container, this.encodedKey(key), valueHash],
+      );
+      if (exists.length > 0) {
+        return;
+      }
+      const maxRow = await this.store.allAsync('SELECT MAX(key_count) FROM data WHERE container = ? AND key = ?', [
+        this.container,
+        this.encodedKey(key),
+      ]);
+      const count = Number(maxRow[0]?.[0] ?? 0);
+      await this.store.runAsync(
+        `INSERT INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.slot(key, count), this.container, this.encodedKey(key), count + 1, valueHash, this.encoder.pack(val)],
+      );
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const rows = await this.store.allAsync(
+      'SELECT value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.container, this.encodedKey(key)],
+    );
+    for (const row of rows) {
+      const raw = row[0];
+      if (raw instanceof Uint8Array) {
+        yield this.decodeValue(raw);
+      }
+    }
+  }
+
+  async getValueCountAsync(key: K): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ? AND hash = ?', [
+      this.container,
+      this.encodedKey(key),
+      hash(val),
+    ]);
+  }
+
+  override async delete(key: K): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+  }
+}

package/src/sqlite-opfs/set.ts

@@ -0,0 +1,29 @@
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Set backed by SQLite. Composes a Map<K, true>. */
+export class SQLiteOPFSAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  readonly #map: SQLiteOPFSAztecMap<K, boolean>;
+
+  constructor(store: AztecSQLiteOPFSStore, name: string) {
+    this.#map = new SQLiteOPFSAztecMap<K, boolean>(store, name);
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.#map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.#map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.#map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.#map.keysAsync(range);
+  }
+}

package/src/sqlite-opfs/singleton.ts

@@ -0,0 +1,48 @@
+import { Encoder } from 'msgpackr';
+import { hash } from 'ohash';
+import { toBufferKey } from 'ordered-binary';
+
+import type { Value } from '../interfaces/common.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Stores a single value identified by `name`. */
+export class SQLiteOPFSAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
+  readonly #container: string;
+  readonly #slot: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot]);
+    if (rows.length === 0) {
+      return undefined;
+    }
+    const raw = rows[0][0];
+    if (raw instanceof Uint8Array) {
+      return this.#encoder.unpack(raw) as T;
+    }
+    return undefined;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const { changes } = await this.store.runAsync(
+      `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [this.#slot, this.#container, toBufferKey([this.#slot]), 1, hash(val), this.#encoder.pack(val)],
+    );
+    return changes > 0;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.store.runAsync('DELETE FROM data WHERE slot = ?', [this.#slot]);
+    return true;
+  }
+}