@aztec-labs/kv-store 6.0.0-nightly.20260829

package/src/lmdb-v2/write_transaction.ts

@@ -0,0 +1,315 @@
+import {
+  dedupeSortedArray,
+  findInSortedArray,
+  findIndexInSortedArray,
+  insertIntoSortedArray,
+  merge,
+  removeAnyOf,
+  removeFromSortedArray,
+} from '@aztec-labs/foundation/array';
+
+import { type Batch, Database, LMDBMessageType } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+import { keyCmp, singleKeyCmp } from './utils.js';
+
+export class WriteTransaction extends ReadTransaction {
+  // exposed for tests
+  public readonly dataBatch: Batch = {
+    addEntries: [],
+    removeEntries: [],
+  };
+  public readonly indexBatch: Batch = {
+    addEntries: [],
+    removeEntries: [],
+  };
+
+  set(key: Uint8Array, value: Uint8Array): Promise<void> {
+    this.assertIsOpen();
+
+    const addEntry = findInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+    if (!addEntry) {
+      insertIntoSortedArray(this.dataBatch.addEntries, [key, [value]], keyCmp);
+    } else {
+      addEntry[1] = [value];
+    }
+
+    const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+    if (removeEntryIndex > -1) {
+      this.dataBatch.removeEntries.splice(removeEntryIndex, 1);
+    }
+
+    return Promise.resolve();
+  }
+
+  remove(key: Uint8Array): Promise<void> {
+    const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+    if (removeEntryIndex === -1) {
+      insertIntoSortedArray(this.dataBatch.removeEntries, [key, null], keyCmp);
+    }
+
+    const addEntryIndex = findIndexInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+    if (addEntryIndex > -1) {
+      this.dataBatch.addEntries.splice(addEntryIndex, 1);
+    }
+
+    return Promise.resolve();
+  }
+
+  public override async get(key: Buffer): Promise<Uint8Array | undefined> {
+    this.assertIsOpen();
+
+    const addEntry = findInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
+    if (addEntry) {
+      return addEntry[1][0];
+    }
+    const removeEntryIdx = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
+    if (removeEntryIdx > -1) {
+      return undefined;
+    }
+
+    return await super.get(key);
+  }
+
+  setIndex(key: Buffer, ...values: Buffer[]): Promise<void> {
+    this.assertIsOpen();
+
+    const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+    const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+
+    if (removeEntries) {
+      if (removeEntries[1]) {
+        // check if we were deleting these values and update
+        removeAnyOf(removeEntries[1], values, Buffer.compare);
+      }
+
+      if (!removeEntries[1] || removeEntries[1].length === 0) {
+        // either we were deleting the entire key previously
+        // or after cleaning up duplicates, we don't have anything else to delete
+        removeFromSortedArray(this.indexBatch.removeEntries, removeEntries, keyCmp);
+      }
+    }
+
+    if (addEntries) {
+      merge(addEntries[1], values, Buffer.compare);
+      dedupeSortedArray(addEntries[1], Buffer.compare);
+    } else {
+      insertIntoSortedArray(this.indexBatch.addEntries, [key, values], keyCmp);
+    }
+
+    return Promise.resolve();
+  }
+
+  removeIndex(key: Buffer, ...values: Buffer[]): Promise<void> {
+    this.assertIsOpen();
+
+    const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+    const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+
+    if (values.length === 0) {
+      // special case, we're deleting the entire key
+      if (addEntries) {
+        removeFromSortedArray(this.indexBatch.addEntries, addEntries, keyCmp);
+      }
+
+      if (removeEntries) {
+        removeEntries[1] = null;
+      } else {
+        insertIntoSortedArray(this.indexBatch.removeEntries, [key, null], keyCmp);
+      }
+
+      return Promise.resolve();
+    }
+
+    if (addEntries) {
+      removeAnyOf(addEntries[1], values, Buffer.compare);
+      if (addEntries[1].length === 0) {
+        removeFromSortedArray(this.indexBatch.addEntries, addEntries, keyCmp);
+      }
+    }
+
+    if (removeEntries) {
+      removeEntries[1] ??= [];
+      merge(removeEntries[1], values, Buffer.compare);
+      dedupeSortedArray(removeEntries[1], Buffer.compare);
+    } else {
+      insertIntoSortedArray(this.indexBatch.removeEntries, [key, values], keyCmp);
+    }
+
+    return Promise.resolve();
+  }
+
+  public override async getIndex(key: Buffer): Promise<Uint8Array[]> {
+    this.assertIsOpen();
+
+    const removeEntries = findInSortedArray(this.indexBatch.removeEntries, key, singleKeyCmp);
+    if (removeEntries && removeEntries[1] === null) {
+      return [];
+    }
+
+    const addEntries = findInSortedArray(this.indexBatch.addEntries, key, singleKeyCmp);
+    const results = await super.getIndex(key);
+
+    if (addEntries) {
+      merge(results, addEntries[1], Buffer.compare);
+      dedupeSortedArray(results, Buffer.compare);
+    }
+
+    if (removeEntries && Array.isArray(removeEntries[1])) {
+      removeAnyOf(results, removeEntries[1], Buffer.compare);
+    }
+
+    return results;
+  }
+
+  public override async *iterate(
+    startKey: Uint8Array,
+    endKey?: Uint8Array,
+    reverse?: boolean,
+    limit?: number,
+  ): AsyncIterable<[Uint8Array, Uint8Array]> {
+    yield* this.#iterate(
+      super.iterate(startKey, endKey, reverse),
+      this.dataBatch,
+      startKey,
+      endKey,
+      reverse,
+      limit,
+      (committed, toAdd) => (toAdd.length > 0 ? toAdd[0] : committed),
+      vals => vals[0],
+    );
+  }
+
+  public override async *iterateIndex(
+    startKey: Uint8Array,
+    endKey?: Uint8Array,
+    reverse?: boolean,
+    limit?: number,
+  ): AsyncIterable<[Uint8Array, Uint8Array[]]> {
+    yield* this.#iterate(
+      super.iterateIndex(startKey, endKey, reverse),
+      this.indexBatch,
+      startKey,
+      endKey,
+      reverse,
+      limit,
+      (committed, toAdd, toRemove) => {
+        if (toAdd.length > 0) {
+          merge(committed, toAdd, Buffer.compare);
+          dedupeSortedArray(committed, Buffer.compare);
+        }
+        if (toRemove.length > 0) {
+          removeAnyOf(committed, toRemove, Buffer.compare);
+        }
+        return committed;
+      },
+      vals => vals,
+    );
+  }
+
+  async *#iterate<T>(
+    iterator: AsyncIterable<[Uint8Array, T]>,
+    batch: Batch,
+    startKey: Uint8Array,
+    endKey: Uint8Array | undefined,
+    reverse: boolean = false,
+    limit: number | undefined,
+    merge: (committed: T, toAdd: Uint8Array[], toRemove: Uint8Array[]) => T,
+    map: (vals: Uint8Array[]) => T,
+  ): AsyncIterable<[Uint8Array, T]> {
+    this.assertIsOpen();
+
+    // Snapshot both add and remove entries at the start of iteration to ensure consistency
+    const uncommittedEntries = [...batch.addEntries];
+    const removeEntries = [...batch.removeEntries];
+    // used to check we're in the right order when comparing between a key and uncommittedEntries
+    let cmpDirection = -1;
+    if (reverse) {
+      cmpDirection = 1;
+      uncommittedEntries.reverse();
+    }
+
+    let uncommittedEntriesIdx = 0;
+    while (uncommittedEntriesIdx < uncommittedEntries.length) {
+      const entry = uncommittedEntries[uncommittedEntriesIdx];
+      // go to the first key in our cache that would be captured by the iterator
+      if (Buffer.compare(entry[0], startKey) !== cmpDirection) {
+        break;
+      }
+      uncommittedEntriesIdx++;
+    }
+
+    let count = 0;
+    // helper to early return if we've reached our limit
+    const checkLimit = typeof limit === 'number' ? () => count < limit : () => true;
+    for await (const [key, values] of iterator) {
+      // yield every key that we have cached that's captured by the iterator
+      while (uncommittedEntriesIdx < uncommittedEntries.length && checkLimit()) {
+        const entry = uncommittedEntries[uncommittedEntriesIdx];
+        if (endKey && Buffer.compare(entry[0], endKey) !== cmpDirection) {
+          break;
+        }
+
+        if (Buffer.compare(entry[0], key) === cmpDirection) {
+          count++;
+          yield [entry[0], map(entry[1])];
+        } else {
+          break;
+        }
+        uncommittedEntriesIdx++;
+      }
+
+      if (!checkLimit()) {
+        // we reached the imposed `limit`
+        break;
+      }
+
+      const toRemove = findInSortedArray(removeEntries, key, singleKeyCmp);
+
+      // at this point we've either exhausted all uncommitted entries,
+      // we reached a key strictly greater/smaller than `key`
+      // or we found the key itself
+      // check if it's the key and use the uncommitted value
+      let toAdd: Uint8Array[] = [];
+      if (
+        uncommittedEntriesIdx < uncommittedEntries.length &&
+        Buffer.compare(uncommittedEntries[uncommittedEntriesIdx][0], key) === 0
+      ) {
+        toAdd = uncommittedEntries[uncommittedEntriesIdx][1];
+        uncommittedEntriesIdx++;
+      }
+
+      if (toRemove && !toRemove[1]) {
+        // we were told to delete this key entirely
+        continue;
+      } else {
+        const mergedValues = merge(values, toAdd, toRemove?.[1] ?? []);
+        if (mergedValues) {
+          count++;
+          yield [key, mergedValues];
+        }
+      }
+    }
+
+    // emit all the uncommitted data that would be captured by this iterator
+    while (uncommittedEntriesIdx < uncommittedEntries.length && checkLimit()) {
+      const entry = uncommittedEntries[uncommittedEntriesIdx];
+      if (endKey && Buffer.compare(entry[0], endKey) !== cmpDirection) {
+        break;
+      }
+      count++;
+      yield [entry[0], map(entry[1])];
+      uncommittedEntriesIdx++;
+    }
+  }
+
+  public async commit() {
+    this.assertIsOpen();
+    this.close();
+    await this.channel.sendMessage(LMDBMessageType.BATCH, {
+      batches: new Map([
+        [Database.DATA, this.dataBatch],
+        [Database.INDEX, this.indexBatch],
+      ]),
+    });
+  }
+}

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from '#msgpackr';
+import { toBufferKey } from '#ordered-binary';
+import { hash } from 'ohash';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/errors.ts

@@ -0,0 +1,94 @@
+/**
+ * Typed error surface for sqlite3mc-backed page-level encryption failures.
+ *
+ * Three concrete failure modes are surfaced:
+ *
+ *   - `invalid_key_length`: caller-side pre-flight (key not 32 bytes).
+ *   - `encryption_not_supported_for_ephemeral`: caller-side pre-flight (encryption was requested on an ephemeral
+ *     `:memory:` store, which sqlite3mc does not support).
+ *   - `decrypt_failed`: runtime failure raised when sqlite3mc cannot decode page 1 of an existing database. Covers
+ *     both "wrong key supplied" and "no key supplied to an encrypted DB".
+ */
+export type SqliteEncryptionErrorCode =
+  | 'invalid_key_length'
+  | 'encryption_not_supported_for_ephemeral'
+  | 'decrypt_failed';
+
+/**
+ * Error thrown by sqlite-opfs when an encryption operation fails.
+ **/
+export class SqliteEncryptionError extends Error {
+  readonly code: SqliteEncryptionErrorCode;
+
+  constructor(code: SqliteEncryptionErrorCode, message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteEncryptionError';
+    this.code = code;
+  }
+}
+
+/**
+ * Strings raised by sqlite3mc when page 1 cannot be decoded.
+ **/
+const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
+  /file is not a database/i,
+  /file is encrypted or is not a database/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known sqlite3mc decrypt-failure
+ * strings.
+ **/
+export function isDecryptFailureMessage(message: string): boolean {
+  return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
+}
+
+/**
+ * Error thrown by sqlite-opfs when the on-disk database image is corrupt
+ * (`SQLITE_CORRUPT`, result code 11 — "database disk image is malformed").
+ *
+ * Distinct from {@link SqliteEncryptionError}: no key recovers a corrupt image,
+ * so there is nothing to retry. The only escape is to delete the store and start
+ * fresh, so consumers pattern-match on `instanceof SqliteCorruptionError` to wipe
+ * and reopen rather than surfacing a dead-end error.
+ **/
+export class SqliteCorruptionError extends Error {
+  constructor(message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteCorruptionError';
+  }
+}
+
+/** Error thrown when another browser context already owns a store's OPFS pool. */
+export class SqlitePoolBusyError extends Error {
+  constructor(public readonly poolDirectory: string) {
+    super(`SQLite-OPFS pool "${poolDirectory}" is already in use by another store instance`);
+    this.name = 'SqlitePoolBusyError';
+  }
+}
+
+/** Error thrown when the browser context does not expose the Web Locks API required by the OPFS SAH pool. */
+export class SqliteWebLocksUnavailableError extends Error {
+  constructor() {
+    super('SQLite-OPFS requires the Web Locks API, but it is unavailable in this browser context');
+    this.name = 'SqliteWebLocksUnavailableError';
+  }
+}
+
+/**
+ * Strings/codes raised by SQLite when a database image is corrupt. Kept disjoint
+ * from {@link SQLITE3MC_DECRYPT_ERROR_PATTERNS}: "file is not a database"
+ * (SQLITE_NOTADB) is the decrypt signal, whereas a malformed image is the
+ * genuinely-unrecoverable SQLITE_CORRUPT.
+ **/
+const SQLITE_CORRUPTION_ERROR_PATTERNS: readonly RegExp[] = [
+  /database disk image is malformed/i,
+  /\bSQLITE_CORRUPT\b/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known SQLite corruption strings.
+ **/
+export function isCorruptionMessage(message: string): boolean {
+  return SQLITE_CORRUPTION_ERROR_PATTERNS.some(p => p.test(message));
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,27 @@
+import { createLogger } from '@aztec-labs/foundation/log';
+
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+export {
+  SqliteCorruptionError,
+  SqliteEncryptionError,
+  SqlitePoolBusyError,
+  SqliteWebLocksUnavailableError,
+} from './errors.js';
+export type { SqliteEncryptionErrorCode } from './errors.js';
+export { OPFS_POOL_DIR_PREFIX, deletePoolDirectory, deleteStore, listStores, storePoolDirectory } from './manage.js';
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}