@aztec/kv-store 0.0.1-commit.f504929 → 0.0.1-commit.f5a9928

package/src/sqlite-opfs/messages.ts

@@ -0,0 +1,39 @@
+/**
+ * RPC protocol between the main thread and the SQLite worker.
+ * All requests carry a unique `id`; responses echo the same id so the main thread
+ * can resolve the right pending promise.
+ */
+import type { SqliteEncryptionErrorCode } from './errors.js';
+
+/** Matches `@sqlite.org/sqlite-wasm`'s internal SqlValue type. Boolean is not a native SQLite type. */
+export type SqlValue = string | number | bigint | null | Uint8Array;
+
+/** A row returned in 'array' rowMode — one value per column, in select order. */
+export type ResultRow = SqlValue[];
+
+export type WorkerRequest =
+  | { type: 'init'; id: number; dbName: string; ephemeral: boolean; poolDirectory?: string; encryptionKey?: Uint8Array }
+  | { type: 'close'; id: number }
+  | { type: 'deleteDb'; id: number; dbName: string }
+  | { type: 'run'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'all'; id: number; sql: string; bind?: SqlValue[] }
+  | { type: 'export'; id: number }
+  | { type: 'begin'; id: number }
+  | { type: 'commit'; id: number }
+  | { type: 'rollback'; id: number };
+
+export type WorkerResponse =
+  | { type: 'ok'; id: number; rows?: ResultRow[]; changes?: number; bytes?: Uint8Array }
+  | {
+      type: 'err';
+      id: number;
+      message: string;
+      /**
+       * Set when the worker detected an encryption-shaped failure. The main thread
+       * uses this to re-throw the error as a {@link SqliteEncryptionError} with the
+       * matching code. Absent on non-encryption failures (untyped paths preserved).
+       */
+      encryptionCode?: SqliteEncryptionErrorCode;
+    };
+
+export type WorkerRequestType = WorkerRequest['type'];

package/src/sqlite-opfs/multi_map.ts

@@ -0,0 +1,74 @@
+import { hash } from 'ohash';
+
+import type { Key, Value } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+
+/**
+ * Multi-map backed by SQLite. Extends the base map with always-incrementing
+ * `key_count` per-key so the same slot is never reused across deletions — this
+ * matches the IndexedDB backend's sparse-multi-map semantics.
+ */
+export class SQLiteOPFSAztecMultiMap<K extends Key, V extends Value>
+  extends SQLiteOPFSAztecMap<K, V>
+  implements AztecAsyncMultiMap<K, V>
+{
+  override async set(key: K, val: V): Promise<void> {
+    const valueHash = hash(val);
+    await this.store.transactionAsync(async () => {
+      const exists = await this.store.allAsync(
+        'SELECT 1 FROM data WHERE container = ? AND key = ? AND hash = ? LIMIT 1',
+        [this.container, this.encodedKey(key), valueHash],
+      );
+      if (exists.length > 0) {
+        return;
+      }
+      const maxRow = await this.store.allAsync('SELECT MAX(key_count) FROM data WHERE container = ? AND key = ?', [
+        this.container,
+        this.encodedKey(key),
+      ]);
+      const count = Number(maxRow[0]?.[0] ?? 0);
+      await this.store.runAsync(
+        `INSERT INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.slot(key, count), this.container, this.encodedKey(key), count + 1, valueHash, this.encoder.pack(val)],
+      );
+    });
+  }
+
+  async *getValuesAsync(key: K): AsyncIterableIterator<V> {
+    const rows = await this.store.allAsync(
+      'SELECT value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.container, this.encodedKey(key)],
+    );
+    for (const row of rows) {
+      const raw = row[0];
+      if (raw instanceof Uint8Array) {
+        yield this.decodeValue(raw);
+      }
+    }
+  }
+
+  async getValueCountAsync(key: K): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async deleteValue(key: K, val: V): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ? AND hash = ?', [
+      this.container,
+      this.encodedKey(key),
+      hash(val),
+    ]);
+  }
+
+  override async delete(key: K): Promise<void> {
+    await this.store.runAsync('DELETE FROM data WHERE container = ? AND key = ?', [
+      this.container,
+      this.encodedKey(key),
+    ]);
+  }
+}

package/src/sqlite-opfs/pool_integrity.ts

@@ -0,0 +1,327 @@
+import { normalizePoolDirectory } from './pool_lock.js';
+
+// The constants below mirror the opaque-file header format of the pinned opfs-sahpool VFS
+// (`yarn-project/sqlite3mc-wasm/vendor/jswasm/sqlite3.mjs`, class `OpfsSAHPool`). The pool names its files randomly
+// under `.opaque/` and prepends a header that maps each one back to its logical SQLite path:
+//
+//   bytes [0, 512)    logical path, NUL-terminated UTF-8 (HEADER_MAX_PATH_SIZE)
+//   bytes [512, 516)  SQLite open-flags of the file, big-endian uint32 (HEADER_FLAGS_SIZE)
+//   bytes [516, 524)  digest over the preceding 516 bytes, two uint32 words (HEADER_DIGEST_SIZE)
+//
+// Database content starts at byte 4096 (the pool's SECTOR_SIZE). We re-read the header ourselves rather than asking
+// the VFS because the upstream pool "repairs" anything it cannot validate by disassociating the file — destroying
+// exactly the evidence this module exists to quarantine. The browser regression test writes a real pool through the
+// pinned VFS before duplicating an opaque file, so a vendor upgrade that changes the layout breaks detection loudly
+// rather than silently.
+const OPAQUE_DIRECTORY = '.opaque';
+const HEADER_MAX_PATH_SIZE = 512;
+const HEADER_FLAGS_SIZE = 4;
+const HEADER_DIGEST_SIZE = 8;
+const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE;
+const HEADER_SIZE = HEADER_CORPUS_SIZE + HEADER_DIGEST_SIZE;
+
+// Standard SQLite open-flag bit values (sqlite3.h). Restated as literals because the header stores them numerically
+// and this module runs on the main thread, without the sqlite3 WASM bundle that defines `capi.SQLITE_OPEN_*`.
+const SQLITE_OPEN_DELETEONCLOSE = 0x00000008;
+const SQLITE_OPEN_MEMORY = 0x00000080;
+const SQLITE_OPEN_MAIN_DB = 0x00000100;
+const SQLITE_OPEN_MAIN_JOURNAL = 0x00000800;
+const SQLITE_OPEN_SUPER_JOURNAL = 0x00004000;
+const SQLITE_OPEN_WAL = 0x00080000;
+
+// A live association must name one of the file types the pool persists; transient types (temp DBs, statement
+// journals) never survive in a valid header.
+const PERSISTENT_FILE_TYPES =
+  SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_MAIN_JOURNAL | SQLITE_OPEN_SUPER_JOURNAL | SQLITE_OPEN_WAL;
+
+// The upstream VFS repurposes SQLITE_OPEN_MEMORY — meaningless for a file that exists on disk — as a header version
+// marker: headers written with it set carry a real digest, while legacy headers leave it unset and store all-zero
+// digest words.
+const FLAG_COMPUTE_DIGEST_V2 = SQLITE_OPEN_MEMORY;
+
+export const OPFS_QUARANTINE_ROOT_DIRECTORY = '.aztec-sqlite-quarantine';
+
+export interface DuplicatePoolAssociation {
+  logicalPath: string;
+  opaqueFileNames: string[];
+}
+
+export interface PoolQuarantineMetadata {
+  formatVersion: 1;
+  originalPoolDirectory: string;
+  quarantinedAt: string;
+  duplicateAssociations: DuplicatePoolAssociation[];
+}
+
+export interface PoolQuarantineResult extends PoolQuarantineMetadata {
+  quarantineDirectory: string;
+}
+
+/**
+ * Detects duplicate SAH logical-file associations and, if found, copies the complete pool into quarantine before
+ * removing the original. The caller must hold the pool's exclusive Web Lock for the whole operation.
+ */
+export async function quarantineDuplicatePool(poolDirectory: string): Promise<PoolQuarantineResult | undefined> {
+  poolDirectory = normalizePoolDirectory(poolDirectory);
+  const root = await navigator.storage.getDirectory();
+  const source = await getDirectory(root, poolDirectory);
+  if (!source) {
+    return undefined;
+  }
+  const opaque = await getChildDirectory(source, OPAQUE_DIRECTORY);
+  if (!opaque) {
+    return undefined;
+  }
+
+  const duplicateAssociations = await findDuplicateAssociations(opaque);
+  if (duplicateAssociations.length === 0) {
+    return undefined;
+  }
+
+  const quarantineRoot = await root.getDirectoryHandle(OPFS_QUARANTINE_ROOT_DIRECTORY, { create: true });
+  const quarantineName = createQuarantineName();
+  const destination = await quarantineRoot.getDirectoryHandle(quarantineName, { create: true });
+  const metadata: PoolQuarantineMetadata = {
+    formatVersion: 1,
+    originalPoolDirectory: poolDirectory,
+    quarantinedAt: new Date().toISOString(),
+    duplicateAssociations,
+  };
+
+  let quarantineComplete = false;
+  try {
+    await copyDirectory(source, destination);
+    await verifyDirectoryCopy(source, destination);
+    await writeJson(destination, 'quarantine.json', metadata);
+    quarantineComplete = true;
+    await removeDirectory(root, poolDirectory);
+  } catch (err) {
+    if (!quarantineComplete) {
+      await quarantineRoot.removeEntry(quarantineName, { recursive: true }).catch(() => {});
+    }
+    throw err;
+  }
+
+  return {
+    ...metadata,
+    quarantineDirectory: `${OPFS_QUARANTINE_ROOT_DIRECTORY}/${quarantineName}`,
+  };
+}
+
+async function findDuplicateAssociations(opaque: FileSystemDirectoryHandle): Promise<DuplicatePoolAssociation[]> {
+  const associations = new Map<string, string[]>();
+  for await (const [opaqueName, handle] of opaque.entries()) {
+    if (handle.kind !== 'file') {
+      continue;
+    }
+    const logicalPath = await readAssociatedPath(handle as FileSystemFileHandle);
+    if (logicalPath) {
+      const names = associations.get(logicalPath) ?? [];
+      names.push(opaqueName);
+      associations.set(logicalPath, names);
+    }
+  }
+  return [...associations.entries()]
+    .filter(([, names]) => names.length > 1)
+    .map(([logicalPath, opaqueFileNames]) => ({ logicalPath, opaqueFileNames: opaqueFileNames.sort() }))
+    .sort((a, b) => a.logicalPath.localeCompare(b.logicalPath));
+}
+
+/**
+ * Returns the logical SQLite path an opaque SAH file is associated with, or undefined if the file is not a live,
+ * valid association. Applies the same checks as the vendored pool's `getAssociatedPath`: a non-empty NUL-terminated
+ * path, open-flags naming a persistent file type without DELETEONCLOSE, and a matching header digest. Files failing
+ * any check are the pool's free-list or garbage entries — the VFS itself would disassociate them on open — so they
+ * cannot participate in a duplicate mapping.
+ */
+async function readAssociatedPath(handle: FileSystemFileHandle): Promise<string | undefined> {
+  const file = await handle.getFile();
+  if (file.size < HEADER_SIZE) {
+    return undefined;
+  }
+  const header = new Uint8Array(await file.slice(0, HEADER_SIZE).arrayBuffer());
+  const pathEnd = header.subarray(0, HEADER_MAX_PATH_SIZE).indexOf(0);
+  if (pathEnd <= 0) {
+    return undefined;
+  }
+
+  const dataView = new DataView(header.buffer, header.byteOffset, header.byteLength);
+  const flags = dataView.getUint32(HEADER_MAX_PATH_SIZE);
+  if ((flags & SQLITE_OPEN_DELETEONCLOSE) !== 0 || (flags & PERSISTENT_FILE_TYPES) === 0) {
+    return undefined;
+  }
+  if (!hasValidDigest(header, flags)) {
+    return undefined;
+  }
+
+  try {
+    return new TextDecoder('utf-8', { fatal: true }).decode(header.subarray(0, pathEnd));
+  } catch {
+    return undefined;
+  }
+}
+
+/**
+ * Byte-for-byte port of the vendored pool's `computeDigest`, checked against the digest words stored in the header.
+ */
+function hasValidDigest(header: Uint8Array, flags: number): boolean {
+  let expected0 = 0;
+  let expected1 = 0;
+  if ((flags & FLAG_COMPUTE_DIGEST_V2) !== 0) {
+    // These seeds (0xdeadbeef, 0x41c6ce57) and odd multipliers (2654435761, 104729) are the upstream author's choices
+    // (a cyrb53-hash variant) and carry no meaning here beyond having to match the vendored implementation bit for
+    // bit.
+    expected0 = 0xdeadbeef;
+    expected1 = 0x41c6ce57;
+    for (const value of header.subarray(0, HEADER_CORPUS_SIZE)) {
+      expected0 = Math.imul(expected0 ^ value, 2654435761);
+      expected1 = Math.imul(expected1 ^ value, 104729);
+    }
+    expected0 >>>= 0;
+    expected1 >>>= 0;
+  }
+  const dataView = new DataView(header.buffer, header.byteOffset, header.byteLength);
+  return (
+    dataView.getUint32(HEADER_CORPUS_SIZE, true) === expected0 &&
+    dataView.getUint32(HEADER_CORPUS_SIZE + 4, true) === expected1
+  );
+}
+
+async function getDirectory(
+  root: FileSystemDirectoryHandle,
+  path: string,
+): Promise<FileSystemDirectoryHandle | undefined> {
+  let current = root;
+  try {
+    for (const segment of path.split('/')) {
+      current = await current.getDirectoryHandle(segment);
+    }
+    return current;
+  } catch (err) {
+    if (err instanceof DOMException && err.name === 'NotFoundError') {
+      return undefined;
+    }
+    throw err;
+  }
+}
+
+async function getChildDirectory(
+  parent: FileSystemDirectoryHandle,
+  name: string,
+): Promise<FileSystemDirectoryHandle | undefined> {
+  try {
+    return await parent.getDirectoryHandle(name);
+  } catch (err) {
+    if (err instanceof DOMException && err.name === 'NotFoundError') {
+      return undefined;
+    }
+    throw err;
+  }
+}
+
+async function removeDirectory(root: FileSystemDirectoryHandle, path: string): Promise<void> {
+  const segments = path.split('/');
+  const name = segments.pop()!;
+  let parent = root;
+  for (const segment of segments) {
+    parent = await parent.getDirectoryHandle(segment);
+  }
+  await parent.removeEntry(name, { recursive: true });
+}
+
+async function copyDirectory(source: FileSystemDirectoryHandle, destination: FileSystemDirectoryHandle): Promise<void> {
+  for await (const [name, handle] of source.entries()) {
+    if (handle.kind === 'directory') {
+      const childDestination = await destination.getDirectoryHandle(name, { create: true });
+      await copyDirectory(handle as FileSystemDirectoryHandle, childDestination);
+      continue;
+    }
+
+    const sourceFile = await (handle as FileSystemFileHandle).getFile();
+    const destinationHandle = await destination.getFileHandle(name, { create: true });
+    const writable = await destinationHandle.createWritable();
+    try {
+      await writable.write(sourceFile);
+      await writable.close();
+    } catch (err) {
+      await writable.abort().catch(() => {});
+      throw err;
+    }
+  }
+}
+
+async function verifyDirectoryCopy(
+  source: FileSystemDirectoryHandle,
+  destination: FileSystemDirectoryHandle,
+): Promise<void> {
+  const sourceEntries = await getSortedEntries(source);
+  const destinationEntries = await getSortedEntries(destination);
+  if (
+    sourceEntries.length !== destinationEntries.length ||
+    sourceEntries.some(([name, handle], index) => {
+      const destinationEntry = destinationEntries[index];
+      return name !== destinationEntry[0] || handle.kind !== destinationEntry[1].kind;
+    })
+  ) {
+    throw new Error('Failed to verify quarantined OPFS directory structure');
+  }
+
+  for (let i = 0; i < sourceEntries.length; i++) {
+    const [, sourceHandle] = sourceEntries[i];
+    const [, destinationHandle] = destinationEntries[i];
+    if (sourceHandle.kind === 'directory') {
+      await verifyDirectoryCopy(
+        sourceHandle as FileSystemDirectoryHandle,
+        destinationHandle as FileSystemDirectoryHandle,
+      );
+    } else {
+      await verifyFilesEqual(sourceHandle as FileSystemFileHandle, destinationHandle as FileSystemFileHandle);
+    }
+  }
+}
+
+async function getSortedEntries(directory: FileSystemDirectoryHandle): Promise<[string, FileSystemHandle][]> {
+  const entries: [string, FileSystemHandle][] = [];
+  for await (const entry of directory.entries()) {
+    entries.push(entry);
+  }
+  return entries.sort(([a], [b]) => a.localeCompare(b));
+}
+
+async function verifyFilesEqual(sourceHandle: FileSystemFileHandle, destinationHandle: FileSystemFileHandle) {
+  const source = await sourceHandle.getFile();
+  const destination = await destinationHandle.getFile();
+  if (source.size !== destination.size) {
+    throw new Error(`Failed to verify quarantined OPFS file "${source.name}"`);
+  }
+
+  const chunkSize = 1024 * 1024;
+  for (let offset = 0; offset < source.size; offset += chunkSize) {
+    const [sourceChunk, destinationChunk] = await Promise.all([
+      source.slice(offset, offset + chunkSize).arrayBuffer(),
+      destination.slice(offset, offset + chunkSize).arrayBuffer(),
+    ]);
+    const sourceBytes = new Uint8Array(sourceChunk);
+    const destinationBytes = new Uint8Array(destinationChunk);
+    if (sourceBytes.some((value, index) => value !== destinationBytes[index])) {
+      throw new Error(`Failed to verify quarantined OPFS file "${source.name}"`);
+    }
+  }
+}
+
+async function writeJson(directory: FileSystemDirectoryHandle, name: string, value: unknown): Promise<void> {
+  const handle = await directory.getFileHandle(name, { create: true });
+  const writable = await handle.createWritable();
+  try {
+    await writable.write(JSON.stringify(value, undefined, 2));
+    await writable.close();
+  } catch (err) {
+    await writable.abort().catch(() => {});
+    throw err;
+  }
+}
+
+function createQuarantineName(): string {
+  const random = globalThis.crypto.getRandomValues(new Uint8Array(8));
+  return `${Date.now()}-${[...random].map(byte => byte.toString(16).padStart(2, '0')).join('')}`;
+}

package/src/sqlite-opfs/pool_lock.ts

@@ -0,0 +1,62 @@
+import { SqlitePoolBusyError, SqliteWebLocksUnavailableError } from './errors.js';
+
+export const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
+
+const WEB_LOCK_PREFIX = 'aztec.sqlite-opfs.pool:';
+
+export interface PoolLockLease {
+  release(): Promise<void>;
+}
+
+export function normalizePoolDirectory(poolDirectory?: string): string {
+  const path = poolDirectory
+    ?.split('/')
+    .filter(segment => segment.length > 0)
+    .join('/');
+  return path || DEFAULT_SAH_POOL_DIRECTORY;
+}
+
+export async function acquirePoolLock(poolDirectory?: string): Promise<PoolLockLease> {
+  if (!navigator.locks) {
+    throw new SqliteWebLocksUnavailableError();
+  }
+  const normalizedDirectory = normalizePoolDirectory(poolDirectory);
+  const acquired = Promise.withResolvers<Lock | null>();
+  const hold = Promise.withResolvers<void>();
+  const request = navigator.locks.request(
+    `${WEB_LOCK_PREFIX}${normalizedDirectory}`,
+    { mode: 'exclusive', ifAvailable: true },
+    async lock => {
+      acquired.resolve(lock);
+      if (lock) {
+        await hold.promise;
+      }
+    },
+  );
+  request.catch(acquired.reject);
+
+  if (!(await acquired.promise)) {
+    await request;
+    throw new SqlitePoolBusyError(normalizedDirectory);
+  }
+
+  let released = false;
+  return {
+    release: async () => {
+      if (!released) {
+        released = true;
+        hold.resolve();
+      }
+      await request;
+    },
+  };
+}
+
+export async function withPoolLock<T>(poolDirectory: string, callback: () => Promise<T>): Promise<T> {
+  const lease = await acquirePoolLock(poolDirectory);
+  try {
+    return await callback();
+  } finally {
+    await lease.release();
+  }
+}

package/src/sqlite-opfs/set.ts

@@ -0,0 +1,29 @@
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import { SQLiteOPFSAztecMap } from './map.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Set backed by SQLite. Composes a Map<K, true>. */
+export class SQLiteOPFSAztecSet<K extends Key> implements AztecAsyncSet<K> {
+  readonly #map: SQLiteOPFSAztecMap<K, boolean>;
+
+  constructor(store: AztecSQLiteOPFSStore, name: string) {
+    this.#map = new SQLiteOPFSAztecMap<K, boolean>(store, name);
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.#map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.#map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.#map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    yield* this.#map.keysAsync(range);
+  }
+}

package/src/sqlite-opfs/singleton.ts

@@ -0,0 +1,48 @@
+import { Encoder } from '#msgpackr';
+import { toBufferKey } from '#ordered-binary';
+import { hash } from 'ohash';
+
+import type { Value } from '../interfaces/common.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/** Stores a single value identified by `name`. */
+export class SQLiteOPFSAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
+  readonly #container: string;
+  readonly #slot: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#container = `singleton:${name}`;
+    this.#slot = `singleton:${name}:value`;
+  }
+
+  async getAsync(): Promise<T | undefined> {
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot]);
+    if (rows.length === 0) {
+      return undefined;
+    }
+    const raw = rows[0][0];
+    if (raw instanceof Uint8Array) {
+      return this.#encoder.unpack(raw) as T;
+    }
+    return undefined;
+  }
+
+  async set(val: T): Promise<boolean> {
+    const { changes } = await this.store.runAsync(
+      `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+       VALUES (?, ?, ?, ?, ?, ?)`,
+      [this.#slot, this.#container, toBufferKey([this.#slot]), 1, hash(val), this.#encoder.pack(val)],
+    );
+    return changes > 0;
+  }
+
+  async delete(): Promise<boolean> {
+    await this.store.runAsync('DELETE FROM data WHERE slot = ?', [this.#slot]);
+    return true;
+  }
+}