@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.017a351

package/src/lmdb-v2/store.ts

@@ -1,19 +1,20 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
 import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
 
 import { AsyncLocalStorage } from 'async_hooks';
-import { rm } from 'fs/promises';
+import { mkdir, rm } from 'fs/promises';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Key, StoreSize } from '../interfaces/common.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
 import type { AztecAsyncCounter } from '../interfaces/counter.js';
-import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
 import type { AztecAsyncSet } from '../interfaces/set.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
 import { LMDBArray } from './array.js';
-import { LMDBMap, LMDBMultiMap } from './map.js';
+import { LMDBMap } from './map.js';
 import {
   Database,
   type LMDBMessageChannel,
@@ -21,10 +22,14 @@ import {
   type LMDBRequestBody,
   type LMDBResponseBody,
 } from './message.js';
+import { LMDBMultiMap } from './multi_map.js';
 import { ReadTransaction } from './read_transaction.js';
+import { LMDBSet } from './set.js';
 import { LMDBSingleValue } from './singleton.js';
 import { WriteTransaction } from './write_transaction.js';
 
+export { execInReadTx, execInWriteTx } from './tx-helpers.js';
+
 export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
   private open = false;
   private channel: MsgpackChannel<LMDBMessageType, LMDBRequestBody, LMDBResponseBody>;
@@ -38,9 +43,10 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     maxReaders: number,
     private log: Logger,
     private cleanup?: () => Promise<void>,
+    ephemeral: boolean = false,
   ) {
     this.log.info(`Starting data store with maxReaders ${maxReaders}`);
-    this.channel = new MsgpackChannel(new NativeLMDBStore(dataDir, mapSize, maxReaders));
+    this.channel = new MsgpackChannel(new NativeLMDBStore(dataDir, mapSize, maxReaders, ephemeral));
     // leave one reader to always be available for regular, atomic, reads
     this.availableCursors = new Semaphore(maxReaders - 1);
   }
@@ -70,13 +76,20 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     dbMapSizeKb: number = 10 * 1024 * 1024,
     maxReaders: number = 16,
     cleanup?: () => Promise<void>,
-    log = createLogger('kv-store:lmdb-v2'),
+    bindings?: LoggerBindings,
+    ephemeral: boolean = false,
   ) {
-    const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup);
+    const log = createLogger('kv-store:lmdb-v2', bindings);
+    const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup, ephemeral);
     await db.start();
     return db;
   }
 
+  public async backupTo(dstPath: string, compact = true) {
+    await mkdir(dstPath, { recursive: true });
+    await this.channel.sendMessage(LMDBMessageType.COPY_STORE, { dstPath, compact });
+  }
+
   public getReadTx(): ReadTransaction {
     if (!this.open) {
       throw new Error('Store is closed');
@@ -92,24 +105,24 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return currentWrite;
   }
 
-  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
     return new LMDBMap(this, name);
   }
 
-  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
     return new LMDBMultiMap(this, name);
   }
 
-  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
     return new LMDBSingleValue(this, name);
   }
 
-  openArray<T>(name: string): AztecAsyncArray<T> {
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
     return new LMDBArray(this, name);
   }
 
-  openSet<K extends Key>(_name: string): AztecAsyncSet<K> {
-    throw new Error('Not implemented');
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    return new LMDBSet(this, name);
   }
 
   openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
@@ -150,10 +163,6 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return Promise.resolve();
   }
 
-  fork(): Promise<AztecAsyncKVStore> {
-    throw new Error('Not implemented');
-  }
-
   async delete(): Promise<void> {
     await this.close();
     await rm(this.dataDir, { recursive: true, force: true, maxRetries: 3 });
@@ -204,34 +213,9 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     const resp = await this.sendMessage(LMDBMessageType.STATS, undefined);
     return {
       mappingSize: Number(resp.dbMapSizeBytes),
+      physicalFileSize: Number(resp.dbPhysicalFileSizeBytes),
       actualSize: resp.stats.reduce((s, db) => Number(db.totalUsedSize) + s, 0),
       numItems: resp.stats.reduce((s, db) => Number(db.numDataItems) + s, 0),
     };
   }
 }
-
-export function execInWriteTx<T>(store: AztecLMDBStoreV2, fn: (tx: WriteTransaction) => Promise<T>): Promise<T> {
-  const currentWrite = store.getCurrentWriteTx();
-  if (currentWrite) {
-    return fn(currentWrite);
-  } else {
-    return store.transactionAsync(fn);
-  }
-}
-
-export async function execInReadTx<T>(
-  store: AztecLMDBStoreV2,
-  fn: (tx: ReadTransaction) => T | Promise<T>,
-): Promise<T> {
-  const currentWrite = store.getCurrentWriteTx();
-  if (currentWrite) {
-    return await fn(currentWrite);
-  } else {
-    const tx = store.getReadTx();
-    try {
-      return await fn(tx);
-    } finally {
-      tx.close();
-    }
-  }
-}

package/src/lmdb-v2/tx-helpers.ts

@@ -0,0 +1,29 @@
+import type { ReadTransaction } from './read_transaction.js';
+import type { AztecLMDBStoreV2 } from './store.js';
+import type { WriteTransaction } from './write_transaction.js';
+
+export function execInWriteTx<T>(store: AztecLMDBStoreV2, fn: (tx: WriteTransaction) => Promise<T>): Promise<T> {
+  const currentWrite = store.getCurrentWriteTx();
+  if (currentWrite) {
+    return fn(currentWrite);
+  } else {
+    return store.transactionAsync(fn);
+  }
+}
+
+export async function execInReadTx<T>(
+  store: AztecLMDBStoreV2,
+  fn: (tx: ReadTransaction) => T | Promise<T>,
+): Promise<T> {
+  const currentWrite = store.getCurrentWriteTx();
+  if (currentWrite) {
+    return await fn(currentWrite);
+  } else {
+    const tx = store.getReadTx();
+    try {
+      return await fn(tx);
+    } finally {
+      tx.close();
+    }
+  }
+}

package/src/lmdb-v2/utils.ts

@@ -2,124 +2,6 @@ import { MAXIMUM_KEY, fromBufferKey, toBufferKey } from 'ordered-binary';
 
 import type { Key } from '../interfaces/common.js';
 
-type Cmp<T> = (a: T, b: T) => -1 | 0 | 1;
-
-export function dedupeSortedArray<T>(arr: T[], cmp: Cmp<T>): void {
-  for (let i = 0; i < arr.length; i++) {
-    let j = i + 1;
-    for (; j < arr.length; j++) {
-      const res = cmp(arr[i], arr[j]);
-      if (res === 0) {
-        continue;
-      } else if (res < 0) {
-        break;
-      } else {
-        throw new Error('Array not sorted');
-      }
-    }
-
-    if (j - i > 1) {
-      arr.splice(i + 1, j - i - 1);
-    }
-  }
-}
-
-export function insertIntoSortedArray<T>(arr: T[], item: T, cmp: (a: T, b: T) => number): void {
-  let left = 0;
-  let right = arr.length;
-
-  while (left < right) {
-    const mid = (left + right) >> 1;
-    const comparison = cmp(arr[mid], item);
-
-    if (comparison < 0) {
-      left = mid + 1;
-    } else {
-      right = mid;
-    }
-  }
-
-  arr.splice(left, 0, item);
-}
-
-export function findIndexInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): number {
-  let start = 0;
-  let end = values.length - 1;
-
-  while (start <= end) {
-    const mid = start + (((end - start) / 2) | 0);
-    const res = cmp(values[mid], needle);
-    if (res === 0) {
-      return mid;
-    } else if (res > 0) {
-      end = mid - 1;
-    } else {
-      start = mid + 1;
-    }
-  }
-
-  return -1;
-}
-
-export function findInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): T | undefined {
-  const idx = findIndexInSortedArray(values, needle, cmp);
-  return idx > -1 ? values[idx] : undefined;
-}
-
-export function removeAnyOf<T, N>(arr: T[], vals: N[], cmp: (a: T, b: N) => -1 | 0 | 1): void {
-  let writeIdx = 0;
-  let readIdx = 0;
-  let valIdx = 0;
-
-  while (readIdx < arr.length && valIdx < vals.length) {
-    const comparison = cmp(arr[readIdx], vals[valIdx]);
-
-    if (comparison < 0) {
-      arr[writeIdx++] = arr[readIdx++];
-    } else if (comparison > 0) {
-      valIdx++;
-    } else {
-      readIdx++;
-    }
-  }
-
-  while (readIdx < arr.length) {
-    arr[writeIdx++] = arr[readIdx++];
-  }
-
-  arr.length = writeIdx;
-}
-
-export function removeFromSortedArray<T, N>(arr: T[], val: N, cmp: (a: T, b: N) => -1 | 0 | 1) {
-  const idx = findIndexInSortedArray(arr, val, cmp);
-  if (idx > -1) {
-    arr.splice(idx, 1);
-  }
-}
-
-export function merge<T>(arr: T[], toInsert: T[], cmp: (a: T, b: T) => -1 | 0 | 1): void {
-  const result = new Array<T>(arr.length + toInsert.length);
-  let i = 0,
-    j = 0,
-    k = 0;
-
-  while (i < arr.length && j < toInsert.length) {
-    result[k++] = cmp(arr[i], toInsert[j]) <= 0 ? arr[i++] : toInsert[j++];
-  }
-
-  while (i < arr.length) {
-    result[k++] = arr[i++];
-  }
-  while (j < toInsert.length) {
-    result[k++] = toInsert[j++];
-  }
-
-  for (i = 0; i < result.length; i++) {
-    arr[i] = result[i];
-  }
-  arr.length = result.length;
-}
-
 export function keyCmp(a: [Uint8Array, Uint8Array[] | null], b: [Uint8Array, Uint8Array[] | null]): -1 | 0 | 1 {
   return Buffer.compare(a[0], b[0]);
 }

package/src/lmdb-v2/write_transaction.ts

@@ -1,16 +1,16 @@
-import { type Batch, Database, LMDBMessageType } from './message.js';
-import { ReadTransaction } from './read_transaction.js';
 import {
   dedupeSortedArray,
   findInSortedArray,
   findIndexInSortedArray,
   insertIntoSortedArray,
-  keyCmp,
   merge,
   removeAnyOf,
   removeFromSortedArray,
-  singleKeyCmp,
-} from './utils.js';
+} from '@aztec/foundation/array';
+
+import { type Batch, Database, LMDBMessageType } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+import { keyCmp, singleKeyCmp } from './utils.js';
 
 export class WriteTransaction extends ReadTransaction {
   // exposed for tests
@@ -44,7 +44,7 @@ export class WriteTransaction extends ReadTransaction {
   remove(key: Uint8Array): Promise<void> {
     const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
     if (removeEntryIndex === -1) {
-      this.dataBatch.removeEntries.push([key, null]);
+      insertIntoSortedArray(this.dataBatch.removeEntries, [key, null], keyCmp);
     }
 
     const addEntryIndex = findIndexInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
@@ -163,7 +163,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterate(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array]> {
@@ -181,7 +181,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterateIndex(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array[]]> {
@@ -218,8 +218,9 @@ export class WriteTransaction extends ReadTransaction {
   ): AsyncIterable<[Uint8Array, T]> {
     this.assertIsOpen();
 
-    // make a copy of this in case we're running in reverse
+    // Snapshot both add and remove entries at the start of iteration to ensure consistency
     const uncommittedEntries = [...batch.addEntries];
+    const removeEntries = [...batch.removeEntries];
     // used to check we're in the right order when comparing between a key and uncommittedEntries
     let cmpDirection = -1;
     if (reverse) {
@@ -262,7 +263,7 @@ export class WriteTransaction extends ReadTransaction {
         break;
       }
 
-      const toRemove = findInSortedArray(batch.removeEntries, key, singleKeyCmp);
+      const toRemove = findInSortedArray(removeEntries, key, singleKeyCmp);
 
       // at this point we've either exhausted all uncommitted entries,
       // we reached a key strictly greater/smaller than `key`

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from '#msgpackr';
+import { toBufferKey } from '#ordered-binary';
+import { hash } from 'ohash';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/errors.ts

@@ -0,0 +1,44 @@
+/**
+ * Typed error surface for sqlite3mc-backed page-level encryption failures.
+ *
+ * Three concrete failure modes are surfaced:
+ *
+ *   - `invalid_key_length`: caller-side pre-flight (key not 32 bytes).
+ *   - `encryption_not_supported_for_ephemeral`: caller-side pre-flight (encryption was requested on an ephemeral
+ *     `:memory:` store, which sqlite3mc does not support).
+ *   - `decrypt_failed`: runtime failure raised when sqlite3mc cannot decode page 1 of an existing database. Covers
+ *     both "wrong key supplied" and "no key supplied to an encrypted DB".
+ */
+export type SqliteEncryptionErrorCode =
+  | 'invalid_key_length'
+  | 'encryption_not_supported_for_ephemeral'
+  | 'decrypt_failed';
+
+/**
+ * Error thrown by sqlite-opfs when an encryption operation fails.
+ **/
+export class SqliteEncryptionError extends Error {
+  readonly code: SqliteEncryptionErrorCode;
+
+  constructor(code: SqliteEncryptionErrorCode, message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteEncryptionError';
+    this.code = code;
+  }
+}
+
+/**
+ * Strings raised by sqlite3mc when page 1 cannot be decoded.
+ **/
+const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
+  /file is not a database/i,
+  /file is encrypted or is not a database/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known sqlite3mc decrypt-failure
+ * strings.
+ **/
+export function isDecryptFailureMessage(message: string): boolean {
+  return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,39 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+export { SqliteEncryptionError } from './errors.js';
+export type { SqliteEncryptionErrorCode } from './errors.js';
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  const { dataDirectory } = config;
+  log.info(
+    dataDirectory
+      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}