@aztec/kv-store 0.0.1-commit.b6e433891 → 0.0.1-commit.b9865e97

package/src/lmdb-v2/factory.ts

@@ -1,9 +1,9 @@
 import { EthAddress } from '@aztec/foundation/eth-address';
 import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
-import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
+import { DatabaseVersionManager, type SchemaVersionMismatchPolicy } from '@aztec/stdlib/database-version/manager';
 import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
-import { mkdir, mkdtemp, rm } from 'fs/promises';
+import { copyFile, mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
@@ -11,14 +11,21 @@ import { AztecLMDBStoreV2 } from './store.js';
 
 const MAX_READERS = 16;
 
+/** Optional versioning hooks for persistent LMDB stores. */
+export type CreateStoreOptions = {
+  onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
+  schemaVersionMismatchPolicy?: SchemaVersionMismatchPolicy;
+};
+
 export async function createStore(
   name: string,
   schemaVersion: number,
   config: DataStoreConfig,
   bindings?: LoggerBindings,
+  options: CreateStoreOptions = {},
 ): Promise<AztecLMDBStoreV2> {
   const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
-  const { dataDirectory, l1Contracts } = config;
+  const { dataDirectory, rollupAddress: rollupFromConfig } = config;
 
   let store: AztecLMDBStoreV2;
   if (typeof dataDirectory !== 'undefined') {
@@ -26,7 +33,7 @@ export async function createStore(
     const subDir = join(dataDirectory, name);
     await mkdir(subDir, { recursive: true });
 
-    const rollupAddress = l1Contracts ? l1Contracts.rollupAddress : EthAddress.ZERO;
+    const rollupAddress = rollupFromConfig ?? EthAddress.ZERO;
 
     // Create a version manager
     const versionManager = new DatabaseVersionManager({
@@ -35,6 +42,8 @@ export async function createStore(
       dataDirectory: subDir,
       onOpen: dbDirectory =>
         AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
+      onUpgrade: options.onUpgrade,
+      schemaVersionMismatchPolicy: options.schemaVersionMismatchPolicy,
     });
 
     log.info(
@@ -48,9 +57,14 @@ export async function createStore(
   return store;
 }
 
+/**
+ * Open a persistent on-disk store rooted in OS tmpdir. Caller chooses whether to
+ * auto-remove the directory on close. Uses standard durable LMDB flags (fsync on
+ * every commit) — for full in-memory / NOSYNC semantics use `openEphemeralStore`.
+ */
 export async function openTmpStore(
   name: string,
-  ephemeral: boolean = true,
+  cleanupTmpDir: boolean = true,
   dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
   maxReaders = MAX_READERS,
   bindings?: LoggerBindings,
@@ -61,7 +75,7 @@ export async function openTmpStore(
 
   // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
   const cleanup = async () => {
-    if (ephemeral) {
+    if (cleanupTmpDir) {
       try {
         await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
         log.debug(`Deleted temporary data store: ${dataDir}`);
@@ -73,9 +87,35 @@ export async function openTmpStore(
     }
   };
 
-  // For temporary stores, we don't need to worry about versioning
-  // as they are ephemeral and get cleaned up after use
-  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings);
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, false);
+}
+
+/**
+ * Open a fully-ephemeral store: a fresh tmpdir backing file, LMDB opened with
+ * MDB_NOSYNC | MDB_NOMETASYNC (no fsync, kernel-only writeback), and the directory
+ * unconditionally removed on close. Intended for tests and worker-local scratch
+ * stores where durability is explicitly not desired.
+ */
+export async function openEphemeralStore(
+  name: string,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  log.debug(`Created ephemeral data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
+
+  const cleanup = async () => {
+    try {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      log.debug(`Deleted ephemeral data store: ${dataDir}`);
+    } catch (err) {
+      log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
 }
 
 export async function openStoreAt(
@@ -89,6 +129,36 @@ export async function openStoreAt(
   return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
 }
 
+/**
+ * Open a fully-ephemeral store seeded from an existing `data.mdb` file. Creates a fresh tmpdir,
+ * copies `srcDataMdbPath` into it as `data.mdb`, opens with MDB_NOSYNC | MDB_NOMETASYNC, and
+ * removes the tmpdir on close. Intended for worker clones of a main-thread-built store — the
+ * source file is never written to.
+ */
+export async function cloneEphemeralStoreFrom(
+  srcDataMdbPath: string,
+  name: string,
+  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
+  maxReaders = MAX_READERS,
+  bindings?: LoggerBindings,
+): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
+  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
+  await copyFile(srcDataMdbPath, join(dataDir, 'data.mdb'));
+  log.debug(`Cloned ephemeral data store at: ${dataDir} from ${srcDataMdbPath} (LMDB v2)`);
+
+  const cleanup = async () => {
+    try {
+      await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+      log.debug(`Deleted ephemeral data store: ${dataDir}`);
+    } catch (err) {
+      log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
+    }
+  };
+
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
+}
+
 export async function openVersionedStoreAt(
   dataDirectory: string,
   schemaVersion: number,

package/src/lmdb-v2/read_transaction.ts

@@ -66,20 +66,20 @@ export class ReadTransaction {
   ): AsyncIterable<[Uint8Array, T]> {
     this.assertIsOpen();
 
-    const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
-      key: startKey,
-      reverse,
-      count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
-      onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
-      db,
-    });
-
-    const cursor = response.cursor;
-    let entries = response.entries;
-    let done = typeof cursor !== 'number';
-    let count = 0;
-
+    let cursor: number | undefined;
     try {
+      const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
+        key: startKey,
+        reverse,
+        count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
+        onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
+        db,
+      });
+
+      cursor = response.cursor ?? undefined;
+      let entries = response.entries;
+      let done = typeof cursor !== 'number';
+      let count = 0;
       // emit the first page and any subsequent pages in a while loop
       // NB: end contition is in the middle of the while loop
       while (entries.length > 0) {
@@ -125,17 +125,17 @@ export class ReadTransaction {
   async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
     this.assertIsOpen();
 
-    const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
-      key: startKey,
-      reverse,
-      count: 0,
-      onePage: false,
-      db,
-    });
-
-    const cursor = response.cursor;
-
+    let cursor: number | undefined;
     try {
+      const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
+        key: startKey,
+        reverse,
+        count: 0,
+        onePage: false,
+        db,
+      });
+
+      cursor = response.cursor ?? undefined;
       if (!cursor) {
         return 0;
       }

package/src/lmdb-v2/store.ts

@@ -43,9 +43,10 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     maxReaders: number,
     private log: Logger,
     private cleanup?: () => Promise<void>,
+    ephemeral: boolean = false,
   ) {
     this.log.info(`Starting data store with maxReaders ${maxReaders}`);
-    this.channel = new MsgpackChannel(new NativeLMDBStore(dataDir, mapSize, maxReaders));
+    this.channel = new MsgpackChannel(new NativeLMDBStore(dataDir, mapSize, maxReaders, ephemeral));
     // leave one reader to always be available for regular, atomic, reads
     this.availableCursors = new Semaphore(maxReaders - 1);
   }
@@ -76,9 +77,10 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     maxReaders: number = 16,
     cleanup?: () => Promise<void>,
     bindings?: LoggerBindings,
+    ephemeral: boolean = false,
   ) {
     const log = createLogger('kv-store:lmdb-v2', bindings);
-    const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup);
+    const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup, ephemeral);
     await db.start();
     return db;
   }

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from '#msgpackr';
+import { toBufferKey } from '#ordered-binary';
+import { hash } from 'ohash';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/errors.ts

@@ -0,0 +1,44 @@
+/**
+ * Typed error surface for sqlite3mc-backed page-level encryption failures.
+ *
+ * Three concrete failure modes are surfaced:
+ *
+ *   - `invalid_key_length`: caller-side pre-flight (key not 32 bytes).
+ *   - `encryption_not_supported_for_ephemeral`: caller-side pre-flight (encryption was requested on an ephemeral
+ *     `:memory:` store, which sqlite3mc does not support).
+ *   - `decrypt_failed`: runtime failure raised when sqlite3mc cannot decode page 1 of an existing database. Covers
+ *     both "wrong key supplied" and "no key supplied to an encrypted DB".
+ */
+export type SqliteEncryptionErrorCode =
+  | 'invalid_key_length'
+  | 'encryption_not_supported_for_ephemeral'
+  | 'decrypt_failed';
+
+/**
+ * Error thrown by sqlite-opfs when an encryption operation fails.
+ **/
+export class SqliteEncryptionError extends Error {
+  readonly code: SqliteEncryptionErrorCode;
+
+  constructor(code: SqliteEncryptionErrorCode, message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteEncryptionError';
+    this.code = code;
+  }
+}
+
+/**
+ * Strings raised by sqlite3mc when page 1 cannot be decoded.
+ **/
+const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
+  /file is not a database/i,
+  /file is encrypted or is not a database/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known sqlite3mc decrypt-failure
+ * strings.
+ **/
+export function isDecryptFailureMessage(message: string): boolean {
+  return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,39 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+export { SqliteEncryptionError } from './errors.js';
+export type { SqliteEncryptionErrorCode } from './errors.js';
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  const { dataDirectory } = config;
+  log.info(
+    dataDirectory
+      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}