@aztec/kv-store 0.0.1-commit.b655e406 → 0.0.1-commit.b9865e97

package/src/sqlite-opfs/array.ts

@@ -0,0 +1,124 @@
+import { Encoder } from '#msgpackr';
+import { toBufferKey } from '#ordered-binary';
+import { hash } from 'ohash';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { Value } from '../interfaces/common.js';
+import type { AztecSQLiteOPFSStore } from './store.js';
+
+/**
+ * Persistent array backed by SQLite. Entries share a common `key` (the array name)
+ * and are ordered by `key_count`, which doubles as the 1-indexed slot number.
+ */
+export class SQLiteOPFSAztecArray<T extends Value> implements AztecAsyncArray<T> {
+  readonly #name: string;
+  readonly #container: string;
+  readonly #encoder = new Encoder();
+
+  constructor(
+    private readonly store: AztecSQLiteOPFSStore,
+    name: string,
+  ) {
+    this.#name = name;
+    this.#container = `array:${name}`;
+  }
+
+  async lengthAsync(): Promise<number> {
+    const rows = await this.store.allAsync('SELECT COUNT(*) FROM data WHERE container = ? AND key = ?', [
+      this.#container,
+      this.#encodedKey(),
+    ]);
+    return Number(rows[0]?.[0] ?? 0);
+  }
+
+  async push(...vals: T[]): Promise<number> {
+    if (vals.length === 0) {
+      return this.lengthAsync();
+    }
+    return await this.store.transactionAsync(async () => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await this.store.runAsync(
+          `INSERT INTO data (slot, container, key, key_count, hash, value)
+           VALUES (?, ?, ?, ?, ?, ?)`,
+          [this.#slot(length), this.#container, this.#encodedKey(), length + 1, hash(val), this.#encoder.pack(val)],
+        );
+        length += 1;
+      }
+      return length;
+    });
+  }
+
+  async pop(): Promise<T | undefined> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return undefined;
+      }
+      const slot = this.#slot(length - 1);
+      const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [slot]);
+      await this.store.runAsync('DELETE FROM data WHERE slot = ?', [slot]);
+      const raw = rows[0]?.[0];
+      return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+    });
+  }
+
+  async atAsync(index: number): Promise<T | undefined> {
+    const length = await this.lengthAsync();
+    const resolved = index < 0 ? length + index : index;
+    if (resolved < 0 || resolved >= length) {
+      return undefined;
+    }
+    const rows = await this.store.allAsync('SELECT value FROM data WHERE slot = ? LIMIT 1', [this.#slot(resolved)]);
+    const raw = rows[0]?.[0];
+    return raw instanceof Uint8Array ? (this.#encoder.unpack(raw) as T) : undefined;
+  }
+
+  async setAt(index: number, val: T): Promise<boolean> {
+    return await this.store.transactionAsync(async () => {
+      const length = await this.lengthAsync();
+      const resolved = index < 0 ? length + index : index;
+      if (resolved < 0 || resolved >= length) {
+        return false;
+      }
+      await this.store.runAsync(
+        `INSERT OR REPLACE INTO data (slot, container, key, key_count, hash, value)
+         VALUES (?, ?, ?, ?, ?, ?)`,
+        [this.#slot(resolved), this.#container, this.#encodedKey(), resolved + 1, hash(val), this.#encoder.pack(val)],
+      );
+      return true;
+    });
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    const rows = await this.store.allAsync(
+      'SELECT key_count, value FROM data WHERE container = ? AND key = ? ORDER BY key_count ASC',
+      [this.#container, this.#encodedKey()],
+    );
+    for (const row of rows) {
+      const keyCount = Number(row[0]);
+      const raw = row[1];
+      if (raw instanceof Uint8Array) {
+        yield [keyCount - 1, this.#encoder.unpack(raw) as T];
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [, val] of this.entriesAsync()) {
+      yield val;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+
+  #encodedKey(): Buffer {
+    return toBufferKey([this.#name]);
+  }
+
+  #slot(index: number): string {
+    return `array:${this.#name}:slot:${index}`;
+  }
+}

package/src/sqlite-opfs/errors.ts

@@ -0,0 +1,44 @@
+/**
+ * Typed error surface for sqlite3mc-backed page-level encryption failures.
+ *
+ * Three concrete failure modes are surfaced:
+ *
+ *   - `invalid_key_length`: caller-side pre-flight (key not 32 bytes).
+ *   - `encryption_not_supported_for_ephemeral`: caller-side pre-flight (encryption was requested on an ephemeral
+ *     `:memory:` store, which sqlite3mc does not support).
+ *   - `decrypt_failed`: runtime failure raised when sqlite3mc cannot decode page 1 of an existing database. Covers
+ *     both "wrong key supplied" and "no key supplied to an encrypted DB".
+ */
+export type SqliteEncryptionErrorCode =
+  | 'invalid_key_length'
+  | 'encryption_not_supported_for_ephemeral'
+  | 'decrypt_failed';
+
+/**
+ * Error thrown by sqlite-opfs when an encryption operation fails.
+ **/
+export class SqliteEncryptionError extends Error {
+  readonly code: SqliteEncryptionErrorCode;
+
+  constructor(code: SqliteEncryptionErrorCode, message: string, opts?: { cause?: unknown }) {
+    super(message, opts?.cause !== undefined ? { cause: opts.cause } : undefined);
+    this.name = 'SqliteEncryptionError';
+    this.code = code;
+  }
+}
+
+/**
+ * Strings raised by sqlite3mc when page 1 cannot be decoded.
+ **/
+const SQLITE3MC_DECRYPT_ERROR_PATTERNS: readonly RegExp[] = [
+  /file is not a database/i,
+  /file is encrypted or is not a database/i,
+];
+
+/**
+ * Returns `true` if `message` matches one of the known sqlite3mc decrypt-failure
+ * strings.
+ **/
+export function isDecryptFailureMessage(message: string): boolean {
+  return SQLITE3MC_DECRYPT_ERROR_PATTERNS.some(p => p.test(message));
+}

package/src/sqlite-opfs/index.ts

@@ -0,0 +1,39 @@
+import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
+
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
+import { AztecSQLiteOPFSStore } from './store.js';
+
+export { AztecSQLiteOPFSStore } from './store.js';
+export { SqliteEncryptionError } from './errors.js';
+export type { SqliteEncryptionErrorCode } from './errors.js';
+
+export async function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
+  const { dataDirectory } = config;
+  log.info(
+    dataDirectory
+      ? `Creating ${name} SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`
+      : `Creating ${name} ephemeral SQLite-OPFS data store with map size ${config.dataStoreMapSizeKb} KB`,
+  );
+  const store = await AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false);
+  return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.rollupAddress, log);
+}
+
+export function openTmpStore(ephemeral: boolean = false): Promise<AztecSQLiteOPFSStore> {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), undefined, ephemeral);
+}
+
+/**
+ * Convenience helper for tests and consumers that want an encrypted sqlite-opfs
+ * store without dealing with the full `open()` parameter order. Key must be 32
+ * bytes. Creates a fresh persistent store (sqlite3mc does not support encryption
+ * on ephemeral `:memory:` databases) in an auto-generated OPFS directory.
+ */
+export function openEncryptedStore(encryptionKey: Uint8Array, name?: string, poolDirectory?: string) {
+  return AztecSQLiteOPFSStore.open(createLogger('kv-store:sqlite-opfs'), name, false, poolDirectory, encryptionKey);
+}

package/src/sqlite-opfs/internal/ordered-binary-browser.js

@@ -0,0 +1,465 @@
+/* eslint-disable */
+/**
+ * Vendored from `ordered-binary@1.5.3/index.js` with one targeted edit:
+ * the `readString` definition (originally a dynamically-generated
+ * unrolled-loop function) is replaced with a hand-written interpreted
+ * equivalent that does the same work without code-generation-from-strings.
+ *
+ * Why vendor: upstream has no CSP-safe build, and the codegen is a
+ * module-init-time string-passed-to-evaluator that trips MV3 CSP
+ * (`script-src 'self' 'wasm-unsafe-eval'`). The codegen is a perf
+ * optimization on the string-decoding path only — the interpreted
+ * version is functionally identical, just slower per call.
+ *
+ * Parity with upstream is enforced by `ordered-binary-browser.test.ts`,
+ * which compares byte-exact output for a representative key set on every
+ * Node-mode test run. If `ordered-binary` is bumped, re-vendor by
+ * recopying upstream and reapplying the patch in the section marked
+ * `// CSP PATCH BEGIN` ... `// CSP PATCH END`.
+ */
+/*
+control character types:
+1 - metadata
+2 - symbols
+6 - false
+7 - true
+8- 16 - negative doubles
+16-24 positive doubles
+27 - String starts with a character 27 or less or is an empty string
+0 - multipart separator
+> 27 normal string characters
+*/
+/*
+ * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
+ */
+
+const float64Array = new Float64Array(2);
+const int32Array = new Int32Array(float64Array.buffer, 0, 4);
+let nullTerminate = false;
+let textEncoder;
+try {
+  textEncoder = new TextEncoder();
+} catch (error) {}
+
+/*
+ * Convert arbitrary scalar values to buffer bytes with type preservation and type-appropriate ordering
+ */
+export function writeKey(key, target, position, inSequence) {
+  let targetView = target.dataView;
+  if (!targetView) {
+    targetView = target.dataView = new DataView(target.buffer, target.byteOffset, ((target.byteLength + 3) >> 2) << 2);
+  }
+  switch (typeof key) {
+    case 'string':
+      let strLength = key.length;
+      let c1 = key.charCodeAt(0);
+      if (!(c1 >= 28)) {
+        // escape character
+        target[position++] = 27;
+      }
+      if (strLength < 0x40) {
+        let i, c2;
+        for (i = 0; i < strLength; i++) {
+          c1 = key.charCodeAt(i);
+          if (c1 <= 4) {
+            target[position++] = 4;
+            target[position++] = c1;
+          } else if (c1 < 0x80) {
+            target[position++] = c1;
+          } else if (c1 < 0x800) {
+            target[position++] = (c1 >> 6) | 0xc0;
+            target[position++] = (c1 & 0x3f) | 0x80;
+          } else if ((c1 & 0xfc00) === 0xd800 && ((c2 = key.charCodeAt(i + 1)) & 0xfc00) === 0xdc00) {
+            c1 = 0x10000 + ((c1 & 0x03ff) << 10) + (c2 & 0x03ff);
+            i++;
+            target[position++] = (c1 >> 18) | 0xf0;
+            target[position++] = ((c1 >> 12) & 0x3f) | 0x80;
+            target[position++] = ((c1 >> 6) & 0x3f) | 0x80;
+            target[position++] = (c1 & 0x3f) | 0x80;
+          } else {
+            target[position++] = (c1 >> 12) | 0xe0;
+            target[position++] = ((c1 >> 6) & 0x3f) | 0x80;
+            target[position++] = (c1 & 0x3f) | 0x80;
+          }
+        }
+      } else {
+        if (target.utf8Write) {
+          position += target.utf8Write(key, position, target.byteLength - position);
+        } else {
+          position += textEncoder.encodeInto(key, target.subarray(position)).written;
+        }
+        if (position > target.length - 4) {
+          throw new RangeError('String does not fit in target buffer');
+        }
+      }
+      break;
+    case 'number':
+      float64Array[0] = key;
+      let lowInt = int32Array[0];
+      let highInt = int32Array[1];
+      let length;
+      if (key < 0) {
+        targetView.setInt32(position + 4, ~((lowInt >>> 4) | (highInt << 28)));
+        targetView.setInt32(position + 0, (highInt ^ 0x7fffffff) >>> 4);
+        targetView.setInt32(position + 8, ((lowInt & 0xf) ^ 0xf) << 4, true); // just always do the null termination here
+        return position + 9;
+      } else if (lowInt & 0xf || inSequence) {
+        length = 9;
+      } else if (lowInt & 0xfffff) {
+        length = 8;
+      } else if (lowInt || highInt & 0xf) {
+        length = 6;
+      } else {
+        length = 4;
+      }
+      // switching order to go to little endian
+      targetView.setInt32(position + 0, (highInt >>> 4) | 0x10000000);
+      targetView.setInt32(position + 4, (lowInt >>> 4) | (highInt << 28));
+      // if (length == 9 || nullTerminate)
+      targetView.setInt32(position + 8, (lowInt & 0xf) << 4, true);
+      return position + length;
+    case 'object':
+      if (key) {
+        if (Array.isArray(key)) {
+          for (let i = 0, l = key.length; i < l; i++) {
+            if (i > 0) {
+              target[position++] = 0;
+            }
+            position = writeKey(key[i], target, position, true);
+          }
+          break;
+        } else if (key instanceof Uint8Array) {
+          target.set(key, position);
+          position += key.length;
+          break;
+        } else {
+          throw new Error('Unable to serialize object as a key: ' + JSON.stringify(key));
+        }
+      } // null
+      else {
+        target[position++] = 0;
+      }
+      break;
+    case 'boolean':
+      targetView.setUint32(position++, key ? 7 : 6, true);
+      return position;
+    case 'bigint':
+      let asFloat = Number(key);
+      if (BigInt(asFloat) > key) {
+        float64Array[0] = asFloat;
+        if (asFloat > 0) {
+          if (int32Array[0]) {
+            int32Array[0]--;
+          } else {
+            int32Array[1]--;
+            int32Array[0] = 0xffffffff;
+          }
+        } else {
+          if (int32Array[0] < 0xffffffff) {
+            int32Array[0]++;
+          } else {
+            int32Array[1]++;
+            int32Array[0] = 0;
+          }
+        }
+        asFloat = float64Array[0];
+      }
+      let difference = key - BigInt(asFloat);
+      if (difference === 0n) {
+        return writeKey(asFloat, target, position, inSequence);
+      }
+      writeKey(asFloat, target, position, inSequence);
+      position += 9; // always increment by 9 if we are adding fractional bits
+      let exponent = BigInt(((int32Array[1] >> 20) & 0x7ff) - 1079);
+      let nextByte = difference >> exponent;
+      target[position - 1] |= Number(nextByte);
+      difference -= nextByte << exponent;
+      let first = true;
+      while (difference || first) {
+        first = false;
+        exponent -= 7n;
+        let nextByte = difference >> exponent;
+        target[position++] = Number(nextByte) | 0x80;
+        difference -= nextByte << exponent;
+      }
+      return position;
+    case 'undefined':
+      return position;
+    // undefined is interpreted as the absence of a key, signified by zero length
+    case 'symbol':
+      target[position++] = 2;
+      return writeKey(key.description, target, position, inSequence);
+    default:
+      throw new Error('Can not serialize key of type ' + typeof key);
+  }
+  if (nullTerminate && !inSequence) {
+    targetView.setUint32(position, 0);
+  }
+  return position;
+}
+
+let position;
+export function readKey(buffer, start, end, inSequence) {
+  position = start;
+  let controlByte = buffer[position];
+  let value;
+  if (controlByte < 24) {
+    if (controlByte < 8) {
+      position++;
+      if (controlByte == 6) {
+        value = false;
+      } else if (controlByte == 7) {
+        value = true;
+      } else if (controlByte == 0) {
+        value = null;
+      } else if (controlByte == 2) {
+        value = Symbol.for(readStringSafely(buffer, end));
+      } else {
+        return Uint8Array.prototype.slice.call(buffer, start, end);
+      }
+    } else {
+      let dataView;
+      try {
+        dataView =
+          buffer.dataView ||
+          (buffer.dataView = new DataView(buffer.buffer, buffer.byteOffset, ((buffer.byteLength + 3) >> 2) << 2));
+      } catch (error) {
+        // if it is write at the end of the ArrayBuffer, we may need to retry with the exact remaining bytes
+        dataView =
+          buffer.dataView ||
+          (buffer.dataView = new DataView(
+            buffer.buffer,
+            buffer.byteOffset,
+            buffer.buffer.byteLength - buffer.byteOffset,
+          ));
+      }
+
+      let highInt = dataView.getInt32(position) << 4;
+      let size = end - position;
+      let lowInt;
+      if (size > 4) {
+        lowInt = dataView.getInt32(position + 4);
+        highInt |= lowInt >>> 28;
+        if (size <= 6) {
+          // clear the last bits
+          lowInt &= -0x10000;
+        }
+        lowInt = lowInt << 4;
+        if (size > 8) {
+          lowInt = lowInt | (buffer[position + 8] >> 4);
+        }
+      } else {
+        lowInt = 0;
+      }
+      if (controlByte < 16) {
+        // negative gets negated
+        highInt = highInt ^ 0x7fffffff;
+        lowInt = ~lowInt;
+      }
+      int32Array[1] = highInt;
+      int32Array[0] = lowInt;
+      value = float64Array[0];
+      position += 9;
+      if (size > 9 && buffer[position] > 0) {
+        // convert the float to bigint, and then we will add precision as we enumerate through the
+        // extra bytes
+        value = BigInt(value);
+        let exponent = (highInt >> 20) & 0x7ff;
+        let next_byte = buffer[position - 1] & 0xf;
+        value += BigInt(next_byte) << BigInt(exponent - 1079);
+        while ((next_byte = buffer[position]) > 0 && position++ < end) {
+          value += BigInt(next_byte & 0x7f) << BigInt((start - position) * 7 + exponent - 1016);
+        }
+      }
+    }
+  } else {
+    if (controlByte == 27) {
+      position++;
+    }
+    value = readStringSafely(buffer, end);
+  }
+  while (position < end) {
+    if (buffer[position] === 0) {
+      position++;
+    }
+    if (inSequence) {
+      encoder.position = position;
+      return value;
+    }
+    let nextValue = readKey(buffer, position, end, true);
+    if (value instanceof Array) {
+      value.push(nextValue);
+    } else {
+      value = [value, nextValue];
+    }
+  }
+  return value;
+}
+export const enableNullTermination = () => (nullTerminate = true);
+
+export const encoder = {
+  writeKey,
+  readKey,
+  enableNullTermination,
+};
+let targetBuffer = [];
+let targetPosition = 0;
+const hasNodeBuffer = typeof Buffer !== 'undefined';
+const ByteArrayAllocate = hasNodeBuffer ? Buffer.allocUnsafeSlow : Uint8Array;
+export const toBufferKey = key => {
+  let newBuffer;
+  if (targetPosition + 100 > targetBuffer.length) {
+    targetBuffer = new ByteArrayAllocate(8192);
+    targetPosition = 0;
+    newBuffer = true;
+  }
+  try {
+    let result = targetBuffer.slice(targetPosition, (targetPosition = writeKey(key, targetBuffer, targetPosition)));
+    if (targetPosition > targetBuffer.length) {
+      if (newBuffer) {
+        throw new Error('Key is too large');
+      }
+      return toBufferKey(key);
+    }
+    return result;
+  } catch (error) {
+    if (newBuffer) {
+      throw error;
+    }
+    targetPosition = targetBuffer.length;
+    return toBufferKey(key);
+  }
+};
+export const fromBufferKey = sourceBuffer => {
+  return readKey(sourceBuffer, 0, sourceBuffer.length);
+};
+const fromCharCode = String.fromCharCode;
+
+let pendingSurrogate;
+function finishUtf8(byte1, src) {
+  if ((byte1 & 0xe0) === 0xc0) {
+    // 2 bytes
+    const byte2 = src[position++] & 0x3f;
+    return ((byte1 & 0x1f) << 6) | byte2;
+  } else if ((byte1 & 0xf0) === 0xe0) {
+    // 3 bytes
+    const byte2 = src[position++] & 0x3f;
+    const byte3 = src[position++] & 0x3f;
+    return ((byte1 & 0x1f) << 12) | (byte2 << 6) | byte3;
+  } else if ((byte1 & 0xf8) === 0xf0) {
+    // 4 bytes
+    if (pendingSurrogate) {
+      byte1 = pendingSurrogate;
+      pendingSurrogate = null;
+      position += 3;
+      return byte1;
+    }
+    const byte2 = src[position++] & 0x3f;
+    const byte3 = src[position++] & 0x3f;
+    const byte4 = src[position++] & 0x3f;
+    let unit = ((byte1 & 0x07) << 0x12) | (byte2 << 0x0c) | (byte3 << 0x06) | byte4;
+    if (unit > 0xffff) {
+      pendingSurrogate = 0xdc00 | (unit & 0x3ff);
+      unit = (((unit - 0x10000) >>> 10) & 0x3ff) | 0xd800;
+      position -= 4; // reset so we can return the next part of the surrogate pair
+    }
+    return unit;
+  } else {
+    return byte1;
+  }
+}
+
+// CSP PATCH BEGIN — replaces upstream's module-init dynamic-codegen with
+// an interpreted equivalent. The original `makeStringBuilder()` produces
+// a function string that unrolls 0x30 iterations of the loop below for
+// perf; this version does the same work as a real loop, slower per call
+// but CSP-safe (no module-init code generation from strings).
+const readString = (function makeReadString() {
+  function readStr(source) {
+    const codes = [];
+    while (codes.length < 0x30) {
+      let v = source[position++];
+      if (v > 4) {
+        if (v >= 0x80) {
+          v = finishUtf8(v, source);
+        }
+      } else if (v === 4) {
+        v = source[position++];
+      } else {
+        return fromCharCode.apply(null, codes);
+      }
+      codes.push(v);
+    }
+    return fromCharCode.apply(null, codes) + readStr(source);
+  }
+  return readStr;
+})();
+// CSP PATCH END
+
+function readStringSafely(source, end) {
+  if (source[end] > 0) {
+    let previous = source[end];
+    try {
+      // read string expects a null terminator, that is a 0 or undefined from reading past the end of the buffer, so we
+      // have to ensure that, but do so safely, restoring the buffer to its original state
+      source[end] = 0;
+      return readString(source);
+    } finally {
+      source[end] = previous;
+    }
+  } else {
+    return readString(source);
+  }
+}
+export function compareKeys(a, b) {
+  // compare with type consistency that matches binary comparison
+  if (typeof a == 'object') {
+    if (!a) {
+      return b == null ? 0 : -1;
+    }
+    if (a.compare) {
+      if (b == null) {
+        return 1;
+      } else if (b.compare) {
+        return a.compare(b);
+      } else {
+        return -1;
+      }
+    }
+    let arrayComparison;
+    if (b instanceof Array) {
+      let i = 0;
+      while ((arrayComparison = compareKeys(a[i], b[i])) == 0 && i <= a.length) {
+        i++;
+      }
+      return arrayComparison;
+    }
+    arrayComparison = compareKeys(a[0], b);
+    if (arrayComparison == 0 && a.length > 1) {
+      return 1;
+    }
+    return arrayComparison;
+  } else if (typeof a == typeof b) {
+    if (typeof a === 'symbol') {
+      a = Symbol.keyFor(a);
+      b = Symbol.keyFor(b);
+    }
+    return a < b ? -1 : a === b ? 0 : 1;
+  } else if (typeof b == 'object') {
+    if (b instanceof Array) {
+      return -compareKeys(b, a);
+    }
+    return 1;
+  } else {
+    return typeOrder[typeof a] < typeOrder[typeof b] ? -1 : 1;
+  }
+}
+const typeOrder = {
+  symbol: 0,
+  undefined: 1,
+  boolean: 2,
+  number: 3,
+  string: 4,
+};
+export const MINIMUM_KEY = null;
+export const MAXIMUM_KEY = new Uint8Array([0xff]);