@aztec/kv-store 0.76.4 → 0.77.0-testnet-ignition.17

package/src/interfaces/array_test_suite.ts

@@ -2,8 +2,8 @@ import { toArray } from '@aztec/foundation/iterable';
 
 import { expect } from 'chai';
 
-import { type AztecArray, type AztecAsyncArray } from './array.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { AztecArray, AztecAsyncArray } from './array.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
 
 export function describeAztecArray(

package/src/interfaces/common.ts

@@ -1,7 +1,7 @@
 /**
  * The key type for use with the kv-store
  */
-export type Key = string | number;
+export type Key = string | number | Array<string | number>;
 
 /**
  * A range of keys to iterate over.

package/src/interfaces/counter.ts

@@ -1,4 +1,4 @@
-import { type Key, type Range } from './common.js';
+import type { Key, Range } from './common.js';
 
 /**
  * A map that counts how many times it sees a key. Once 0 is reached,  that key is removed from the map.

package/src/interfaces/index.ts

@@ -4,4 +4,4 @@ export * from './counter.js';
 export * from './singleton.js';
 export * from './store.js';
 export * from './set.js';
-export { Range, StoreSize } from './common.js';
+export type { Range, StoreSize } from './common.js';

package/src/interfaces/map.ts

@@ -1,4 +1,4 @@
-import { type Key, type Range } from './common.js';
+import type { Key, Range } from './common.js';
 
 /**
  * A map backed by a persistent store.

package/src/interfaces/map_test_suite.ts

@@ -2,9 +2,9 @@ import { toArray } from '@aztec/foundation/iterable';
 
 import { expect } from 'chai';
 
-import { type Key, type Range } from './common.js';
-import { type AztecAsyncMap, type AztecAsyncMultiMap, type AztecMap, type AztecMultiMap } from './map.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { Key, Range } from './common.js';
+import type { AztecAsyncMap, AztecAsyncMultiMap, AztecMap, AztecMultiMap } from './map.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
 
 export function describeAztecMap(

package/src/interfaces/set.ts

@@ -1,4 +1,4 @@
-import { type Key, type Range } from './common.js';
+import type { Key, Range } from './common.js';
 
 /**
  * A set backed by a persistent store.

package/src/interfaces/set_test_suite.ts

@@ -2,9 +2,9 @@ import { toArray } from '@aztec/foundation/iterable';
 
 import { expect } from 'chai';
 
-import { type Range } from './common.js';
-import { type AztecAsyncSet, type AztecSet } from './set.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { Range } from './common.js';
+import type { AztecAsyncSet, AztecSet } from './set.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
 
 export function describeAztecSet(

package/src/interfaces/singleton_test_suite.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
-import { type AztecAsyncSingleton, type AztecSingleton } from './singleton.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
 
 export function describeAztecSingleton(

package/src/interfaces/store.ts

@@ -1,16 +1,16 @@
-import { type AztecArray, type AztecAsyncArray } from './array.js';
-import { type Key, type StoreSize } from './common.js';
-import { type AztecAsyncCounter, type AztecCounter } from './counter.js';
-import {
-  type AztecAsyncMap,
-  type AztecAsyncMultiMap,
-  type AztecMap,
-  type AztecMapWithSize,
-  type AztecMultiMap,
-  type AztecMultiMapWithSize,
+import type { AztecArray, AztecAsyncArray } from './array.js';
+import type { Key, StoreSize } from './common.js';
+import type { AztecAsyncCounter, AztecCounter } from './counter.js';
+import type {
+  AztecAsyncMap,
+  AztecAsyncMultiMap,
+  AztecMap,
+  AztecMapWithSize,
+  AztecMultiMap,
+  AztecMultiMapWithSize,
 } from './map.js';
-import { type AztecAsyncSet, type AztecSet } from './set.js';
-import { type AztecAsyncSingleton, type AztecSingleton } from './singleton.js';
+import type { AztecAsyncSet, AztecSet } from './set.js';
+import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
 
 /** A key-value store */
 export interface AztecKVStore {

package/src/interfaces/store_test_suite.ts

@@ -1,7 +1,7 @@
 import { expect } from 'chai';
 
-import { type AztecAsyncSingleton, type AztecSingleton } from './singleton.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 import { isSyncStore } from './utils.js';
 
 export function describeAztecStore(

package/src/interfaces/utils.ts

@@ -1,4 +1,4 @@
-import { type AztecAsyncKVStore, type AztecKVStore } from './store.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
 
 /* eslint-disable no-console */
 export const mockLogger = {

package/src/lmdb/array.ts

@@ -1,6 +1,6 @@
-import { type Database, type Key } from 'lmdb';
+import type { Database, Key } from 'lmdb';
 
-import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
+import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
 import { LmdbAztecSingleton } from './singleton.js';
 
 /** The shape of a key that stores a value in an array */

package/src/lmdb/counter.ts

@@ -1,7 +1,7 @@
-import { type Key as BaseKey, type Database } from 'lmdb';
+import type { Key as BaseKey, Database } from 'lmdb';
 
-import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
 import { LmdbAztecMap } from './map.js';
 
 /**

package/src/lmdb/index.ts

@@ -2,7 +2,7 @@ import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import { join } from 'path';
 
-import { type DataStoreConfig } from '../config.js';
+import type { DataStoreConfig } from '../config.js';
 import { initStoreForRollup } from '../utils.js';
 import { AztecLmdbStore } from './store.js';
 

package/src/lmdb/map.ts

@@ -1,7 +1,7 @@
-import { type Database, type RangeOptions } from 'lmdb';
+import type { Database, RangeOptions } from 'lmdb';
 
-import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecAsyncMultiMap, type AztecMapWithSize, type AztecMultiMap } from '../interfaces/map.js';
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncMultiMap, AztecMapWithSize, AztecMultiMap } from '../interfaces/map.js';
 
 /** The slot where a key-value entry would be stored */
 type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];

package/src/lmdb/set.ts

@@ -1,7 +1,7 @@
-import { type Database } from 'lmdb';
+import type { Database } from 'lmdb';
 
-import { type Key, type Range } from '../interfaces/common.js';
-import { type AztecAsyncSet, type AztecSet } from '../interfaces/set.js';
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js';
 import { LmdbAztecMap } from './map.js';
 
 /**

package/src/lmdb/singleton.ts

@@ -1,6 +1,6 @@
-import { type Database, type Key } from 'lmdb';
+import type { Database, Key } from 'lmdb';
 
-import { type AztecAsyncSingleton, type AztecSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js';
 
 /** The slot where this singleton will store its value */
 type ValueSlot = ['singleton', string, 'value'];

package/src/lmdb/store.ts

@@ -6,20 +6,20 @@ import { type Database, type RootDatabase, open } from 'lmdb';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
-import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
-import { type Key, type StoreSize } from '../interfaces/common.js';
-import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
-import {
-  type AztecAsyncMap,
-  type AztecAsyncMultiMap,
-  type AztecMap,
-  type AztecMapWithSize,
-  type AztecMultiMap,
-  type AztecMultiMapWithSize,
+import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
+import type { Key, StoreSize } from '../interfaces/common.js';
+import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
+import type {
+  AztecAsyncMap,
+  AztecAsyncMultiMap,
+  AztecMap,
+  AztecMapWithSize,
+  AztecMultiMap,
+  AztecMultiMapWithSize,
 } from '../interfaces/map.js';
-import { type AztecAsyncSet, type AztecSet } from '../interfaces/set.js';
-import { type AztecAsyncSingleton, type AztecSingleton } from '../interfaces/singleton.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from '../interfaces/store.js';
+import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js';
+import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncKVStore, AztecKVStore } from '../interfaces/store.js';
 import { LmdbAztecArray } from './array.js';
 import { LmdbAztecCounter } from './counter.js';
 import { LmdbAztecMap, LmdbAztecMapWithSize } from './map.js';
@@ -211,7 +211,7 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
     await this.drop();
     await this.close();
     if (this.path) {
-      await fs.rm(this.path, { recursive: true, force: true });
+      await fs.rm(this.path, { recursive: true, force: true, maxRetries: 3 });
       this.#log.verbose(`Deleted database files at ${this.path}`);
     }
   }

package/src/lmdb-v2/array.ts

@@ -0,0 +1,115 @@
+import { Encoder } from 'msgpackr/pack';
+
+import type { AztecAsyncArray } from '../interfaces/array.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { ReadTransaction } from './read_transaction.js';
+import { AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
+import { deserializeKey, serializeKey } from './utils.js';
+
+export class LMDBArray<T> implements AztecAsyncArray<T> {
+  private length: AztecAsyncSingleton<number>;
+  private encoder = new Encoder();
+  private prefix: string;
+
+  constructor(private store: AztecLMDBStoreV2, name: string) {
+    this.length = store.openSingleton(name + ':length');
+    this.prefix = `array:${name}`;
+  }
+
+  pop(): Promise<T | undefined> {
+    return execInWriteTx(this.store, async tx => {
+      const length = await this.lengthAsync();
+      if (length === 0) {
+        return;
+      }
+
+      const val = await tx.get(serializeKey(this.prefix, length - 1));
+      await tx.remove(serializeKey(this.prefix, length - 1));
+
+      await this.length.set(length - 1);
+
+      return val ? this.encoder.unpack(val) : undefined;
+    });
+  }
+
+  push(...vals: T[]): Promise<number> {
+    return execInWriteTx(this.store, async tx => {
+      let length = await this.lengthAsync();
+      for (const val of vals) {
+        await tx.set(serializeKey(this.prefix, length++), this.encoder.pack(val));
+      }
+      await this.length.set(length);
+      return length;
+    });
+  }
+
+  setAt(index: number, val: T): Promise<boolean> {
+    return execInWriteTx(this.store, async tx => {
+      const length = await this.lengthAsync();
+      if (index < 0) {
+        index += length;
+      }
+      if (index < 0 || index >= length) {
+        return false;
+      }
+      await tx.set(serializeKey(this.prefix, index), this.encoder.pack(val));
+      return true;
+    });
+  }
+
+  atAsync(index: number): Promise<T | undefined> {
+    return execInReadTx(this.store, async tx => {
+      const length = await this.lengthAsync();
+      if (index < 0) {
+        index += length;
+      }
+      if (index < 0 || index >= length) {
+        return undefined;
+      }
+
+      const val = await tx.get(serializeKey(this.prefix, index));
+      return val ? this.encoder.unpack(val) : undefined;
+    });
+  }
+
+  async lengthAsync(): Promise<number> {
+    return (await this.length.getAsync()) ?? 0;
+  }
+
+  async *entriesAsync(): AsyncIterableIterator<[number, T]> {
+    // pin array length so that pushes don't affect iteration
+    const length = await this.lengthAsync();
+    if (length === 0) {
+      return;
+    }
+
+    let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
+    const shouldClose = !tx;
+    tx ??= this.store.getReadTx();
+
+    try {
+      for await (const [key, val] of tx.iterate(serializeKey(this.prefix, 0), undefined, false, length)) {
+        const deserializedKey = deserializeKey<number>(this.prefix, key);
+        // if pops happened while iterating we may have read too much. Terminate early
+        if (deserializedKey === false) {
+          break;
+        }
+        yield [deserializedKey, this.encoder.unpack(val)];
+      }
+    } finally {
+      if (shouldClose) {
+        tx.close();
+      }
+    }
+  }
+
+  async *valuesAsync(): AsyncIterableIterator<T> {
+    for await (const [_, value] of this.entriesAsync()) {
+      yield value;
+    }
+  }
+
+  [Symbol.asyncIterator](): AsyncIterableIterator<T> {
+    return this.valuesAsync();
+  }
+}

package/src/lmdb-v2/factory.ts

@@ -1,11 +1,11 @@
-import { EthAddress } from '@aztec/circuits.js';
+import { EthAddress } from '@aztec/foundation/eth-address';
 import { type Logger, createLogger } from '@aztec/foundation/log';
 
 import { mkdir, mkdtemp, readFile, rm, writeFile } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
-import { type DataStoreConfig } from '../config.js';
+import type { DataStoreConfig } from '../config.js';
 import { AztecLMDBStoreV2 } from './store.js';
 
 const ROLLUP_ADDRESS_FILE = 'rollup_address';
@@ -36,7 +36,7 @@ export async function createStore(
             found: localRollupAddress,
           });
 
-          await rm(subDir, { recursive: true, force: true });
+          await rm(subDir, { recursive: true, force: true, maxRetries: 3 });
           await mkdir(subDir, { recursive: true });
         }
 
@@ -68,8 +68,12 @@ export async function openTmpStore(
   // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
   const cleanup = async () => {
     if (ephemeral) {
-      await rm(dataDir, { recursive: true, force: true });
-      log.debug(`Deleted temporary data store: ${dataDir}`);
+      try {
+        await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
+        log.debug(`Deleted temporary data store: ${dataDir}`);
+      } catch (err) {
+        log.warn(`Failed to delete temporary data directory (LMDB v2) ${dataDir}: ${err}`);
+      }
     } else {
       log.debug(`Leaving temporary data store: ${dataDir}`);
     }

package/src/lmdb-v2/map.ts

@@ -2,7 +2,7 @@ import { Encoder } from 'msgpackr';
 
 import type { Key, Range } from '../interfaces/common.js';
 import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
-import { type ReadTransaction } from './read_transaction.js';
+import type { ReadTransaction } from './read_transaction.js';
 import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
 import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
 

package/src/lmdb-v2/singleton.ts

@@ -1,6 +1,6 @@
 import { Encoder } from 'msgpackr';
 
-import { type AztecAsyncSingleton } from '../interfaces/singleton.js';
+import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
 import { serializeKey } from './utils.js';
 

package/src/lmdb-v2/store.ts

@@ -12,6 +12,7 @@ import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
 import type { AztecAsyncSet } from '../interfaces/set.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
+import { LMDBArray } from './array.js';
 import { LMDBMap, LMDBMultiMap } from './map.js';
 import {
   Database,
@@ -103,8 +104,8 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return new LMDBSingleValue(this, name);
   }
 
-  openArray<T>(_name: string): AztecAsyncArray<T> {
-    throw new Error('Not implemented');
+  openArray<T>(name: string): AztecAsyncArray<T> {
+    return new LMDBArray(this, name);
   }
 
   openSet<K extends Key>(_name: string): AztecAsyncSet<K> {
@@ -155,7 +156,7 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
 
   async delete(): Promise<void> {
     await this.close();
-    await rm(this.dataDir, { recursive: true, force: true });
+    await rm(this.dataDir, { recursive: true, force: true, maxRetries: 3 });
     this.log.verbose(`Deleted database files at ${this.dataDir}`);
     await this.cleanup?.();
   }

package/src/lmdb-v2/utils.ts

@@ -1,6 +1,6 @@
 import { MAXIMUM_KEY, fromBufferKey, toBufferKey } from 'ordered-binary';
 
-import { type Key } from '../interfaces/common.js';
+import type { Key } from '../interfaces/common.js';
 
 type Cmp<T> = (a: T, b: T) => -1 | 0 | 1;
 

package/src/stores/l2_tips_store.ts

@@ -1,14 +1,14 @@
-import {
-  type L2BlockId,
-  type L2BlockStreamEvent,
-  type L2BlockStreamEventHandler,
-  type L2BlockStreamLocalDataProvider,
-  type L2BlockTag,
-  type L2Tips,
-} from '@aztec/circuit-types';
+import type {
+  L2BlockId,
+  L2BlockStreamEvent,
+  L2BlockStreamEventHandler,
+  L2BlockStreamLocalDataProvider,
+  L2BlockTag,
+  L2Tips,
+} from '@aztec/stdlib/block';
 
-import { type AztecAsyncMap } from '../interfaces/map.js';
-import { type AztecAsyncKVStore } from '../interfaces/store.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncKVStore } from '../interfaces/store.js';
 
 /** Stores currently synced L2 tips and unfinalized block hashes. */
 export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {

package/src/utils.ts

@@ -1,8 +1,8 @@
-import { type EthAddress } from '@aztec/foundation/eth-address';
-import { type Logger } from '@aztec/foundation/log';
+import type { EthAddress } from '@aztec/foundation/eth-address';
+import type { Logger } from '@aztec/foundation/log';
 
-import { type AztecAsyncSingleton, type AztecSingleton } from './interfaces/singleton.js';
-import { type AztecAsyncKVStore, type AztecKVStore } from './interfaces/store.js';
+import type { AztecAsyncSingleton, AztecSingleton } from './interfaces/singleton.js';
+import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
 import { isSyncStore } from './interfaces/utils.js';
 
 /**