@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.b655e406

package/src/lmdb-v2/set.ts

@@ -0,0 +1,33 @@
+import type { Key, Range } from '../interfaces/common.js';
+import type { AztecAsyncSet } from '../interfaces/set.js';
+import { LMDBMap } from './map.js';
+import type { AztecLMDBStoreV2 } from './store.js';
+
+/**
+ * A set backed by LMDB.
+ */
+export class LMDBSet<K extends Key> implements AztecAsyncSet<K> {
+  private map: LMDBMap<K, boolean>;
+
+  constructor(store: AztecLMDBStoreV2, mapName: string) {
+    this.map = new LMDBMap(store, mapName);
+  }
+
+  hasAsync(key: K): Promise<boolean> {
+    return this.map.hasAsync(key);
+  }
+
+  add(key: K): Promise<void> {
+    return this.map.set(key, true);
+  }
+
+  delete(key: K): Promise<void> {
+    return this.map.delete(key);
+  }
+
+  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
+    for await (const key of this.map.keysAsync(range)) {
+      yield key;
+    }
+  }
+}

package/src/lmdb-v2/singleton.ts

@@ -1,13 +1,17 @@
 import { Encoder } from 'msgpackr';
 
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
+// eslint-disable-next-line import/no-cycle
 import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
 import { serializeKey } from './utils.js';
 
 export class LMDBSingleValue<T> implements AztecAsyncSingleton<T> {
   private key: Uint8Array;
   private encoder = new Encoder();
-  constructor(private store: AztecLMDBStoreV2, name: string) {
+  constructor(
+    private store: AztecLMDBStoreV2,
+    name: string,
+  ) {
     this.key = serializeKey(`singleton:${name}`, 'value');
   }
 

package/src/lmdb-v2/store.ts

@@ -3,17 +3,20 @@ import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
 import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
 
 import { AsyncLocalStorage } from 'async_hooks';
-import { rm } from 'fs/promises';
+import { mkdir, rm } from 'fs/promises';
 
 import type { AztecAsyncArray } from '../interfaces/array.js';
-import type { Key, StoreSize } from '../interfaces/common.js';
+import type { Key, StoreSize, Value } from '../interfaces/common.js';
 import type { AztecAsyncCounter } from '../interfaces/counter.js';
-import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
+import type { AztecAsyncMap } from '../interfaces/map.js';
+import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
 import type { AztecAsyncSet } from '../interfaces/set.js';
 import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
+// eslint-disable-next-line import/no-cycle
 import { LMDBArray } from './array.js';
-import { LMDBMap, LMDBMultiMap } from './map.js';
+// eslint-disable-next-line import/no-cycle
+import { LMDBMap } from './map.js';
 import {
   Database,
   type LMDBMessageChannel,
@@ -21,7 +24,10 @@ import {
   type LMDBRequestBody,
   type LMDBResponseBody,
 } from './message.js';
+import { LMDBMultiMap } from './multi_map.js';
 import { ReadTransaction } from './read_transaction.js';
+import { LMDBSet } from './set.js';
+// eslint-disable-next-line import/no-cycle
 import { LMDBSingleValue } from './singleton.js';
 import { WriteTransaction } from './write_transaction.js';
 
@@ -77,6 +83,11 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return db;
   }
 
+  public async backupTo(dstPath: string, compact = true) {
+    await mkdir(dstPath, { recursive: true });
+    await this.channel.sendMessage(LMDBMessageType.COPY_STORE, { dstPath, compact });
+  }
+
   public getReadTx(): ReadTransaction {
     if (!this.open) {
       throw new Error('Store is closed');
@@ -92,24 +103,24 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return currentWrite;
   }
 
-  openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
+  openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
     return new LMDBMap(this, name);
   }
 
-  openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
+  openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
     return new LMDBMultiMap(this, name);
   }
 
-  openSingleton<T>(name: string): AztecAsyncSingleton<T> {
+  openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
     return new LMDBSingleValue(this, name);
   }
 
-  openArray<T>(name: string): AztecAsyncArray<T> {
+  openArray<T extends Value>(name: string): AztecAsyncArray<T> {
     return new LMDBArray(this, name);
   }
 
-  openSet<K extends Key>(_name: string): AztecAsyncSet<K> {
-    throw new Error('Not implemented');
+  openSet<K extends Key>(name: string): AztecAsyncSet<K> {
+    return new LMDBSet(this, name);
   }
 
   openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
@@ -150,10 +161,6 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     return Promise.resolve();
   }
 
-  fork(): Promise<AztecAsyncKVStore> {
-    throw new Error('Not implemented');
-  }
-
   async delete(): Promise<void> {
     await this.close();
     await rm(this.dataDir, { recursive: true, force: true, maxRetries: 3 });
@@ -204,6 +211,7 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     const resp = await this.sendMessage(LMDBMessageType.STATS, undefined);
     return {
       mappingSize: Number(resp.dbMapSizeBytes),
+      physicalFileSize: Number(resp.dbPhysicalFileSizeBytes),
       actualSize: resp.stats.reduce((s, db) => Number(db.totalUsedSize) + s, 0),
       numItems: resp.stats.reduce((s, db) => Number(db.numDataItems) + s, 0),
     };

package/src/lmdb-v2/write_transaction.ts

@@ -163,7 +163,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterate(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array]> {
@@ -181,7 +181,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterateIndex(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array[]]> {

package/src/stores/index.ts

@@ -1 +1,3 @@
 export * from './l2_tips_store.js';
+
+export { L2TipsMemoryStore, type L2TipsStore } from '@aztec/stdlib/block';

package/src/stores/l2_tips_store.ts

@@ -11,7 +11,7 @@ import type { AztecAsyncMap } from '../interfaces/map.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
 
 /** Stores currently synced L2 tips and unfinalized block hashes. */
-export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
+export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
   private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, number>;
   private readonly l2BlockHashesStore: AztecAsyncMap<number, string>;
 
@@ -47,24 +47,33 @@ export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLoca
 
   public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
     switch (event.type) {
-      case 'blocks-added':
-        for (const block of event.blocks) {
-          await this.l2BlockHashesStore.set(block.number, (await block.header.hash()).toString());
+      case 'blocks-added': {
+        const blocks = event.blocks.map(b => b.block);
+        for (const block of blocks) {
+          await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
         }
-        await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number);
+        await this.l2TipsStore.set('latest', blocks.at(-1)!.number);
         break;
+      }
       case 'chain-pruned':
-        await this.l2TipsStore.set('latest', event.blockNumber);
+        await this.saveTag('latest', event.block);
         break;
       case 'chain-proven':
-        await this.l2TipsStore.set('proven', event.blockNumber);
+        await this.saveTag('proven', event.block);
         break;
       case 'chain-finalized':
-        await this.l2TipsStore.set('finalized', event.blockNumber);
-        for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.blockNumber })) {
+        await this.saveTag('finalized', event.block);
+        for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.block.number })) {
           await this.l2BlockHashesStore.delete(key);
         }
         break;
     }
   }
+
+  private async saveTag(name: L2BlockTag, block: L2BlockId) {
+    await this.l2TipsStore.set(name, block.number);
+    if (block.hash) {
+      await this.l2BlockHashesStore.set(block.number, block.hash);
+    }
+  }
 }

package/dest/interfaces/store_test_suite.d.ts

@@ -1,3 +0,0 @@
-import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
-export declare function describeAztecStore(testName: string, getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>): void;
-//# sourceMappingURL=store_test_suite.d.ts.map

package/dest/interfaces/store_test_suite.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"store_test_suite.d.ts","sourceRoot":"","sources":["../../src/interfaces/store_test_suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGlE,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACnE,wBAAwB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACzE,iBAAiB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,QA6CnE"}

package/dest/interfaces/store_test_suite.js

@@ -1,37 +0,0 @@
-import { expect } from 'chai';
-import { isSyncStore } from './utils.js';
-export function describeAztecStore(testName, getPersistentStore, getPersistentNoPathStore, getEphemeralStore) {
-    describe(testName, ()=>{
-        async function get(store, singleton) {
-            return isSyncStore(store) ? singleton.get() : await singleton.getAsync();
-        }
-        const itForks = async (store)=>{
-            const singleton = store.openSingleton('singleton');
-            await singleton.set('foo');
-            const forkedStore = await store.fork();
-            const forkedSingleton = forkedStore.openSingleton('singleton');
-            expect(await get(store, singleton)).to.equal('foo');
-            await forkedSingleton.set('bar');
-            expect(await get(store, singleton)).to.equal('foo');
-            expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
-            await forkedSingleton.delete();
-            expect(await get(store, singleton)).to.equal('foo');
-            await forkedStore.delete();
-        };
-        it('forks a persistent store', async ()=>{
-            const store = await getPersistentStore();
-            await itForks(store);
-            await store.delete();
-        });
-        it('forks a persistent store with no path', async ()=>{
-            const store = await getPersistentNoPathStore();
-            await itForks(store);
-            await store.delete();
-        });
-        it('forks an ephemeral store', async ()=>{
-            const store = await getEphemeralStore();
-            await itForks(store);
-            await store.delete();
-        });
-    });
-}

package/src/interfaces/store_test_suite.ts

@@ -1,56 +0,0 @@
-import { expect } from 'chai';
-
-import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
-import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
-import { isSyncStore } from './utils.js';
-
-export function describeAztecStore(
-  testName: string,
-  getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-  getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-  getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-) {
-  describe(testName, () => {
-    async function get(
-      store: AztecKVStore | AztecAsyncKVStore,
-      singleton: AztecSingleton<string> | AztecAsyncSingleton<string>,
-    ) {
-      return isSyncStore(store)
-        ? (singleton as AztecSingleton<string>).get()
-        : await (singleton as AztecAsyncSingleton<string>).getAsync();
-    }
-
-    const itForks = async (store: AztecKVStore | AztecAsyncKVStore) => {
-      const singleton = store.openSingleton<string>('singleton');
-      await singleton.set('foo');
-
-      const forkedStore = await store.fork();
-      const forkedSingleton = forkedStore.openSingleton<string>('singleton');
-      expect(await get(store, singleton)).to.equal('foo');
-      await forkedSingleton.set('bar');
-      expect(await get(store, singleton)).to.equal('foo');
-      expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
-      await forkedSingleton.delete();
-      expect(await get(store, singleton)).to.equal('foo');
-      await forkedStore.delete();
-    };
-
-    it('forks a persistent store', async () => {
-      const store = await getPersistentStore();
-      await itForks(store);
-      await store.delete();
-    });
-
-    it('forks a persistent store with no path', async () => {
-      const store = await getPersistentNoPathStore();
-      await itForks(store);
-      await store.delete();
-    });
-
-    it('forks an ephemeral store', async () => {
-      const store = await getEphemeralStore();
-      await itForks(store);
-      await store.delete();
-    });
-  });
-}