@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.1142ef1

package/src/lmdb-v2/utils.ts

@@ -2,124 +2,6 @@ import { MAXIMUM_KEY, fromBufferKey, toBufferKey } from 'ordered-binary';
 
 import type { Key } from '../interfaces/common.js';
 
-type Cmp<T> = (a: T, b: T) => -1 | 0 | 1;
-
-export function dedupeSortedArray<T>(arr: T[], cmp: Cmp<T>): void {
-  for (let i = 0; i < arr.length; i++) {
-    let j = i + 1;
-    for (; j < arr.length; j++) {
-      const res = cmp(arr[i], arr[j]);
-      if (res === 0) {
-        continue;
-      } else if (res < 0) {
-        break;
-      } else {
-        throw new Error('Array not sorted');
-      }
-    }
-
-    if (j - i > 1) {
-      arr.splice(i + 1, j - i - 1);
-    }
-  }
-}
-
-export function insertIntoSortedArray<T>(arr: T[], item: T, cmp: (a: T, b: T) => number): void {
-  let left = 0;
-  let right = arr.length;
-
-  while (left < right) {
-    const mid = (left + right) >> 1;
-    const comparison = cmp(arr[mid], item);
-
-    if (comparison < 0) {
-      left = mid + 1;
-    } else {
-      right = mid;
-    }
-  }
-
-  arr.splice(left, 0, item);
-}
-
-export function findIndexInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): number {
-  let start = 0;
-  let end = values.length - 1;
-
-  while (start <= end) {
-    const mid = start + (((end - start) / 2) | 0);
-    const res = cmp(values[mid], needle);
-    if (res === 0) {
-      return mid;
-    } else if (res > 0) {
-      end = mid - 1;
-    } else {
-      start = mid + 1;
-    }
-  }
-
-  return -1;
-}
-
-export function findInSortedArray<T, N>(values: T[], needle: N, cmp: (a: T, b: N) => number): T | undefined {
-  const idx = findIndexInSortedArray(values, needle, cmp);
-  return idx > -1 ? values[idx] : undefined;
-}
-
-export function removeAnyOf<T, N>(arr: T[], vals: N[], cmp: (a: T, b: N) => -1 | 0 | 1): void {
-  let writeIdx = 0;
-  let readIdx = 0;
-  let valIdx = 0;
-
-  while (readIdx < arr.length && valIdx < vals.length) {
-    const comparison = cmp(arr[readIdx], vals[valIdx]);
-
-    if (comparison < 0) {
-      arr[writeIdx++] = arr[readIdx++];
-    } else if (comparison > 0) {
-      valIdx++;
-    } else {
-      readIdx++;
-    }
-  }
-
-  while (readIdx < arr.length) {
-    arr[writeIdx++] = arr[readIdx++];
-  }
-
-  arr.length = writeIdx;
-}
-
-export function removeFromSortedArray<T, N>(arr: T[], val: N, cmp: (a: T, b: N) => -1 | 0 | 1) {
-  const idx = findIndexInSortedArray(arr, val, cmp);
-  if (idx > -1) {
-    arr.splice(idx, 1);
-  }
-}
-
-export function merge<T>(arr: T[], toInsert: T[], cmp: (a: T, b: T) => -1 | 0 | 1): void {
-  const result = new Array<T>(arr.length + toInsert.length);
-  let i = 0,
-    j = 0,
-    k = 0;
-
-  while (i < arr.length && j < toInsert.length) {
-    result[k++] = cmp(arr[i], toInsert[j]) <= 0 ? arr[i++] : toInsert[j++];
-  }
-
-  while (i < arr.length) {
-    result[k++] = arr[i++];
-  }
-  while (j < toInsert.length) {
-    result[k++] = toInsert[j++];
-  }
-
-  for (i = 0; i < result.length; i++) {
-    arr[i] = result[i];
-  }
-  arr.length = result.length;
-}
-
 export function keyCmp(a: [Uint8Array, Uint8Array[] | null], b: [Uint8Array, Uint8Array[] | null]): -1 | 0 | 1 {
   return Buffer.compare(a[0], b[0]);
 }

package/src/lmdb-v2/write_transaction.ts

@@ -1,16 +1,16 @@
-import { type Batch, Database, LMDBMessageType } from './message.js';
-import { ReadTransaction } from './read_transaction.js';
 import {
   dedupeSortedArray,
   findInSortedArray,
   findIndexInSortedArray,
   insertIntoSortedArray,
-  keyCmp,
   merge,
   removeAnyOf,
   removeFromSortedArray,
-  singleKeyCmp,
-} from './utils.js';
+} from '@aztec/foundation/array';
+
+import { type Batch, Database, LMDBMessageType } from './message.js';
+import { ReadTransaction } from './read_transaction.js';
+import { keyCmp, singleKeyCmp } from './utils.js';
 
 export class WriteTransaction extends ReadTransaction {
   // exposed for tests
@@ -44,7 +44,7 @@ export class WriteTransaction extends ReadTransaction {
   remove(key: Uint8Array): Promise<void> {
     const removeEntryIndex = findIndexInSortedArray(this.dataBatch.removeEntries, key, singleKeyCmp);
     if (removeEntryIndex === -1) {
-      this.dataBatch.removeEntries.push([key, null]);
+      insertIntoSortedArray(this.dataBatch.removeEntries, [key, null], keyCmp);
     }
 
     const addEntryIndex = findIndexInSortedArray(this.dataBatch.addEntries, key, singleKeyCmp);
@@ -163,7 +163,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterate(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array]> {
@@ -181,7 +181,7 @@ export class WriteTransaction extends ReadTransaction {
 
   public override async *iterateIndex(
     startKey: Uint8Array,
-    endKey?: Uint8Array | undefined,
+    endKey?: Uint8Array,
     reverse?: boolean,
     limit?: number,
   ): AsyncIterable<[Uint8Array, Uint8Array[]]> {
@@ -218,8 +218,9 @@ export class WriteTransaction extends ReadTransaction {
   ): AsyncIterable<[Uint8Array, T]> {
     this.assertIsOpen();
 
-    // make a copy of this in case we're running in reverse
+    // Snapshot both add and remove entries at the start of iteration to ensure consistency
     const uncommittedEntries = [...batch.addEntries];
+    const removeEntries = [...batch.removeEntries];
     // used to check we're in the right order when comparing between a key and uncommittedEntries
     let cmpDirection = -1;
     if (reverse) {
@@ -262,7 +263,7 @@ export class WriteTransaction extends ReadTransaction {
         break;
       }
 
-      const toRemove = findInSortedArray(batch.removeEntries, key, singleKeyCmp);
+      const toRemove = findInSortedArray(removeEntries, key, singleKeyCmp);
 
       // at this point we've either exhausted all uncommitted entries,
       // we reached a key strictly greater/smaller than `key`

package/src/stores/index.ts

@@ -1 +1,3 @@
 export * from './l2_tips_store.js';
+
+export { L2TipsMemoryStore, type L2TipsStore } from '@aztec/stdlib/block';

package/src/stores/l2_tips_store.ts

@@ -1,70 +1,88 @@
-import type {
-  L2BlockId,
-  L2BlockStreamEvent,
-  L2BlockStreamEventHandler,
-  L2BlockStreamLocalDataProvider,
-  L2BlockTag,
-  L2Tips,
-} from '@aztec/stdlib/block';
+import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
+import { type L2BlockTag, L2TipsStoreBase } from '@aztec/stdlib/block';
+import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
 
 import type { AztecAsyncMap } from '../interfaces/map.js';
 import type { AztecAsyncKVStore } from '../interfaces/store.js';
 
-/** Stores currently synced L2 tips and unfinalized block hashes. */
-export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
-  private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, number>;
-  private readonly l2BlockHashesStore: AztecAsyncMap<number, string>;
+/**
+ * Persistent implementation of L2 tips store backed by a KV store.
+ * Used by nodes that need to persist chain state across restarts.
+ */
+export class L2TipsKVStore extends L2TipsStoreBase {
+  private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, BlockNumber>;
+  private readonly l2BlockHashesStore: AztecAsyncMap<BlockNumber, string>;
+  private readonly l2BlockNumberToCheckpointNumberStore: AztecAsyncMap<BlockNumber, CheckpointNumber>;
+  private readonly l2CheckpointStore: AztecAsyncMap<CheckpointNumber, Buffer>;
 
-  constructor(store: AztecAsyncKVStore, namespace: string) {
+  constructor(
+    private store: AztecAsyncKVStore,
+    namespace: string,
+  ) {
+    super();
     this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_'));
     this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_'));
+    this.l2BlockNumberToCheckpointNumberStore = store.openMap(
+      [namespace, 'l2_block_number_to_checkpoint_number'].join('_'),
+    );
+    this.l2CheckpointStore = store.openMap([namespace, 'l2_checkpoint_store'].join('_'));
   }
 
-  public getL2BlockHash(number: number): Promise<string | undefined> {
-    return this.l2BlockHashesStore.getAsync(number);
+  protected getTip(tag: L2BlockTag): Promise<BlockNumber | undefined> {
+    return this.l2TipsStore.getAsync(tag);
   }
 
-  public async getL2Tips(): Promise<L2Tips> {
-    return {
-      latest: await this.getL2Tip('latest'),
-      finalized: await this.getL2Tip('finalized'),
-      proven: await this.getL2Tip('proven'),
-    };
+  protected setTip(tag: L2BlockTag, blockNumber: BlockNumber): Promise<void> {
+    return this.l2TipsStore.set(tag, blockNumber);
   }
 
-  private async getL2Tip(tag: L2BlockTag): Promise<L2BlockId> {
-    const blockNumber = await this.l2TipsStore.getAsync(tag);
-    if (blockNumber === undefined || blockNumber === 0) {
-      return { number: 0, hash: undefined };
+  protected getStoredBlockHash(blockNumber: BlockNumber): Promise<string | undefined> {
+    return this.l2BlockHashesStore.getAsync(blockNumber);
+  }
+
+  protected setBlockHash(blockNumber: BlockNumber, hash: string): Promise<void> {
+    return this.l2BlockHashesStore.set(blockNumber, hash);
+  }
+
+  protected async deleteBlockHashesBefore(blockNumber: BlockNumber): Promise<void> {
+    for await (const key of this.l2BlockHashesStore.keysAsync({ end: blockNumber })) {
+      await this.l2BlockHashesStore.delete(key);
+    }
+  }
+
+  protected getCheckpointNumberForBlock(blockNumber: BlockNumber): Promise<CheckpointNumber | undefined> {
+    return this.l2BlockNumberToCheckpointNumberStore.getAsync(blockNumber);
+  }
+
+  protected setCheckpointNumberForBlock(blockNumber: BlockNumber, checkpointNumber: CheckpointNumber): Promise<void> {
+    return this.l2BlockNumberToCheckpointNumberStore.set(blockNumber, checkpointNumber);
+  }
+
+  protected async deleteBlockToCheckpointBefore(blockNumber: BlockNumber): Promise<void> {
+    for await (const key of this.l2BlockNumberToCheckpointNumberStore.keysAsync({ end: blockNumber })) {
+      await this.l2BlockNumberToCheckpointNumberStore.delete(key);
     }
-    const blockHash = await this.l2BlockHashesStore.getAsync(blockNumber);
-    if (!blockHash) {
-      throw new Error(`Block hash not found for block number ${blockNumber}`);
+  }
+
+  protected async getCheckpoint(checkpointNumber: CheckpointNumber): Promise<PublishedCheckpoint | undefined> {
+    const buffer = await this.l2CheckpointStore.getAsync(checkpointNumber);
+    if (!buffer) {
+      return undefined;
     }
+    return PublishedCheckpoint.fromBuffer(buffer);
+  }
 
-    return { number: blockNumber, hash: blockHash };
+  protected saveCheckpointData(checkpoint: PublishedCheckpoint): Promise<void> {
+    return this.l2CheckpointStore.set(checkpoint.checkpoint.number, checkpoint.toBuffer());
   }
 
-  public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
-    switch (event.type) {
-      case 'blocks-added':
-        for (const block of event.blocks) {
-          await this.l2BlockHashesStore.set(block.number, (await block.header.hash()).toString());
-        }
-        await this.l2TipsStore.set('latest', event.blocks.at(-1)!.number);
-        break;
-      case 'chain-pruned':
-        await this.l2TipsStore.set('latest', event.blockNumber);
-        break;
-      case 'chain-proven':
-        await this.l2TipsStore.set('proven', event.blockNumber);
-        break;
-      case 'chain-finalized':
-        await this.l2TipsStore.set('finalized', event.blockNumber);
-        for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.blockNumber })) {
-          await this.l2BlockHashesStore.delete(key);
-        }
-        break;
+  protected async deleteCheckpointsBefore(checkpointNumber: CheckpointNumber): Promise<void> {
+    for await (const key of this.l2CheckpointStore.keysAsync({ end: checkpointNumber })) {
+      await this.l2CheckpointStore.delete(key);
     }
   }
+
+  protected runInTransaction<T>(fn: () => Promise<T>): Promise<T> {
+    return this.store.transactionAsync(fn);
+  }
 }

package/dest/interfaces/store_test_suite.d.ts

@@ -1,3 +0,0 @@
-import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
-export declare function describeAztecStore(testName: string, getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>): void;
-//# sourceMappingURL=store_test_suite.d.ts.map

package/dest/interfaces/store_test_suite.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"store_test_suite.d.ts","sourceRoot":"","sources":["../../src/interfaces/store_test_suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGlE,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACnE,wBAAwB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACzE,iBAAiB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,QA6CnE"}

package/dest/interfaces/store_test_suite.js

@@ -1,37 +0,0 @@
-import { expect } from 'chai';
-import { isSyncStore } from './utils.js';
-export function describeAztecStore(testName, getPersistentStore, getPersistentNoPathStore, getEphemeralStore) {
-    describe(testName, ()=>{
-        async function get(store, singleton) {
-            return isSyncStore(store) ? singleton.get() : await singleton.getAsync();
-        }
-        const itForks = async (store)=>{
-            const singleton = store.openSingleton('singleton');
-            await singleton.set('foo');
-            const forkedStore = await store.fork();
-            const forkedSingleton = forkedStore.openSingleton('singleton');
-            expect(await get(store, singleton)).to.equal('foo');
-            await forkedSingleton.set('bar');
-            expect(await get(store, singleton)).to.equal('foo');
-            expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
-            await forkedSingleton.delete();
-            expect(await get(store, singleton)).to.equal('foo');
-            await forkedStore.delete();
-        };
-        it('forks a persistent store', async ()=>{
-            const store = await getPersistentStore();
-            await itForks(store);
-            await store.delete();
-        });
-        it('forks a persistent store with no path', async ()=>{
-            const store = await getPersistentNoPathStore();
-            await itForks(store);
-            await store.delete();
-        });
-        it('forks an ephemeral store', async ()=>{
-            const store = await getEphemeralStore();
-            await itForks(store);
-            await store.delete();
-        });
-    });
-}

package/src/interfaces/store_test_suite.ts

@@ -1,56 +0,0 @@
-import { expect } from 'chai';
-
-import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
-import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
-import { isSyncStore } from './utils.js';
-
-export function describeAztecStore(
-  testName: string,
-  getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-  getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-  getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
-) {
-  describe(testName, () => {
-    async function get(
-      store: AztecKVStore | AztecAsyncKVStore,
-      singleton: AztecSingleton<string> | AztecAsyncSingleton<string>,
-    ) {
-      return isSyncStore(store)
-        ? (singleton as AztecSingleton<string>).get()
-        : await (singleton as AztecAsyncSingleton<string>).getAsync();
-    }
-
-    const itForks = async (store: AztecKVStore | AztecAsyncKVStore) => {
-      const singleton = store.openSingleton<string>('singleton');
-      await singleton.set('foo');
-
-      const forkedStore = await store.fork();
-      const forkedSingleton = forkedStore.openSingleton<string>('singleton');
-      expect(await get(store, singleton)).to.equal('foo');
-      await forkedSingleton.set('bar');
-      expect(await get(store, singleton)).to.equal('foo');
-      expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
-      await forkedSingleton.delete();
-      expect(await get(store, singleton)).to.equal('foo');
-      await forkedStore.delete();
-    };
-
-    it('forks a persistent store', async () => {
-      const store = await getPersistentStore();
-      await itForks(store);
-      await store.delete();
-    });
-
-    it('forks a persistent store with no path', async () => {
-      const store = await getPersistentNoPathStore();
-      await itForks(store);
-      await store.delete();
-    });
-
-    it('forks an ephemeral store', async () => {
-      const store = await getEphemeralStore();
-      await itForks(store);
-      await store.delete();
-    });
-  });
-}