@aztec/kv-store 0.0.1-commit.e6bd8901 → 0.0.1-commit.ec7ac5448

package/src/interfaces/map_test_suite.ts

@@ -107,6 +107,33 @@ export function describeAztecMap(
       expect(await size()).to.equal(1);
     });
 
+    it('returns 0 for empty map size', async () => {
+      expect(await size()).to.equal(0);
+    });
+
+    it('calculates size correctly across multiple operations', async () => {
+      expect(await size()).to.equal(0);
+
+      // Add items
+      await map.set('a', 'value1');
+      await map.set('b', 'value2');
+      await map.set('c', 'value3');
+      expect(await size()).to.equal(3);
+
+      // Update existing (size should not change)
+      await map.set('b', 'updated');
+      expect(await size()).to.equal(3);
+
+      // Delete some
+      await map.delete('a');
+      expect(await size()).to.equal(2);
+
+      // Delete all
+      await map.delete('b');
+      await map.delete('c');
+      expect(await size()).to.equal(0);
+    });
+
     it('should be able to iterate over entries when there are no keys', async () => {
       expect(await entries()).to.deep.equal([]);
     });
@@ -145,10 +172,11 @@ export function describeAztecMap(
     for (const [name, data] of [
       ['chars', ['a', 'b', 'c', 'd']],
       ['numbers', [1, 2, 3, 4]],
-      // disabled because indexeddb sorts lexigographically
-      // ['negative numbers', [-4, -3, -2, -1]],
+      ['negative numbers', [-4, -3, -2, -1]],
       ['strings', ['aaa', 'bbb', 'ccc', 'ddd']],
       ['zero-based numbers', [0, 1, 2, 3]],
+      ['large numbers', [100, 999, 1000, 1001]],
+      ['mixed negative and positive', [-1000, -1, 1, 1000]],
     ]) {
       it(`supports range queries over ${name} keys`, async () => {
         const [a, b, c, d] = data;

package/src/interfaces/multi_map_test_suite.ts

@@ -104,6 +104,38 @@ export function describeAztecMultiMap(
       expect(await size()).to.equal(1);
     });
 
+    it('returns 0 for empty multimap size', async () => {
+      expect(await size()).to.equal(0);
+    });
+
+    it('calculates size correctly with multiple values per key', async () => {
+      expect(await size()).to.equal(0);
+
+      // Add multiple values for same key
+      await multiMap.set('key1', 'value1');
+      expect(await size()).to.equal(1);
+      await multiMap.set('key1', 'value2');
+      expect(await size()).to.equal(2);
+      await multiMap.set('key1', 'value3');
+      expect(await size()).to.equal(3);
+
+      // Add values for different key
+      await multiMap.set('key2', 'value4');
+      expect(await size()).to.equal(4);
+
+      // Delete one value from key1
+      await multiMap.deleteValue('key1', 'value2');
+      expect(await size()).to.equal(3);
+
+      // Delete entire key
+      await multiMap.delete('key1');
+      expect(await size()).to.equal(1);
+
+      // Delete last key
+      await multiMap.delete('key2');
+      expect(await size()).to.equal(0);
+    });
+
     it('should be able to iterate over entries when there are no keys', async () => {
       expect(await entries()).to.deep.equal([]);
     });

package/src/interfaces/utils.ts

@@ -14,6 +14,7 @@ export const mockLogger = {
   isLevelEnabled: (_level: string) => true,
   module: 'kv-store:mock-logger',
   createChild: () => mockLogger,
+  getBindings: () => ({}),
 };
 /* eslint-enable no-console */
 

package/src/lmdb/index.ts

@@ -1,14 +1,19 @@
 import { type Logger, createLogger } from '@aztec/foundation/log';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
 import { join } from 'path';
 
-import type { DataStoreConfig } from '../config.js';
-import { initStoreForRollup } from '../utils.js';
+import { initStoreForRollupAndSchemaVersion } from '../utils.js';
 import { AztecLmdbStore } from './store.js';
 
 export { AztecLmdbStore } from './store.js';
 
-export function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
+export function createStore(
+  name: string,
+  config: DataStoreConfig,
+  schemaVersion: number | undefined = undefined,
+  log: Logger = createLogger('kv-store'),
+) {
   let { dataDirectory } = config;
   if (typeof dataDirectory !== 'undefined') {
     dataDirectory = join(dataDirectory, name);
@@ -22,7 +27,7 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger =
 
   const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKb, false);
   if (config.l1Contracts?.rollupAddress) {
-    return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
+    return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts.rollupAddress, log);
   }
   return store;
 }

package/src/lmdb/store.ts

@@ -147,21 +147,25 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
   }
 
   /**
-   * Clears all entries in the store & sub DBs.
+   * Clears all entries in the store & sub DBs atomically within a single transaction.
    */
   async clear() {
-    await this.#data.clearAsync();
-    await this.#multiMapData.clearAsync();
-    await this.#rootDb.clearAsync();
+    await this.#rootDb.transaction(async () => {
+      await this.#data.clearAsync();
+      await this.#multiMapData.clearAsync();
+      await this.#rootDb.clearAsync();
+    });
   }
 
   /**
-   * Drops the database & sub DBs.
+   * Drops the database & sub DBs atomically within a single transaction.
    */
   async drop() {
-    await this.#data.drop();
-    await this.#multiMapData.drop();
-    await this.#rootDb.drop();
+    await this.#rootDb.transaction(async () => {
+      await this.#data.drop();
+      await this.#multiMapData.drop();
+      await this.#rootDb.drop();
+    });
   }
 
   /**

package/src/lmdb-v2/factory.ts

@@ -1,12 +1,12 @@
 import { EthAddress } from '@aztec/foundation/eth-address';
-import { type Logger, createLogger } from '@aztec/foundation/log';
-import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
+import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
+import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
+import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
 
 import { mkdir, mkdtemp, rm } from 'fs/promises';
 import { tmpdir } from 'os';
 import { join } from 'path';
 
-import type { DataStoreConfig } from '../config.js';
 import { AztecLMDBStoreV2 } from './store.js';
 
 const MAX_READERS = 16;
@@ -15,8 +15,9 @@ export async function createStore(
   name: string,
   schemaVersion: number,
   config: DataStoreConfig,
-  log: Logger = createLogger('kv-store:lmdb-v2:' + name),
+  bindings?: LoggerBindings,
 ): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
   const { dataDirectory, l1Contracts } = config;
 
   let store: AztecLMDBStoreV2;
@@ -33,7 +34,7 @@ export async function createStore(
       rollupAddress,
       dataDirectory: subDir,
       onOpen: dbDirectory =>
-        AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), log),
+        AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
     });
 
     log.info(
@@ -41,7 +42,7 @@ export async function createStore(
     );
     [store] = await versionManager.open();
   } else {
-    store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, log);
+    store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
   }
 
   return store;
@@ -52,8 +53,9 @@ export async function openTmpStore(
   ephemeral: boolean = true,
   dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
   maxReaders = MAX_READERS,
-  log: Logger = createLogger('kv-store:lmdb-v2:' + name),
+  bindings?: LoggerBindings,
 ): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
   const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
   log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
 
@@ -73,17 +75,18 @@ export async function openTmpStore(
 
   // For temporary stores, we don't need to worry about versioning
   // as they are ephemeral and get cleaned up after use
-  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, log);
+  return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings);
 }
 
 export async function openStoreAt(
   dataDir: string,
   dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
   maxReaders = MAX_READERS,
-  log: Logger = createLogger('kv-store:lmdb-v2'),
+  bindings?: LoggerBindings,
 ): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2', bindings);
   log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
-  return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log);
+  return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
 }
 
 export async function openVersionedStoreAt(
@@ -92,14 +95,15 @@ export async function openVersionedStoreAt(
   rollupAddress: EthAddress,
   dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
   maxReaders = MAX_READERS,
-  log: Logger = createLogger('kv-store:lmdb-v2'),
+  bindings?: LoggerBindings,
 ): Promise<AztecLMDBStoreV2> {
+  const log = createLogger('kv-store:lmdb-v2', bindings);
   log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
   const [store] = await new DatabaseVersionManager({
     schemaVersion,
     rollupAddress,
     dataDirectory,
-    onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log),
+    onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
   }).open();
   return store;
 }

package/src/lmdb-v2/store.ts

@@ -1,4 +1,4 @@
-import { type Logger, createLogger } from '@aztec/foundation/log';
+import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
 import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
 import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
 
@@ -75,8 +75,9 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
     dbMapSizeKb: number = 10 * 1024 * 1024,
     maxReaders: number = 16,
     cleanup?: () => Promise<void>,
-    log = createLogger('kv-store:lmdb-v2'),
+    bindings?: LoggerBindings,
   ) {
+    const log = createLogger('kv-store:lmdb-v2', bindings);
     const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup);
     await db.start();
     return db;

package/src/utils.ts

@@ -1,40 +1,98 @@
-import type { EthAddress } from '@aztec/foundation/eth-address';
+import { EthAddress } from '@aztec/foundation/eth-address';
 import type { Logger } from '@aztec/foundation/log';
+import { DatabaseVersion } from '@aztec/stdlib/database-version/version';
 
 import type { AztecAsyncSingleton, AztecSingleton } from './interfaces/singleton.js';
 import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
 import { isSyncStore } from './interfaces/utils.js';
 
 /**
- * Clears the store if the rollup address does not match the one stored in the database.
- * This is to prevent data from being accidentally shared between different rollup instances.
+ * Clears the store if the schema version or rollup address does not match the one stored in the database.
+ * Also clears if migrating from an older store format that didn't track schema version.
+ * This is to prevent data from being accidentally mixed up between different rollup instances or schema versions.
  * @param store - The store to check
+ * @param targetSchemaVersion - The current schema version
  * @param rollupAddress - The ETH address of the rollup contract
- * @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
+ * @param log - Optional logger
+ * @returns The store (cleared if necessary)
  */
-export async function initStoreForRollup<T extends AztecKVStore | AztecAsyncKVStore>(
+export async function initStoreForRollupAndSchemaVersion<T extends AztecKVStore | AztecAsyncKVStore>(
   store: T,
-  rollupAddress: EthAddress,
+  schemaVersion: number | undefined,
+  rollupAddress: EthAddress | undefined,
   log?: Logger,
 ): Promise<T> {
-  if (!rollupAddress) {
-    throw new Error('Rollup address is required');
-  }
-  const rollupAddressValue = store.openSingleton<ReturnType<EthAddress['toString']>>('rollupAddress');
-  const rollupAddressString = rollupAddress.toString();
-  const storedRollupAddressString = isSyncStore(store)
-    ? (rollupAddressValue as AztecSingleton<ReturnType<EthAddress['toString']>>).get()
-    : await (rollupAddressValue as AztecAsyncSingleton<ReturnType<EthAddress['toString']>>).getAsync();
-
-  if (typeof storedRollupAddressString !== 'undefined' && storedRollupAddressString !== rollupAddressString) {
-    log?.warn(`Rollup address mismatch. Clearing entire database...`, {
-      expected: rollupAddressString,
-      found: storedRollupAddressString,
-    });
+  const targetSchemaVersion = schemaVersion ?? 0;
+  const targetRollupAddress = rollupAddress ?? EthAddress.ZERO;
+  const targetDatabaseVersion = new DatabaseVersion(targetSchemaVersion, targetRollupAddress);
+
+  // DB version: database schema version + rollup address combined)
+  const dbVersion = store.openSingleton<string>('dbVersion');
+
+  const storedDatabaseVersion = isSyncStore(store)
+    ? (dbVersion as AztecSingleton<string>).get()
+    : await (dbVersion as AztecAsyncSingleton<string>).getAsync();
+
+  // Check if this is an old format store (has rollupAddress singleton but no dbVersion)
+  const oldRollupSingleton = store.openSingleton<string>('rollupAddress');
+  const hasOldFormat = isSyncStore(store)
+    ? !storedDatabaseVersion && !!(oldRollupSingleton as AztecSingleton<string>).get()
+    : !storedDatabaseVersion && !!(await (oldRollupSingleton as AztecAsyncSingleton<string>).getAsync());
 
+  if (
+    hasOldFormat ||
+    doesStoreNeedToBeCleared(
+      targetDatabaseVersion,
+      storedDatabaseVersion,
+      targetSchemaVersion,
+      targetRollupAddress,
+      log,
+    )
+  ) {
+    if (hasOldFormat) {
+      log?.warn('Detected old store format without dbVersion, clearing database');
+    }
     await store.clear();
   }
 
-  await rollupAddressValue.set(rollupAddressString);
+  await dbVersion.set(targetDatabaseVersion.toBuffer().toString('utf-8'));
+
   return store;
 }
+
+function doesStoreNeedToBeCleared(
+  targetDatabaseVersion: DatabaseVersion,
+  storedDatabaseVersion: string | undefined,
+  targetSchemaVersion: number,
+  targetRollupAddress: EthAddress,
+  log?: Logger,
+) {
+  if (storedDatabaseVersion) {
+    try {
+      const storedVersion = DatabaseVersion.fromBuffer(Buffer.from(storedDatabaseVersion, 'utf-8'));
+      const cmp = storedVersion.cmp(targetDatabaseVersion);
+
+      if (cmp === undefined) {
+        log?.warn('Rollup address changed, clearing database', {
+          stored: storedVersion.rollupAddress.toString(),
+          current: targetRollupAddress.toString(),
+        });
+        return true;
+      }
+
+      if (cmp !== 0) {
+        log?.warn('Schema version changed, clearing database', {
+          stored: storedVersion.schemaVersion,
+          current: targetSchemaVersion,
+        });
+
+        return true;
+      }
+    } catch (err) {
+      log?.warn('Failed to parse stored version, clearing database', { err });
+      return true;
+    }
+  }
+
+  return false;
+}

package/dest/config.d.ts

@@ -1,17 +0,0 @@
-import { type ConfigMappingsType } from '@aztec/foundation/config';
-import type { EthAddress } from '@aztec/foundation/eth-address';
-export type DataStoreConfig = {
-    dataDirectory: string | undefined;
-    dataStoreMapSizeKb: number;
-    l1Contracts?: {
-        rollupAddress: EthAddress;
-    };
-};
-export declare const dataConfigMappings: ConfigMappingsType<DataStoreConfig>;
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */
-export declare function getDataConfigFromEnv(): DataStoreConfig;
-//# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoiY29uZmlnLmQudHMiLCJzb3VyY2VSb290IjoiIiwic291cmNlcyI6WyIuLi9zcmMvY29uZmlnLnRzIl0sIm5hbWVzIjpbXSwibWFwcGluZ3MiOiJBQUNBLE9BQU8sRUFBRSxLQUFLLGtCQUFrQixFQUE2QyxNQUFNLDBCQUEwQixDQUFDO0FBQzlHLE9BQU8sS0FBSyxFQUFFLFVBQVUsRUFBRSxNQUFNLCtCQUErQixDQUFDO0FBRWhFLE1BQU0sTUFBTSxlQUFlLEdBQUc7SUFDNUIsYUFBYSxFQUFFLE1BQU0sR0FBRyxTQUFTLENBQUM7SUFDbEMsa0JBQWtCLEVBQUUsTUFBTSxDQUFDO0lBQzNCLFdBQVcsQ0FBQyxFQUFFO1FBQUUsYUFBYSxFQUFFLFVBQVUsQ0FBQTtLQUFFLENBQUM7Q0FDN0MsQ0FBQztBQUVGLGVBQU8sTUFBTSxrQkFBa0IsRUFBRSxrQkFBa0IsQ0FBQyxlQUFlLENBZ0JsRSxDQUFDO0FBRUY7Ozs7R0FJRztBQUNILHdCQUFnQixvQkFBb0IsSUFBSSxlQUFlLENBRXREIn0=

package/dest/config.d.ts.map

@@ -1 +0,0 @@
-{"version":3,"file":"config.d.ts","sourceRoot":"","sources":["../src/config.ts"],"names":[],"mappings":"AACA,OAAO,EAAE,KAAK,kBAAkB,EAA6C,MAAM,0BAA0B,CAAC;AAC9G,OAAO,KAAK,EAAE,UAAU,EAAE,MAAM,+BAA+B,CAAC;AAEhE,MAAM,MAAM,eAAe,GAAG;IAC5B,aAAa,EAAE,MAAM,GAAG,SAAS,CAAC;IAClC,kBAAkB,EAAE,MAAM,CAAC;IAC3B,WAAW,CAAC,EAAE;QAAE,aAAa,EAAE,UAAU,CAAA;KAAE,CAAC;CAC7C,CAAC;AAEF,eAAO,MAAM,kBAAkB,EAAE,kBAAkB,CAAC,eAAe,CAgBlE,CAAC;AAEF;;;;GAIG;AACH,wBAAgB,oBAAoB,IAAI,eAAe,CAEtD"}

package/dest/config.js

@@ -1,26 +0,0 @@
-import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
-import { getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
-export const dataConfigMappings = {
-    dataDirectory: {
-        env: 'DATA_DIRECTORY',
-        description: 'Optional dir to store data. If omitted will store in memory.'
-    },
-    dataStoreMapSizeKb: {
-        env: 'DATA_STORE_MAP_SIZE_KB',
-        description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
-        ...numberConfigHelper(128 * 1_024 * 1_024)
-    },
-    l1Contracts: {
-        description: 'The deployed L1 contract addresses',
-        nested: {
-            rollupAddress: l1ContractAddressesMapping.rollupAddress
-        }
-    }
-};
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */ export function getDataConfigFromEnv() {
-    return getConfigFromMappings(dataConfigMappings);
-}

package/src/config.ts

@@ -1,36 +0,0 @@
-import { l1ContractAddressesMapping } from '@aztec/ethereum/l1-contract-addresses';
-import { type ConfigMappingsType, getConfigFromMappings, numberConfigHelper } from '@aztec/foundation/config';
-import type { EthAddress } from '@aztec/foundation/eth-address';
-
-export type DataStoreConfig = {
-  dataDirectory: string | undefined;
-  dataStoreMapSizeKb: number;
-  l1Contracts?: { rollupAddress: EthAddress };
-};
-
-export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
-  dataDirectory: {
-    env: 'DATA_DIRECTORY',
-    description: 'Optional dir to store data. If omitted will store in memory.',
-  },
-  dataStoreMapSizeKb: {
-    env: 'DATA_STORE_MAP_SIZE_KB',
-    description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
-    ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
-  },
-  l1Contracts: {
-    description: 'The deployed L1 contract addresses',
-    nested: {
-      rollupAddress: l1ContractAddressesMapping.rollupAddress,
-    },
-  },
-};
-
-/**
- * Returns the archiver configuration from the environment variables.
- * Note: If an environment variable is not set, the default value is used.
- * @returns The archiver configuration.
- */
-export function getDataConfigFromEnv(): DataStoreConfig {
-  return getConfigFromMappings<DataStoreConfig>(dataConfigMappings);
-}