@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.5daedc8

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (150) hide show
  1. package/dest/config.d.ts +2 -2
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +5 -3
  4. package/dest/indexeddb/array.d.ts +3 -2
  5. package/dest/indexeddb/array.d.ts.map +1 -1
  6. package/dest/indexeddb/array.js +3 -0
  7. package/dest/indexeddb/index.d.ts +1 -1
  8. package/dest/indexeddb/index.js +1 -1
  9. package/dest/indexeddb/map.d.ts +12 -6
  10. package/dest/indexeddb/map.d.ts.map +1 -1
  11. package/dest/indexeddb/map.js +38 -60
  12. package/dest/indexeddb/multi_map.d.ts +13 -0
  13. package/dest/indexeddb/multi_map.d.ts.map +1 -0
  14. package/dest/indexeddb/multi_map.js +92 -0
  15. package/dest/indexeddb/set.d.ts +1 -1
  16. package/dest/indexeddb/set.d.ts.map +1 -1
  17. package/dest/indexeddb/singleton.d.ts +3 -2
  18. package/dest/indexeddb/singleton.d.ts.map +1 -1
  19. package/dest/indexeddb/singleton.js +3 -1
  20. package/dest/indexeddb/store.d.ts +13 -14
  21. package/dest/indexeddb/store.d.ts.map +1 -1
  22. package/dest/indexeddb/store.js +45 -42
  23. package/dest/interfaces/array.d.ts +5 -4
  24. package/dest/interfaces/array.d.ts.map +1 -1
  25. package/dest/interfaces/array.js +1 -3
  26. package/dest/interfaces/array_test_suite.d.ts +1 -1
  27. package/dest/interfaces/common.d.ts +11 -9
  28. package/dest/interfaces/common.d.ts.map +1 -1
  29. package/dest/interfaces/common.js +8 -3
  30. package/dest/interfaces/counter.d.ts +1 -1
  31. package/dest/interfaces/index.d.ts +4 -2
  32. package/dest/interfaces/index.d.ts.map +1 -1
  33. package/dest/interfaces/index.js +2 -0
  34. package/dest/interfaces/map.d.ts +21 -49
  35. package/dest/interfaces/map.d.ts.map +1 -1
  36. package/dest/interfaces/map.js +1 -1
  37. package/dest/interfaces/map_test_suite.d.ts +1 -1
  38. package/dest/interfaces/map_test_suite.d.ts.map +1 -1
  39. package/dest/interfaces/map_test_suite.js +135 -70
  40. package/dest/interfaces/multi_map.d.ts +41 -0
  41. package/dest/interfaces/multi_map.d.ts.map +1 -0
  42. package/dest/interfaces/multi_map.js +3 -0
  43. package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
  44. package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
  45. package/dest/interfaces/multi_map_test_suite.js +339 -0
  46. package/dest/interfaces/set.d.ts +1 -1
  47. package/dest/interfaces/set_test_suite.d.ts +1 -1
  48. package/dest/interfaces/singleton.d.ts +1 -1
  49. package/dest/interfaces/singleton_test_suite.d.ts +1 -1
  50. package/dest/interfaces/store.d.ts +18 -43
  51. package/dest/interfaces/store.d.ts.map +1 -1
  52. package/dest/interfaces/utils.d.ts +2 -1
  53. package/dest/interfaces/utils.d.ts.map +1 -1
  54. package/dest/interfaces/utils.js +2 -1
  55. package/dest/lmdb/array.d.ts +3 -2
  56. package/dest/lmdb/array.d.ts.map +1 -1
  57. package/dest/lmdb/counter.d.ts +1 -1
  58. package/dest/lmdb/counter.d.ts.map +1 -1
  59. package/dest/lmdb/index.d.ts +1 -1
  60. package/dest/lmdb/index.js +2 -2
  61. package/dest/lmdb/map.d.ts +11 -23
  62. package/dest/lmdb/map.d.ts.map +1 -1
  63. package/dest/lmdb/map.js +15 -81
  64. package/dest/lmdb/multi_map.d.ts +13 -0
  65. package/dest/lmdb/multi_map.d.ts.map +1 -0
  66. package/dest/lmdb/multi_map.js +44 -0
  67. package/dest/lmdb/set.d.ts +1 -1
  68. package/dest/lmdb/set.d.ts.map +1 -1
  69. package/dest/lmdb/singleton.d.ts +1 -1
  70. package/dest/lmdb/singleton.d.ts.map +1 -1
  71. package/dest/lmdb/store.d.ts +9 -25
  72. package/dest/lmdb/store.d.ts.map +1 -1
  73. package/dest/lmdb/store.js +11 -31
  74. package/dest/lmdb-v2/array.d.ts +3 -2
  75. package/dest/lmdb-v2/array.d.ts.map +1 -1
  76. package/dest/lmdb-v2/array.js +1 -0
  77. package/dest/lmdb-v2/factory.d.ts +4 -7
  78. package/dest/lmdb-v2/factory.d.ts.map +1 -1
  79. package/dest/lmdb-v2/factory.js +16 -6
  80. package/dest/lmdb-v2/index.d.ts +1 -1
  81. package/dest/lmdb-v2/map.d.ts +11 -44
  82. package/dest/lmdb-v2/map.d.ts.map +1 -1
  83. package/dest/lmdb-v2/map.js +17 -103
  84. package/dest/lmdb-v2/message.d.ts +24 -5
  85. package/dest/lmdb-v2/message.d.ts.map +1 -1
  86. package/dest/lmdb-v2/message.js +6 -4
  87. package/dest/lmdb-v2/multi_map.d.ts +52 -0
  88. package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
  89. package/dest/lmdb-v2/multi_map.js +124 -0
  90. package/dest/lmdb-v2/read_transaction.d.ts +3 -1
  91. package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
  92. package/dest/lmdb-v2/read_transaction.js +34 -0
  93. package/dest/lmdb-v2/set.d.ts +15 -0
  94. package/dest/lmdb-v2/set.d.ts.map +1 -0
  95. package/dest/lmdb-v2/set.js +23 -0
  96. package/dest/lmdb-v2/singleton.d.ts +1 -1
  97. package/dest/lmdb-v2/singleton.d.ts.map +1 -1
  98. package/dest/lmdb-v2/singleton.js +1 -0
  99. package/dest/lmdb-v2/store.d.ts +10 -9
  100. package/dest/lmdb-v2/store.d.ts.map +1 -1
  101. package/dest/lmdb-v2/store.js +19 -7
  102. package/dest/lmdb-v2/utils.d.ts +3 -5
  103. package/dest/lmdb-v2/utils.d.ts.map +1 -1
  104. package/dest/lmdb-v2/write_transaction.d.ts +3 -5
  105. package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
  106. package/dest/stores/index.d.ts +2 -1
  107. package/dest/stores/index.d.ts.map +1 -1
  108. package/dest/stores/index.js +1 -0
  109. package/dest/stores/l2_tips_store.d.ts +3 -2
  110. package/dest/stores/l2_tips_store.d.ts.map +1 -1
  111. package/dest/stores/l2_tips_store.js +18 -9
  112. package/dest/utils.d.ts +1 -1
  113. package/package.json +21 -16
  114. package/src/config.ts +6 -4
  115. package/src/indexeddb/array.ts +5 -1
  116. package/src/indexeddb/index.ts +2 -2
  117. package/src/indexeddb/map.ts +35 -53
  118. package/src/indexeddb/multi_map.ts +91 -0
  119. package/src/indexeddb/singleton.ts +4 -1
  120. package/src/indexeddb/store.ts +66 -56
  121. package/src/interfaces/array.ts +5 -3
  122. package/src/interfaces/common.ts +20 -9
  123. package/src/interfaces/index.ts +3 -1
  124. package/src/interfaces/map.ts +19 -53
  125. package/src/interfaces/map_test_suite.ts +73 -44
  126. package/src/interfaces/multi_map.ts +45 -0
  127. package/src/interfaces/multi_map_test_suite.ts +342 -0
  128. package/src/interfaces/store.ts +18 -53
  129. package/src/interfaces/utils.ts +1 -0
  130. package/src/lmdb/array.ts +2 -1
  131. package/src/lmdb/index.ts +3 -3
  132. package/src/lmdb/map.ts +23 -94
  133. package/src/lmdb/multi_map.ts +51 -0
  134. package/src/lmdb/store.ts +23 -47
  135. package/src/lmdb-v2/array.ts +7 -2
  136. package/src/lmdb-v2/factory.ts +17 -10
  137. package/src/lmdb-v2/map.ts +29 -126
  138. package/src/lmdb-v2/message.ts +23 -0
  139. package/src/lmdb-v2/multi_map.ts +150 -0
  140. package/src/lmdb-v2/read_transaction.ts +40 -0
  141. package/src/lmdb-v2/set.ts +33 -0
  142. package/src/lmdb-v2/singleton.ts +5 -1
  143. package/src/lmdb-v2/store.ts +22 -14
  144. package/src/lmdb-v2/write_transaction.ts +2 -2
  145. package/src/stores/index.ts +2 -0
  146. package/src/stores/l2_tips_store.ts +18 -9
  147. package/dest/interfaces/store_test_suite.d.ts +0 -3
  148. package/dest/interfaces/store_test_suite.d.ts.map +0 -1
  149. package/dest/interfaces/store_test_suite.js +0 -37
  150. package/src/interfaces/store_test_suite.ts +0 -56
package/dest/utils.d.ts CHANGED
@@ -9,4 +9,4 @@ import type { AztecAsyncKVStore, AztecKVStore } from './interfaces/store.js';
9
9
  * @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
10
10
  */
11
11
  export declare function initStoreForRollup<T extends AztecKVStore | AztecAsyncKVStore>(store: T, rollupAddress: EthAddress, log?: Logger): Promise<T>;
12
- //# sourceMappingURL=utils.d.ts.map
12
+ //# sourceMappingURL=data:application/json;base64,eyJ2ZXJzaW9uIjozLCJmaWxlIjoidXRpbHMuZC50cyIsInNvdXJjZVJvb3QiOiIiLCJzb3VyY2VzIjpbIi4uL3NyYy91dGlscy50cyJdLCJuYW1lcyI6W10sIm1hcHBpbmdzIjoiQUFBQSxPQUFPLEtBQUssRUFBRSxVQUFVLEVBQUUsTUFBTSwrQkFBK0IsQ0FBQztBQUNoRSxPQUFPLEtBQUssRUFBRSxNQUFNLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUdwRCxPQUFPLEtBQUssRUFBRSxpQkFBaUIsRUFBRSxZQUFZLEVBQUUsTUFBTSx1QkFBdUIsQ0FBQztBQUc3RTs7Ozs7O0dBTUc7QUFDSCx3QkFBc0Isa0JBQWtCLENBQUMsQ0FBQyxTQUFTLFlBQVksR0FBRyxpQkFBaUIsRUFDakYsS0FBSyxFQUFFLENBQUMsRUFDUixhQUFhLEVBQUUsVUFBVSxFQUN6QixHQUFHLENBQUMsRUFBRSxNQUFNLEdBQ1gsT0FBTyxDQUFDLENBQUMsQ0FBQyxDQXFCWiJ9
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@aztec/kv-store",
3
- "version": "0.0.0-test.1",
3
+ "version": "0.0.1-commit.5daedc8",
4
4
  "type": "module",
5
5
  "exports": {
6
6
  ".": "./dest/interfaces/index.js",
@@ -11,49 +11,50 @@
11
11
  "./config": "./dest/config.js"
12
12
  },
13
13
  "scripts": {
14
- "build": "yarn clean && tsc -b",
15
- "build:dev": "tsc -b --watch",
14
+ "build": "yarn clean && tsgo -b",
15
+ "build:dev": "tsgo -b --watch",
16
16
  "clean": "rm -rf ./dest .tsbuildinfo",
17
- "formatting": "run -T prettier --check ./src && run -T eslint ./src",
18
- "formatting:fix": "run -T eslint --fix ./src && run -T prettier -w ./src",
19
17
  "test:node": "NODE_NO_WARNINGS=1 mocha --config ./.mocharc.json",
20
18
  "test:browser": "wtr --config ./web-test-runner.config.mjs",
21
- "test": "yarn test:node && yarn test:browser && true"
19
+ "test": "yarn test:node",
20
+ "test:jest": "NODE_NO_WARNINGS=1 node --experimental-vm-modules ../node_modules/.bin/jest --passWithNoTests --maxWorkers=${JEST_MAX_WORKERS:-8}"
22
21
  },
23
22
  "inherits": [
24
23
  "../package.common.json",
25
24
  "./package.local.json"
26
25
  ],
27
26
  "dependencies": {
28
- "@aztec/ethereum": "0.0.0-test.1",
29
- "@aztec/foundation": "0.0.0-test.1",
30
- "@aztec/native": "0.0.0-test.1",
31
- "@aztec/stdlib": "0.0.0-test.1",
27
+ "@aztec/ethereum": "0.0.1-commit.5daedc8",
28
+ "@aztec/foundation": "0.0.1-commit.5daedc8",
29
+ "@aztec/native": "0.0.1-commit.5daedc8",
30
+ "@aztec/stdlib": "0.0.1-commit.5daedc8",
32
31
  "idb": "^8.0.0",
33
32
  "lmdb": "^3.2.0",
34
33
  "msgpackr": "^1.11.2",
34
+ "ohash": "^2.0.11",
35
35
  "ordered-binary": "^1.5.3"
36
36
  },
37
37
  "devDependencies": {
38
- "@jest/globals": "^29.5.0",
38
+ "@jest/globals": "^30.0.0",
39
39
  "@types/chai": "^5.0.1",
40
40
  "@types/chai-as-promised": "^8.0.1",
41
- "@types/jest": "^29.5.0",
41
+ "@types/jest": "^30.0.0",
42
42
  "@types/mocha": "^10.0.10",
43
43
  "@types/mocha-each": "^2.0.4",
44
- "@types/node": "^18.7.23",
44
+ "@types/node": "^22.15.17",
45
45
  "@types/sinon": "^17.0.3",
46
+ "@typescript/native-preview": "7.0.0-dev.20251126.1",
46
47
  "@web/dev-server-esbuild": "^1.0.3",
47
48
  "@web/test-runner": "^0.19.0",
48
49
  "@web/test-runner-playwright": "^0.11.0",
49
50
  "chai": "^5.1.2",
50
51
  "chai-as-promised": "^8.0.1",
51
- "jest": "^29.5.0",
52
+ "jest": "^30.0.0",
52
53
  "mocha": "^10.8.2",
53
54
  "mocha-each": "^2.0.1",
54
55
  "sinon": "^19.0.2",
55
56
  "ts-node": "^10.9.1",
56
- "typescript": "^5.0.4"
57
+ "typescript": "^5.3.3"
57
58
  },
58
59
  "files": [
59
60
  "dest",
@@ -61,7 +62,7 @@
61
62
  "!*.test.*"
62
63
  ],
63
64
  "engines": {
64
- "node": ">=18"
65
+ "node": ">=20.10"
65
66
  },
66
67
  "jest": {
67
68
  "extensionsToTreatAsEsm": [
@@ -94,6 +95,10 @@
94
95
  "testTimeout": 120000,
95
96
  "setupFiles": [
96
97
  "../../foundation/src/jest/setup.mjs"
98
+ ],
99
+ "testEnvironment": "../../foundation/src/jest/env.mjs",
100
+ "setupFilesAfterEnv": [
101
+ "../../foundation/src/jest/setupAfterEnv.mjs"
97
102
  ]
98
103
  }
99
104
  }
package/src/config.ts CHANGED
@@ -4,7 +4,7 @@ import type { EthAddress } from '@aztec/foundation/eth-address';
4
4
 
5
5
  export type DataStoreConfig = {
6
6
  dataDirectory: string | undefined;
7
- dataStoreMapSizeKB: number;
7
+ dataStoreMapSizeKb: number;
8
8
  l1Contracts?: { rollupAddress: EthAddress };
9
9
  };
10
10
 
@@ -13,14 +13,16 @@ export const dataConfigMappings: ConfigMappingsType<DataStoreConfig> = {
13
13
  env: 'DATA_DIRECTORY',
14
14
  description: 'Optional dir to store data. If omitted will store in memory.',
15
15
  },
16
- dataStoreMapSizeKB: {
16
+ dataStoreMapSizeKb: {
17
17
  env: 'DATA_STORE_MAP_SIZE_KB',
18
- description: 'DB mapping size to be applied to all key/value stores',
18
+ description: 'The maximum possible size of a data store DB in KB. Can be overridden by component-specific options.',
19
19
  ...numberConfigHelper(128 * 1_024 * 1_024), // Defaulted to 128 GB
20
20
  },
21
21
  l1Contracts: {
22
22
  description: 'The deployed L1 contract addresses',
23
- nested: l1ContractAddressesMapping,
23
+ nested: {
24
+ rollupAddress: l1ContractAddressesMapping.rollupAddress,
25
+ },
24
26
  },
25
27
  };
26
28
 
@@ -1,12 +1,14 @@
1
1
  import type { IDBPDatabase, IDBPObjectStore } from 'idb';
2
+ import { hash } from 'ohash';
2
3
 
3
4
  import type { AztecAsyncArray } from '../interfaces/array.js';
5
+ import type { Value } from '../interfaces/common.js';
4
6
  import type { AztecIDBSchema } from './store.js';
5
7
 
6
8
  /**
7
9
  * A persistent array backed by IndexedDB.
8
10
  */
9
- export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
11
+ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T> {
10
12
  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
11
13
  #rootDB: IDBPDatabase<AztecIDBSchema>;
12
14
  #container: string;
@@ -39,6 +41,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
39
41
  for (const val of vals) {
40
42
  await this.db.put({
41
43
  value: val,
44
+ hash: hash(val),
42
45
  container: this.#container,
43
46
  key: this.#name,
44
47
  keyCount: length + 1,
@@ -86,6 +89,7 @@ export class IndexedDBAztecArray<T> implements AztecAsyncArray<T> {
86
89
 
87
90
  await this.db.put({
88
91
  value: val,
92
+ hash: hash(val),
89
93
  container: this.#container,
90
94
  key: this.#name,
91
95
  keyCount: index + 1,
@@ -14,8 +14,8 @@ export async function createStore(name: string, config: DataStoreConfig, log: Lo
14
14
 
15
15
  log.info(
16
16
  dataDirectory
17
- ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
18
- : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
17
+ ? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKb} KB`
18
+ : `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
19
19
  );
20
20
  const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
21
21
  if (config.l1Contracts?.rollupAddress) {
@@ -1,22 +1,23 @@
1
1
  import type { IDBPDatabase, IDBPObjectStore } from 'idb';
2
+ import { hash } from 'ohash';
2
3
 
3
- import type { Key, Range } from '../interfaces/common.js';
4
- import type { AztecAsyncMultiMap } from '../interfaces/map.js';
4
+ import type { Key, Range, Value } from '../interfaces/common.js';
5
+ import type { AztecAsyncMap } from '../interfaces/map.js';
5
6
  import type { AztecIDBSchema } from './store.js';
6
7
 
7
8
  /**
8
9
  * A map backed by IndexedDB.
9
10
  */
10
- export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
11
+ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
11
12
  protected name: string;
12
- #container: string;
13
+ protected container: string;
13
14
 
14
15
  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
15
16
  #rootDB: IDBPDatabase<AztecIDBSchema>;
16
17
 
17
18
  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
18
19
  this.name = mapName;
19
- this.#container = `map:${mapName}`;
20
+ this.container = `map:${mapName}`;
20
21
  this.#rootDB = rootDB;
21
22
  }
22
23
 
@@ -29,41 +30,38 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
29
30
  }
30
31
 
31
32
  async getAsync(key: K): Promise<V | undefined> {
32
- const data = await this.db.get(this.#slot(key));
33
+ const data = await this.db.get(this.slot(key));
33
34
  return data?.value as V;
34
35
  }
35
36
 
36
- async *getValuesAsync(key: K): AsyncIterableIterator<V> {
37
- const index = this.db.index('keyCount');
38
- const rangeQuery = IDBKeyRange.bound(
39
- [this.#container, this.#normalizeKey(key), 0],
40
- [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
41
- false,
42
- false,
43
- );
44
- for await (const cursor of index.iterate(rangeQuery)) {
45
- yield cursor.value.value as V;
46
- }
47
- }
48
-
49
37
  async hasAsync(key: K): Promise<boolean> {
50
38
  const result = (await this.getAsync(key)) !== undefined;
51
39
  return result;
52
40
  }
53
41
 
42
+ async sizeAsync(): Promise<number> {
43
+ const index = this.db.index('key');
44
+ const rangeQuery = IDBKeyRange.bound([this.container, ''], [this.container, '\uffff']);
45
+ return await index.count(rangeQuery);
46
+ }
47
+
54
48
  async set(key: K, val: V): Promise<void> {
55
- const count = await this.db
56
- .index('key')
57
- .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
58
49
  await this.db.put({
59
50
  value: val,
60
- container: this.#container,
61
- key: this.#normalizeKey(key),
62
- keyCount: count + 1,
63
- slot: this.#slot(key, count),
51
+ hash: hash(val),
52
+ container: this.container,
53
+ key: this.normalizeKey(key),
54
+ keyCount: 1,
55
+ slot: this.slot(key),
64
56
  });
65
57
  }
66
58
 
59
+ async setMany(entries: { key: K; value: V }[]): Promise<void> {
60
+ for (const { key, value } of entries) {
61
+ await this.set(key, value);
62
+ }
63
+ }
64
+
67
65
  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
68
66
  throw new Error('Not implemented');
69
67
  }
@@ -77,30 +75,14 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
77
75
  }
78
76
 
79
77
  async delete(key: K): Promise<void> {
80
- await this.db.delete(this.#slot(key));
81
- }
82
-
83
- async deleteValue(key: K, val: V): Promise<void> {
84
- const index = this.db.index('keyCount');
85
- const rangeQuery = IDBKeyRange.bound(
86
- [this.#container, this.#normalizeKey(key), 0],
87
- [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
88
- false,
89
- false,
90
- );
91
- for await (const cursor of index.iterate(rangeQuery)) {
92
- if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
93
- await cursor.delete();
94
- return;
95
- }
96
- }
78
+ await this.db.delete(this.slot(key));
97
79
  }
98
80
 
99
81
  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
100
82
  const index = this.db.index('key');
101
83
  const rangeQuery = IDBKeyRange.bound(
102
- [this.#container, range.start ?? ''],
103
- [this.#container, range.end ?? '\uffff'],
84
+ [this.container, range.start ? this.normalizeKey(range.start) : ''],
85
+ [this.container, range.end ? this.normalizeKey(range.end) : '\uffff'],
104
86
  !!range.reverse,
105
87
  !range.reverse,
106
88
  );
@@ -109,7 +91,7 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
109
91
  if (range.limit && count >= range.limit) {
110
92
  return;
111
93
  }
112
- yield [cursor.value.key, cursor.value.value] as [K, V];
94
+ yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
113
95
  count++;
114
96
  }
115
97
  }
@@ -122,21 +104,21 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
122
104
 
123
105
  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
124
106
  for await (const [key, _] of this.entriesAsync(range)) {
125
- yield this.#denormalizeKey(key as string);
107
+ yield key;
126
108
  }
127
109
  }
128
110
 
129
111
  #denormalizeKey(key: string): K {
130
- const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
131
- return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
112
+ const denormalizedKey = key.split(',').map(part => (part.startsWith('n_') ? Number(part.slice(2)) : part));
113
+ return (denormalizedKey.length > 1 ? denormalizedKey : denormalizedKey[0]) as K;
132
114
  }
133
115
 
134
- #normalizeKey(key: K): string {
116
+ protected normalizeKey(key: K): string {
135
117
  const arrayKey = Array.isArray(key) ? key : [key];
136
- return arrayKey.join(',');
118
+ return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
137
119
  }
138
120
 
139
- #slot(key: K, index: number = 0): string {
140
- return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
121
+ protected slot(key: K, index: number = 0): string {
122
+ return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
141
123
  }
142
124
  }
@@ -0,0 +1,91 @@
1
+ import { hash } from 'ohash';
2
+
3
+ import type { Key, Value } from '../interfaces/common.js';
4
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
5
+ import { IndexedDBAztecMap } from './map.js';
6
+
7
+ /**
8
+ * A multi map backed by IndexedDB.
9
+ */
10
+ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
11
+ extends IndexedDBAztecMap<K, V>
12
+ implements AztecAsyncMultiMap<K, V>
13
+ {
14
+ override async set(key: K, val: V): Promise<void> {
15
+ // Inserting repeated values is a no-op
16
+ const exists = !!(await this.db
17
+ .index('hash')
18
+ .get(
19
+ IDBKeyRange.bound(
20
+ [this.container, this.normalizeKey(key), hash(val)],
21
+ [this.container, this.normalizeKey(key), hash(val)],
22
+ ),
23
+ ));
24
+ if (exists) {
25
+ return;
26
+ }
27
+ // Get the maximum keyCount for the given key
28
+ // In order to support sparse multimaps, we cannot rely
29
+ // on just counting the number of entries for the key, since we would repeat slots
30
+ // if we delete an entry
31
+ // set -> container:key:0 (keyCount = 1)
32
+ // set -> container:key:1 (keyCount = 2)
33
+ // delete -> container:key:0 (keyCount = 1)
34
+ // set -> container:key:1 <--- already exists!
35
+ // Instead, we iterate in reverse order to get the last inserted entry
36
+ const index = this.db.index('keyCount');
37
+ const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
38
+ const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
39
+ const count = maxEntry?.value?.keyCount ?? 0;
40
+ await this.db.put({
41
+ value: val,
42
+ hash: hash(val),
43
+ container: this.container,
44
+ key: this.normalizeKey(key),
45
+ keyCount: count + 1,
46
+ slot: this.slot(key, count),
47
+ });
48
+ }
49
+
50
+ async *getValuesAsync(key: K): AsyncIterableIterator<V> {
51
+ // Iterate over the whole range of keyCount for the given key
52
+ const index = this.db.index('keyCount');
53
+ const rangeQuery = IDBKeyRange.bound(
54
+ [this.container, this.normalizeKey(key), 0],
55
+ [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
56
+ false,
57
+ false,
58
+ );
59
+ for await (const cursor of index.iterate(rangeQuery)) {
60
+ yield cursor.value.value as V;
61
+ }
62
+ }
63
+
64
+ getValueCountAsync(key: K): Promise<number> {
65
+ // Count entries over the keyCount index range for this key
66
+ const index = this.db.index('keyCount');
67
+ const rangeQuery = IDBKeyRange.bound(
68
+ [this.container, this.normalizeKey(key), 0],
69
+ [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
70
+ false,
71
+ false,
72
+ );
73
+ return index.count(rangeQuery);
74
+ }
75
+
76
+ async deleteValue(key: K, val: V): Promise<void> {
77
+ // Since we know the value, we can hash it and directly query the "hash" index
78
+ // to avoid having to iterate over all the values
79
+ const fullKey = await this.db
80
+ .index('hash')
81
+ .getKey(
82
+ IDBKeyRange.bound(
83
+ [this.container, this.normalizeKey(key), hash(val)],
84
+ [this.container, this.normalizeKey(key), hash(val)],
85
+ ),
86
+ );
87
+ if (fullKey) {
88
+ await this.db.delete(fullKey);
89
+ }
90
+ }
91
+ }
@@ -1,12 +1,14 @@
1
1
  import type { IDBPDatabase, IDBPObjectStore } from 'idb';
2
+ import { hash } from 'ohash';
2
3
 
4
+ import type { Value } from '../interfaces/common.js';
3
5
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
4
6
  import type { AztecIDBSchema } from './store.js';
5
7
 
6
8
  /**
7
9
  * Stores a single value in IndexedDB.
8
10
  */
9
- export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
11
+ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
10
12
  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
11
13
  #rootDB: IDBPDatabase<AztecIDBSchema>;
12
14
  #container: string;
@@ -38,6 +40,7 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
38
40
  key: this.#slot,
39
41
  keyCount: 1,
40
42
  value: val,
43
+ hash: hash(val),
41
44
  });
42
45
  return result !== undefined;
43
46
  }
@@ -1,26 +1,36 @@
1
1
  import type { Logger } from '@aztec/foundation/log';
2
+ import { SerialQueue } from '@aztec/foundation/queue';
2
3
 
3
4
  import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
4
5
 
5
6
  import type { AztecAsyncArray } from '../interfaces/array.js';
6
- import type { Key, StoreSize } from '../interfaces/common.js';
7
+ import type { Key, StoreSize, Value } from '../interfaces/common.js';
7
8
  import type { AztecAsyncCounter } from '../interfaces/counter.js';
8
- import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
9
+ import type { AztecAsyncMap } from '../interfaces/map.js';
10
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
9
11
  import type { AztecAsyncSet } from '../interfaces/set.js';
10
12
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
11
13
  import type { AztecAsyncKVStore } from '../interfaces/store.js';
12
14
  import { IndexedDBAztecArray } from './array.js';
13
15
  import { IndexedDBAztecMap } from './map.js';
16
+ import { IndexedDBAztecMultiMap } from './multi_map.js';
14
17
  import { IndexedDBAztecSet } from './set.js';
15
18
  import { IndexedDBAztecSingleton } from './singleton.js';
16
19
 
17
- export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
20
+ export type StoredData<V extends Value> = {
21
+ value: V;
22
+ container: string;
23
+ key: string;
24
+ keyCount: number;
25
+ slot: string;
26
+ hash: string;
27
+ };
18
28
 
19
29
  export interface AztecIDBSchema extends DBSchema {
20
30
  data: {
21
31
  value: StoredData<any>;
22
32
  key: string;
23
- indexes: { container: string; key: string; keyCount: number };
33
+ indexes: { container: string; key: string; keyCount: number; hash: string };
24
34
  };
25
35
  }
26
36
 
@@ -29,18 +39,27 @@ export interface AztecIDBSchema extends DBSchema {
29
39
  */
30
40
 
31
41
  export class AztecIndexedDBStore implements AztecAsyncKVStore {
32
- #log: Logger;
33
42
  #rootDB: IDBPDatabase<AztecIDBSchema>;
34
43
  #name: string;
44
+ #txQueue: SerialQueue;
35
45
 
36
46
  #containers = new Set<
37
- IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
47
+ | IndexedDBAztecArray<any>
48
+ | IndexedDBAztecMap<any, any>
49
+ | IndexedDBAztecMultiMap<any, any>
50
+ | IndexedDBAztecSet<any>
51
+ | IndexedDBAztecSingleton<any>
38
52
  >();
39
53
 
40
- constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
54
+ constructor(
55
+ rootDB: IDBPDatabase<AztecIDBSchema>,
56
+ public readonly isEphemeral: boolean,
57
+ name: string,
58
+ ) {
41
59
  this.#rootDB = rootDB;
42
- this.#log = log;
43
60
  this.#name = name;
61
+ this.#txQueue = new SerialQueue();
62
+ this.#txQueue.start();
44
63
  }
45
64
  /**
46
65
  * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
@@ -53,50 +72,32 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
53
72
  * @returns The store
54
73
  */
55
74
  static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
56
- name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
75
+ name = name && !ephemeral ? name : globalThis.crypto.getRandomValues(new Uint8Array(16)).join('');
57
76
  log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
58
77
  const rootDB = await openDB<AztecIDBSchema>(name, 1, {
59
78
  upgrade(db) {
60
79
  const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
61
80
 
62
81
  objectStore.createIndex('key', ['container', 'key'], { unique: false });
63
- objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
82
+ // Keep count of the maximum number of keys ever inserted in the container
83
+ // This allows unique slots for repeated keys, which is useful for multi-maps
84
+ objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: true });
85
+ // Keep an index on the pair key-hash for a given container, allowing us to efficiently
86
+ // delete unique values from multi-maps
87
+ objectStore.createIndex('hash', ['container', 'key', 'hash'], { unique: true });
64
88
  },
65
89
  });
66
90
 
67
- const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
91
+ const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, name);
68
92
  return kvStore;
69
93
  }
70
94
 
71
- /**
72
- * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
73
- * @returns A new AztecIndexedDBStore.
74
- */
75
- async fork(): Promise<AztecAsyncKVStore> {
76
- const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
77
- this.#log.verbose(`Forking store to ${forkedStore.#name}`);
78
-
79
- // Copy old data to new store
80
- const oldData = this.#rootDB.transaction('data').store;
81
- const dataToWrite = [];
82
- for await (const cursor of oldData.iterate()) {
83
- dataToWrite.push(cursor.value);
84
- }
85
- const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
86
- for (const data of dataToWrite) {
87
- await tx.add(data);
88
- }
89
-
90
- this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
91
- return forkedStore;
92
- }
93
-
94
95
  /**
95
96
  * Creates a new AztecMap in the store.
96
97
  * @param name - Name of the map
97
98
  * @returns A new AztecMap
98
99
  */
99
- openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
100
+ openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
100
101
  const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
101
102
  this.#containers.add(map);
102
103
  return map;
@@ -118,8 +119,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
118
119
  * @param name - Name of the map
119
120
  * @returns A new AztecMultiMap
120
121
  */
121
- openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
122
- const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
122
+ openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
123
+ const multimap = new IndexedDBAztecMultiMap<K, V>(this.#rootDB, name);
123
124
  this.#containers.add(multimap);
124
125
  return multimap;
125
126
  }
@@ -133,7 +134,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
133
134
  * @param name - Name of the array
134
135
  * @returns A new AztecArray
135
136
  */
136
- openArray<T>(name: string): AztecAsyncArray<T> {
137
+ openArray<T extends Value>(name: string): AztecAsyncArray<T> {
137
138
  const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
138
139
  this.#containers.add(array);
139
140
  return array;
@@ -144,7 +145,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
144
145
  * @param name - Name of the singleton
145
146
  * @returns A new AztecSingleton
146
147
  */
147
- openSingleton<T>(name: string): AztecAsyncSingleton<T> {
148
+ openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
148
149
  const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
149
150
  this.#containers.add(singleton);
150
151
  return singleton;
@@ -155,22 +156,27 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
155
156
  * @param callback - Function to execute in a transaction
156
157
  * @returns A promise that resolves to the return value of the callback
157
158
  */
158
- async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
159
- const tx = this.#rootDB.transaction('data', 'readwrite');
160
- for (const container of this.#containers) {
161
- container.db = tx.store;
162
- }
163
- // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
164
- // By then, the tx would be closed
165
- const runningPromise = callback();
166
- // Wait for the transaction to finish
167
- await tx.done;
168
- for (const container of this.#containers) {
169
- container.db = undefined;
170
- }
171
- // Return the result of the callback.
172
- // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
173
- return await runningPromise;
159
+ transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
160
+ // We can only have one transaction at a time for the same store
161
+ // So we need to wait for the current one to finish
162
+ return this.#txQueue.put(async () => {
163
+ const tx = this.#rootDB.transaction('data', 'readwrite');
164
+ for (const container of this.#containers) {
165
+ container.db = tx.store;
166
+ }
167
+ // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
168
+ // By then, the tx would be closed
169
+ const runningPromise = callback();
170
+ // Wait for the transaction to finish
171
+ await tx.done;
172
+ for (const container of this.#containers) {
173
+ container.db = undefined;
174
+ }
175
+
176
+ // Return the result of the callback.
177
+ // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
178
+ return await runningPromise;
179
+ });
174
180
  }
175
181
 
176
182
  /**
@@ -188,10 +194,14 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
188
194
  }
189
195
 
190
196
  estimateSize(): Promise<StoreSize> {
191
- return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
197
+ return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
192
198
  }
193
199
 
194
200
  close(): Promise<void> {
195
201
  return Promise.resolve();
196
202
  }
203
+
204
+ backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
205
+ throw new Error('Method not implemented.');
206
+ }
197
207
  }