@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.017a351

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (221) hide show
  1. package/README.md +8 -1
  2. package/dest/bench/shared_map_bench.d.ts +19 -0
  3. package/dest/bench/shared_map_bench.d.ts.map +1 -0
  4. package/dest/bench/shared_map_bench.js +91 -0
  5. package/dest/indexeddb/array.d.ts +3 -2
  6. package/dest/indexeddb/array.d.ts.map +1 -1
  7. package/dest/indexeddb/array.js +24 -7
  8. package/dest/indexeddb/index.d.ts +3 -3
  9. package/dest/indexeddb/index.d.ts.map +1 -1
  10. package/dest/indexeddb/index.js +4 -7
  11. package/dest/indexeddb/map.d.ts +18 -6
  12. package/dest/indexeddb/map.d.ts.map +1 -1
  13. package/dest/indexeddb/map.js +51 -63
  14. package/dest/indexeddb/multi_map.d.ts +14 -0
  15. package/dest/indexeddb/multi_map.d.ts.map +1 -0
  16. package/dest/indexeddb/multi_map.js +107 -0
  17. package/dest/indexeddb/set.d.ts +1 -1
  18. package/dest/indexeddb/set.d.ts.map +1 -1
  19. package/dest/indexeddb/singleton.d.ts +3 -2
  20. package/dest/indexeddb/singleton.d.ts.map +1 -1
  21. package/dest/indexeddb/singleton.js +6 -2
  22. package/dest/indexeddb/store.d.ts +15 -16
  23. package/dest/indexeddb/store.d.ts.map +1 -1
  24. package/dest/indexeddb/store.js +51 -46
  25. package/dest/interfaces/array.d.ts +5 -4
  26. package/dest/interfaces/array.d.ts.map +1 -1
  27. package/dest/interfaces/array.js +1 -3
  28. package/dest/interfaces/array_test_suite.d.ts +1 -1
  29. package/dest/interfaces/array_test_suite.d.ts.map +1 -1
  30. package/dest/interfaces/array_test_suite.js +33 -34
  31. package/dest/interfaces/common.d.ts +11 -9
  32. package/dest/interfaces/common.d.ts.map +1 -1
  33. package/dest/interfaces/common.js +8 -3
  34. package/dest/interfaces/counter.d.ts +1 -1
  35. package/dest/interfaces/index.d.ts +4 -2
  36. package/dest/interfaces/index.d.ts.map +1 -1
  37. package/dest/interfaces/index.js +2 -0
  38. package/dest/interfaces/map.d.ts +21 -49
  39. package/dest/interfaces/map.d.ts.map +1 -1
  40. package/dest/interfaces/map.js +1 -1
  41. package/dest/interfaces/map_test_suite.d.ts +1 -1
  42. package/dest/interfaces/map_test_suite.d.ts.map +1 -1
  43. package/dest/interfaces/map_test_suite.js +193 -83
  44. package/dest/interfaces/multi_map.d.ts +41 -0
  45. package/dest/interfaces/multi_map.d.ts.map +1 -0
  46. package/dest/interfaces/multi_map.js +3 -0
  47. package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
  48. package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
  49. package/dest/interfaces/multi_map_test_suite.js +363 -0
  50. package/dest/interfaces/set.d.ts +1 -1
  51. package/dest/interfaces/set_test_suite.d.ts +1 -1
  52. package/dest/interfaces/set_test_suite.d.ts.map +1 -1
  53. package/dest/interfaces/set_test_suite.js +13 -14
  54. package/dest/interfaces/singleton.d.ts +1 -1
  55. package/dest/interfaces/singleton_test_suite.d.ts +1 -1
  56. package/dest/interfaces/singleton_test_suite.d.ts.map +1 -1
  57. package/dest/interfaces/singleton_test_suite.js +6 -7
  58. package/dest/interfaces/store.d.ts +18 -43
  59. package/dest/interfaces/store.d.ts.map +1 -1
  60. package/dest/interfaces/utils.d.ts +3 -1
  61. package/dest/interfaces/utils.d.ts.map +1 -1
  62. package/dest/interfaces/utils.js +3 -1
  63. package/dest/lmdb/array.d.ts +3 -2
  64. package/dest/lmdb/array.d.ts.map +1 -1
  65. package/dest/lmdb/array.js +4 -2
  66. package/dest/lmdb/counter.d.ts +1 -1
  67. package/dest/lmdb/counter.d.ts.map +1 -1
  68. package/dest/lmdb/index.d.ts +3 -3
  69. package/dest/lmdb/index.d.ts.map +1 -1
  70. package/dest/lmdb/index.js +6 -6
  71. package/dest/lmdb/map.d.ts +11 -23
  72. package/dest/lmdb/map.d.ts.map +1 -1
  73. package/dest/lmdb/map.js +15 -81
  74. package/dest/lmdb/multi_map.d.ts +13 -0
  75. package/dest/lmdb/multi_map.d.ts.map +1 -0
  76. package/dest/lmdb/multi_map.js +44 -0
  77. package/dest/lmdb/set.d.ts +1 -1
  78. package/dest/lmdb/set.d.ts.map +1 -1
  79. package/dest/lmdb/singleton.d.ts +1 -1
  80. package/dest/lmdb/singleton.d.ts.map +1 -1
  81. package/dest/lmdb/store.d.ts +11 -27
  82. package/dest/lmdb/store.d.ts.map +1 -1
  83. package/dest/lmdb/store.js +24 -40
  84. package/dest/lmdb-v2/array.d.ts +4 -3
  85. package/dest/lmdb-v2/array.d.ts.map +1 -1
  86. package/dest/lmdb-v2/array.js +4 -2
  87. package/dest/lmdb-v2/factory.d.ts +32 -10
  88. package/dest/lmdb-v2/factory.d.ts.map +1 -1
  89. package/dest/lmdb-v2/factory.js +83 -18
  90. package/dest/lmdb-v2/index.d.ts +1 -1
  91. package/dest/lmdb-v2/map.d.ts +12 -45
  92. package/dest/lmdb-v2/map.d.ts.map +1 -1
  93. package/dest/lmdb-v2/map.js +17 -104
  94. package/dest/lmdb-v2/message.d.ts +24 -5
  95. package/dest/lmdb-v2/message.d.ts.map +1 -1
  96. package/dest/lmdb-v2/message.js +6 -4
  97. package/dest/lmdb-v2/multi_map.d.ts +52 -0
  98. package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
  99. package/dest/lmdb-v2/multi_map.js +123 -0
  100. package/dest/lmdb-v2/read_transaction.d.ts +3 -1
  101. package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
  102. package/dest/lmdb-v2/read_transaction.js +47 -11
  103. package/dest/lmdb-v2/set.d.ts +15 -0
  104. package/dest/lmdb-v2/set.d.ts.map +1 -0
  105. package/dest/lmdb-v2/set.js +23 -0
  106. package/dest/lmdb-v2/singleton.d.ts +2 -2
  107. package/dest/lmdb-v2/singleton.d.ts.map +1 -1
  108. package/dest/lmdb-v2/singleton.js +1 -1
  109. package/dest/lmdb-v2/store.d.ts +13 -13
  110. package/dest/lmdb-v2/store.d.ts.map +1 -1
  111. package/dest/lmdb-v2/store.js +22 -32
  112. package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
  113. package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
  114. package/dest/lmdb-v2/tx-helpers.js +21 -0
  115. package/dest/lmdb-v2/utils.d.ts +3 -14
  116. package/dest/lmdb-v2/utils.d.ts.map +1 -1
  117. package/dest/lmdb-v2/utils.js +0 -94
  118. package/dest/lmdb-v2/write_transaction.d.ts +3 -5
  119. package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
  120. package/dest/lmdb-v2/write_transaction.js +9 -5
  121. package/dest/sqlite-opfs/array.d.ts +21 -0
  122. package/dest/sqlite-opfs/array.d.ts.map +1 -0
  123. package/dest/sqlite-opfs/array.js +128 -0
  124. package/dest/sqlite-opfs/errors.d.ts +27 -0
  125. package/dest/sqlite-opfs/errors.d.ts.map +1 -0
  126. package/dest/sqlite-opfs/errors.js +34 -0
  127. package/dest/sqlite-opfs/index.d.ts +16 -0
  128. package/dest/sqlite-opfs/index.d.ts.map +1 -0
  129. package/dest/sqlite-opfs/index.js +22 -0
  130. package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts +32 -0
  131. package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts.map +1 -0
  132. package/dest/sqlite-opfs/internal/ordered-binary-browser.js +448 -0
  133. package/dest/sqlite-opfs/map.d.ts +35 -0
  134. package/dest/sqlite-opfs/map.d.ts.map +1 -0
  135. package/dest/sqlite-opfs/map.js +163 -0
  136. package/dest/sqlite-opfs/messages.d.ts +66 -0
  137. package/dest/sqlite-opfs/messages.d.ts.map +1 -0
  138. package/dest/sqlite-opfs/messages.js +5 -0
  139. package/dest/sqlite-opfs/multi_map.d.ts +16 -0
  140. package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
  141. package/dest/sqlite-opfs/multi_map.js +67 -0
  142. package/dest/sqlite-opfs/set.d.ts +13 -0
  143. package/dest/sqlite-opfs/set.d.ts.map +1 -0
  144. package/dest/sqlite-opfs/set.js +19 -0
  145. package/dest/sqlite-opfs/singleton.d.ts +13 -0
  146. package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
  147. package/dest/sqlite-opfs/singleton.js +48 -0
  148. package/dest/sqlite-opfs/store.d.ts +79 -0
  149. package/dest/sqlite-opfs/store.d.ts.map +1 -0
  150. package/dest/sqlite-opfs/store.js +281 -0
  151. package/dest/sqlite-opfs/worker.d.ts +2 -0
  152. package/dest/sqlite-opfs/worker.d.ts.map +1 -0
  153. package/dest/sqlite-opfs/worker.js +249 -0
  154. package/dest/stores/index.d.ts +2 -1
  155. package/dest/stores/index.d.ts.map +1 -1
  156. package/dest/stores/index.js +1 -0
  157. package/dest/stores/l2_tips_store.d.ts +25 -9
  158. package/dest/stores/l2_tips_store.d.ts.map +1 -1
  159. package/dest/stores/l2_tips_store.js +64 -46
  160. package/dest/utils.d.ts +9 -6
  161. package/dest/utils.d.ts.map +1 -1
  162. package/dest/utils.js +51 -16
  163. package/package.json +44 -33
  164. package/src/bench/shared_map_bench.ts +111 -0
  165. package/src/indexeddb/array.ts +9 -5
  166. package/src/indexeddb/index.ts +11 -9
  167. package/src/indexeddb/map.ts +51 -56
  168. package/src/indexeddb/multi_map.ts +105 -0
  169. package/src/indexeddb/singleton.ts +5 -2
  170. package/src/indexeddb/store.ts +77 -60
  171. package/src/interfaces/array.ts +5 -3
  172. package/src/interfaces/array_test_suite.ts +33 -35
  173. package/src/interfaces/common.ts +20 -9
  174. package/src/interfaces/index.ts +3 -1
  175. package/src/interfaces/map.ts +19 -53
  176. package/src/interfaces/map_test_suite.ts +113 -58
  177. package/src/interfaces/multi_map.ts +45 -0
  178. package/src/interfaces/multi_map_test_suite.ts +372 -0
  179. package/src/interfaces/set_test_suite.ts +13 -15
  180. package/src/interfaces/singleton_test_suite.ts +6 -8
  181. package/src/interfaces/store.ts +18 -53
  182. package/src/interfaces/utils.ts +2 -0
  183. package/src/lmdb/array.ts +2 -1
  184. package/src/lmdb/index.ts +13 -8
  185. package/src/lmdb/map.ts +23 -94
  186. package/src/lmdb/multi_map.ts +51 -0
  187. package/src/lmdb/store.ts +36 -56
  188. package/src/lmdb-v2/array.ts +8 -3
  189. package/src/lmdb-v2/factory.ts +107 -26
  190. package/src/lmdb-v2/map.ts +30 -127
  191. package/src/lmdb-v2/message.ts +23 -0
  192. package/src/lmdb-v2/multi_map.ts +150 -0
  193. package/src/lmdb-v2/read_transaction.ts +53 -13
  194. package/src/lmdb-v2/set.ts +33 -0
  195. package/src/lmdb-v2/singleton.ts +6 -2
  196. package/src/lmdb-v2/store.ts +28 -44
  197. package/src/lmdb-v2/tx-helpers.ts +29 -0
  198. package/src/lmdb-v2/utils.ts +0 -118
  199. package/src/lmdb-v2/write_transaction.ts +11 -10
  200. package/src/sqlite-opfs/array.ts +124 -0
  201. package/src/sqlite-opfs/errors.ts +44 -0
  202. package/src/sqlite-opfs/index.ts +39 -0
  203. package/src/sqlite-opfs/internal/ordered-binary-browser.js +465 -0
  204. package/src/sqlite-opfs/map.ts +163 -0
  205. package/src/sqlite-opfs/messages.ts +39 -0
  206. package/src/sqlite-opfs/multi_map.ts +74 -0
  207. package/src/sqlite-opfs/set.ts +29 -0
  208. package/src/sqlite-opfs/singleton.ts +48 -0
  209. package/src/sqlite-opfs/store.ts +296 -0
  210. package/src/sqlite-opfs/worker.ts +231 -0
  211. package/src/stores/index.ts +2 -0
  212. package/src/stores/l2_tips_store.ts +68 -49
  213. package/src/utils.ts +79 -21
  214. package/dest/config.d.ts +0 -17
  215. package/dest/config.d.ts.map +0 -1
  216. package/dest/config.js +0 -24
  217. package/dest/interfaces/store_test_suite.d.ts +0 -3
  218. package/dest/interfaces/store_test_suite.d.ts.map +0 -1
  219. package/dest/interfaces/store_test_suite.js +0 -37
  220. package/src/config.ts +0 -34
  221. package/src/interfaces/store_test_suite.ts +0 -56
package/src/lmdb/map.ts CHANGED
@@ -1,7 +1,7 @@
1
1
  import type { Database, RangeOptions } from 'lmdb';
2
2
 
3
- import type { Key, Range } from '../interfaces/common.js';
4
- import type { AztecAsyncMultiMap, AztecMapWithSize, AztecMultiMap } from '../interfaces/map.js';
3
+ import type { Key, Range, Value } from '../interfaces/common.js';
4
+ import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js';
5
5
 
6
6
  /** The slot where a key-value entry would be stored */
7
7
  type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
@@ -9,7 +9,7 @@ type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
9
9
  /**
10
10
  * A map backed by LMDB.
11
11
  */
12
- export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V> {
12
+ export class LmdbAztecMap<K extends Key, V extends Value> implements AztecMap<K, V>, AztecAsyncMap<K, V> {
13
13
  protected db: Database<[K, V], MapValueSlot<K>>;
14
14
  protected name: string;
15
15
 
@@ -39,26 +39,6 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
39
39
  return Promise.resolve(this.get(key));
40
40
  }
41
41
 
42
- *getValues(key: K): IterableIterator<V> {
43
- const transaction = this.db.useReadTransaction();
44
- try {
45
- const values = this.db.getValues(this.slot(key), {
46
- transaction,
47
- });
48
- for (const value of values) {
49
- yield value?.[1];
50
- }
51
- } finally {
52
- transaction.done();
53
- }
54
- }
55
-
56
- async *getValuesAsync(key: K): AsyncIterableIterator<V> {
57
- for (const value of this.getValues(key)) {
58
- yield value;
59
- }
60
- }
61
-
62
42
  has(key: K): boolean {
63
43
  return this.db.doesExist(this.slot(key));
64
44
  }
@@ -71,6 +51,12 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
71
51
  await this.db.put(this.slot(key), [key, val]);
72
52
  }
73
53
 
54
+ async setMany(entries: { key: K; value: V }[]): Promise<void> {
55
+ for (const { key, value } of entries) {
56
+ await this.set(key, value);
57
+ }
58
+ }
59
+
74
60
  swap(key: K, fn: (val: V | undefined) => V): Promise<void> {
75
61
  return this.db.childTransaction(() => {
76
62
  const slot = this.slot(key);
@@ -90,10 +76,6 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
90
76
  await this.db.remove(this.slot(key));
91
77
  }
92
78
 
93
- async deleteValue(key: K, val: V): Promise<void> {
94
- await this.db.remove(this.slot(key), [key, val]);
95
- }
96
-
97
79
  *entries(range: Range<K> = {}): IterableIterator<[K, V]> {
98
80
  const transaction = this.db.useReadTransaction();
99
81
 
@@ -106,16 +88,16 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
106
88
  ? this.slot(range.end)
107
89
  : this.endSentinel
108
90
  : range.start
109
- ? this.slot(range.start)
110
- : this.startSentinel;
91
+ ? this.slot(range.start)
92
+ : this.startSentinel;
111
93
 
112
94
  const end = reverse
113
95
  ? range.start
114
96
  ? this.slot(range.start)
115
97
  : this.startSentinel
116
98
  : range.end
117
- ? this.slot(range.end)
118
- : this.endSentinel;
99
+ ? this.slot(range.end)
100
+ : this.endSentinel;
119
101
 
120
102
  const lmdbRange: RangeOptions = {
121
103
  start,
@@ -137,7 +119,7 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
137
119
  }
138
120
  }
139
121
 
140
- async *entriesAsync(range?: Range<K> | undefined): AsyncIterableIterator<[K, V]> {
122
+ async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
141
123
  for (const entry of this.entries(range)) {
142
124
  yield entry;
143
125
  }
@@ -155,6 +137,15 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
155
137
  }
156
138
  }
157
139
 
140
+ size(): number {
141
+ const iterator = this.db.getRange({ start: this.startSentinel, end: this.endSentinel });
142
+ return iterator.asArray.length;
143
+ }
144
+
145
+ sizeAsync(): Promise<number> {
146
+ return Promise.resolve(this.size());
147
+ }
148
+
158
149
  *keys(range: Range<K> = {}): IterableIterator<K> {
159
150
  for (const [key, _] of this.entries(range)) {
160
151
  yield key;
@@ -184,65 +175,3 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
184
175
  }
185
176
  }
186
177
  }
187
-
188
- export class LmdbAztecMapWithSize<K extends Key, V>
189
- extends LmdbAztecMap<K, V>
190
- implements AztecMapWithSize<K, V>, AztecAsyncMultiMap<K, V>
191
- {
192
- #sizeCache?: number;
193
-
194
- constructor(rootDb: Database, mapName: string) {
195
- super(rootDb, mapName);
196
- }
197
-
198
- override async set(key: K, val: V): Promise<void> {
199
- await this.db.childTransaction(() => {
200
- const exists = this.db.doesExist(this.slot(key));
201
- this.db.putSync(this.slot(key), [key, val], {
202
- appendDup: true,
203
- });
204
- if (!exists) {
205
- this.#sizeCache = undefined; // Invalidate cache
206
- }
207
- });
208
- }
209
-
210
- override async delete(key: K): Promise<void> {
211
- await this.db.childTransaction(async () => {
212
- const exists = this.db.doesExist(this.slot(key));
213
- if (exists) {
214
- await this.db.remove(this.slot(key));
215
- this.#sizeCache = undefined; // Invalidate cache
216
- }
217
- });
218
- }
219
-
220
- override async deleteValue(key: K, val: V): Promise<void> {
221
- await this.db.childTransaction(async () => {
222
- const exists = this.db.doesExist(this.slot(key));
223
- if (exists) {
224
- await this.db.remove(this.slot(key), [key, val]);
225
- this.#sizeCache = undefined; // Invalidate cache
226
- }
227
- });
228
- }
229
-
230
- /**
231
- * Gets the size of the map by counting entries.
232
- * @returns The number of entries in the map
233
- */
234
- size(): number {
235
- if (this.#sizeCache === undefined) {
236
- this.#sizeCache = this.db.getCount({
237
- start: this.startSentinel,
238
- end: this.endSentinel,
239
- });
240
- }
241
- return this.#sizeCache;
242
- }
243
-
244
- // Reset cache on clear/drop operations
245
- clearCache() {
246
- this.#sizeCache = undefined;
247
- }
248
- }
@@ -0,0 +1,51 @@
1
+ import type { Key, Value } from '../interfaces/common.js';
2
+ import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js';
3
+ import { LmdbAztecMap } from './map.js';
4
+
5
+ /**
6
+ * A map backed by LMDB.
7
+ */
8
+ export class LmdbAztecMultiMap<K extends Key, V extends Value>
9
+ extends LmdbAztecMap<K, V>
10
+ implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V>
11
+ {
12
+ *getValues(key: K): IterableIterator<V> {
13
+ const transaction = this.db.useReadTransaction();
14
+ try {
15
+ const values = this.db.getValues(this.slot(key), {
16
+ transaction,
17
+ });
18
+ for (const value of values) {
19
+ yield value?.[1];
20
+ }
21
+ } finally {
22
+ transaction.done();
23
+ }
24
+ }
25
+
26
+ async *getValuesAsync(key: K): AsyncIterableIterator<V> {
27
+ for (const value of this.getValues(key)) {
28
+ yield value;
29
+ }
30
+ }
31
+
32
+ getValueCountAsync(key: K): Promise<number> {
33
+ const transaction = this.db.useReadTransaction();
34
+ try {
35
+ const values = this.db.getValues(this.slot(key), {
36
+ transaction,
37
+ });
38
+ let count = 0;
39
+ for (const _ of values) {
40
+ count++;
41
+ }
42
+ return Promise.resolve(count);
43
+ } finally {
44
+ transaction.done();
45
+ }
46
+ }
47
+
48
+ async deleteValue(key: K, val: V): Promise<void> {
49
+ await this.db.remove(this.slot(key), [key, val]);
50
+ }
51
+ }
package/src/lmdb/store.ts CHANGED
@@ -1,4 +1,4 @@
1
- import { randomBytes } from '@aztec/foundation/crypto';
1
+ import { randomBytes } from '@aztec/foundation/crypto/random';
2
2
  import { createLogger } from '@aztec/foundation/log';
3
3
 
4
4
  import { promises as fs, mkdirSync } from 'fs';
@@ -7,22 +7,17 @@ import { tmpdir } from 'os';
7
7
  import { join } from 'path';
8
8
 
9
9
  import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
10
- import type { Key, StoreSize } from '../interfaces/common.js';
10
+ import type { Key, StoreSize, Value } from '../interfaces/common.js';
11
11
  import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
12
- import type {
13
- AztecAsyncMap,
14
- AztecAsyncMultiMap,
15
- AztecMap,
16
- AztecMapWithSize,
17
- AztecMultiMap,
18
- AztecMultiMapWithSize,
19
- } from '../interfaces/map.js';
12
+ import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js';
13
+ import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js';
20
14
  import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js';
21
15
  import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js';
22
16
  import type { AztecAsyncKVStore, AztecKVStore } from '../interfaces/store.js';
23
17
  import { LmdbAztecArray } from './array.js';
24
18
  import { LmdbAztecCounter } from './counter.js';
25
- import { LmdbAztecMap, LmdbAztecMapWithSize } from './map.js';
19
+ import { LmdbAztecMap } from './map.js';
20
+ import { LmdbAztecMultiMap } from './multi_map.js';
26
21
  import { LmdbAztecSet } from './set.js';
27
22
  import { LmdbAztecSingleton } from './singleton.js';
28
23
 
@@ -37,7 +32,11 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
37
32
  #multiMapData: Database<unknown, Key>;
38
33
  #log = createLogger('kv-store:lmdb');
39
34
 
40
- constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path: string) {
35
+ constructor(
36
+ rootDb: RootDatabase,
37
+ public readonly isEphemeral: boolean,
38
+ private path: string,
39
+ ) {
41
40
  this.#rootDb = rootDb;
42
41
 
43
42
  // big bucket to store all the data
@@ -80,27 +79,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
80
79
  return new AztecLmdbStore(rootDb, ephemeral, dbPath);
81
80
  }
82
81
 
83
- /**
84
- * Forks the current DB into a new DB by backing it up to a temporary location and opening a new lmdb db.
85
- * @returns A new AztecLmdbStore.
86
- */
87
- async fork() {
88
- const baseDir = this.path;
89
- this.#log.debug(`Forking store with basedir ${baseDir}`);
90
- const forkPath = await fs.mkdtemp(join(baseDir, 'aztec-store-fork-'));
91
- this.#log.verbose(`Forking store to ${forkPath}`);
92
- await this.#rootDb.backup(forkPath, false);
93
- const forkDb = open(forkPath, { noSync: this.isEphemeral });
94
- this.#log.debug(`Forked store at ${forkPath} opened successfully`);
95
- return new AztecLmdbStore(forkDb, this.isEphemeral, forkPath);
96
- }
97
-
98
82
  /**
99
83
  * Creates a new AztecMap in the store.
100
84
  * @param name - Name of the map
101
85
  * @returns A new AztecMap
102
86
  */
103
- openMap<K extends Key, V>(name: string): AztecMap<K, V> & AztecAsyncMap<K, V> {
87
+ openMap<K extends Key, V extends Value>(name: string): AztecMap<K, V> & AztecAsyncMap<K, V> {
104
88
  return new LmdbAztecMap(this.#data, name);
105
89
  }
106
90
 
@@ -118,37 +102,20 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
118
102
  * @param name - Name of the map
119
103
  * @returns A new AztecMultiMap
120
104
  */
121
- openMultiMap<K extends Key, V>(name: string): AztecMultiMap<K, V> & AztecAsyncMultiMap<K, V> {
122
- return new LmdbAztecMap(this.#multiMapData, name);
105
+ openMultiMap<K extends Key, V extends Value>(name: string): AztecMultiMap<K, V> & AztecAsyncMultiMap<K, V> {
106
+ return new LmdbAztecMultiMap(this.#multiMapData, name);
123
107
  }
124
108
 
125
109
  openCounter<K extends Key>(name: string): AztecCounter<K> & AztecAsyncCounter<K> {
126
110
  return new LmdbAztecCounter(this.#data, name);
127
111
  }
128
- /**
129
- * Creates a new AztecMultiMapWithSize in the store. A multi-map with size stores multiple values for a single key automatically.
130
- * @param name - Name of the map
131
- * @returns A new AztecMultiMapWithSize
132
- */
133
- openMultiMapWithSize<K extends Key, V>(name: string): AztecMultiMapWithSize<K, V> {
134
- return new LmdbAztecMapWithSize(this.#multiMapData, name);
135
- }
136
-
137
- /**
138
- * Creates a new AztecMapWithSize in the store.
139
- * @param name - Name of the map
140
- * @returns A new AztecMapWithSize
141
- */
142
- openMapWithSize<K extends Key, V>(name: string): AztecMapWithSize<K, V> {
143
- return new LmdbAztecMapWithSize(this.#data, name);
144
- }
145
112
 
146
113
  /**
147
114
  * Creates a new AztecArray in the store.
148
115
  * @param name - Name of the array
149
116
  * @returns A new AztecArray
150
117
  */
151
- openArray<T>(name: string): AztecArray<T> & AztecAsyncArray<T> {
118
+ openArray<T extends Value>(name: string): AztecArray<T> & AztecAsyncArray<T> {
152
119
  return new LmdbAztecArray(this.#data, name);
153
120
  }
154
121
 
@@ -180,21 +147,25 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
180
147
  }
181
148
 
182
149
  /**
183
- * Clears all entries in the store & sub DBs.
150
+ * Clears all entries in the store & sub DBs atomically within a single transaction.
184
151
  */
185
152
  async clear() {
186
- await this.#data.clearAsync();
187
- await this.#multiMapData.clearAsync();
188
- await this.#rootDb.clearAsync();
153
+ await this.#rootDb.transaction(async () => {
154
+ await this.#data.clearAsync();
155
+ await this.#multiMapData.clearAsync();
156
+ await this.#rootDb.clearAsync();
157
+ });
189
158
  }
190
159
 
191
160
  /**
192
- * Drops the database & sub DBs.
161
+ * Drops the database & sub DBs atomically within a single transaction.
193
162
  */
194
163
  async drop() {
195
- await this.#data.drop();
196
- await this.#multiMapData.drop();
197
- await this.#rootDb.drop();
164
+ await this.#rootDb.transaction(async () => {
165
+ await this.#data.drop();
166
+ await this.#multiMapData.drop();
167
+ await this.#rootDb.drop();
168
+ });
198
169
  }
199
170
 
200
171
  /**
@@ -224,10 +195,15 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
224
195
  if ('mapSize' in stats && typeof stats.mapSize === 'number') {
225
196
  mapSize = stats.mapSize;
226
197
  }
198
+ let physicalFileSize = 0;
199
+ if ('physicalFileSize' in stats && typeof stats.physicalFileSize === 'number') {
200
+ physicalFileSize = stats.physicalFileSize;
201
+ }
227
202
  const dataResult = this.estimateSubDBSize(this.#data);
228
203
  const multiResult = this.estimateSubDBSize(this.#multiMapData);
229
204
  return Promise.resolve({
230
205
  mappingSize: mapSize,
206
+ physicalFileSize: physicalFileSize,
231
207
  actualSize: dataResult.actualSize + multiResult.actualSize,
232
208
  numItems: dataResult.numItems + multiResult.numItems,
233
209
  });
@@ -260,4 +236,8 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
260
236
  }
261
237
  return { actualSize, numItems };
262
238
  }
239
+
240
+ backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
241
+ throw new Error('Method not implemented.');
242
+ }
263
243
  }
@@ -1,17 +1,22 @@
1
1
  import { Encoder } from 'msgpackr/pack';
2
2
 
3
3
  import type { AztecAsyncArray } from '../interfaces/array.js';
4
+ import type { Value } from '../interfaces/common.js';
4
5
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
5
6
  import type { ReadTransaction } from './read_transaction.js';
6
- import { AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
7
+ import type { AztecLMDBStoreV2 } from './store.js';
8
+ import { execInReadTx, execInWriteTx } from './tx-helpers.js';
7
9
  import { deserializeKey, serializeKey } from './utils.js';
8
10
 
9
- export class LMDBArray<T> implements AztecAsyncArray<T> {
11
+ export class LMDBArray<T extends Value> implements AztecAsyncArray<T> {
10
12
  private length: AztecAsyncSingleton<number>;
11
13
  private encoder = new Encoder();
12
14
  private prefix: string;
13
15
 
14
- constructor(private store: AztecLMDBStoreV2, name: string) {
16
+ constructor(
17
+ private store: AztecLMDBStoreV2,
18
+ name: string,
19
+ ) {
15
20
  this.length = store.openSingleton(name + ':length');
16
21
  this.prefix = `array:${name}`;
17
22
  }
@@ -1,23 +1,31 @@
1
1
  import { EthAddress } from '@aztec/foundation/eth-address';
2
- import { type Logger, createLogger } from '@aztec/foundation/log';
3
- import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
2
+ import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
3
+ import { DatabaseVersionManager, type SchemaVersionMismatchPolicy } from '@aztec/stdlib/database-version/manager';
4
+ import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
4
5
 
5
- import { mkdir, mkdtemp, rm } from 'fs/promises';
6
+ import { copyFile, mkdir, mkdtemp, rm } from 'fs/promises';
6
7
  import { tmpdir } from 'os';
7
8
  import { join } from 'path';
8
9
 
9
- import type { DataStoreConfig } from '../config.js';
10
10
  import { AztecLMDBStoreV2 } from './store.js';
11
11
 
12
12
  const MAX_READERS = 16;
13
13
 
14
+ /** Optional versioning hooks for persistent LMDB stores. */
15
+ export type CreateStoreOptions = {
16
+ onUpgrade?: (dataDir: string, currentVersion: number, latestVersion: number) => Promise<void>;
17
+ schemaVersionMismatchPolicy?: SchemaVersionMismatchPolicy;
18
+ };
19
+
14
20
  export async function createStore(
15
21
  name: string,
16
22
  schemaVersion: number,
17
23
  config: DataStoreConfig,
18
- log: Logger = createLogger('kv-store:lmdb-v2:' + name),
24
+ bindings?: LoggerBindings,
25
+ options: CreateStoreOptions = {},
19
26
  ): Promise<AztecLMDBStoreV2> {
20
- const { dataDirectory, l1Contracts } = config;
27
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
28
+ const { dataDirectory, rollupAddress: rollupFromConfig } = config;
21
29
 
22
30
  let store: AztecLMDBStoreV2;
23
31
  if (typeof dataDirectory !== 'undefined') {
@@ -25,37 +33,49 @@ export async function createStore(
25
33
  const subDir = join(dataDirectory, name);
26
34
  await mkdir(subDir, { recursive: true });
27
35
 
28
- const rollupAddress = l1Contracts ? l1Contracts.rollupAddress : EthAddress.ZERO;
36
+ const rollupAddress = rollupFromConfig ?? EthAddress.ZERO;
29
37
 
30
38
  // Create a version manager
31
- const versionManager = new DatabaseVersionManager(schemaVersion, rollupAddress, subDir, dbDirectory =>
32
- AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKB, MAX_READERS, () => Promise.resolve(), log),
33
- );
39
+ const versionManager = new DatabaseVersionManager({
40
+ schemaVersion,
41
+ rollupAddress,
42
+ dataDirectory: subDir,
43
+ onOpen: dbDirectory =>
44
+ AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
45
+ onUpgrade: options.onUpgrade,
46
+ schemaVersionMismatchPolicy: options.schemaVersionMismatchPolicy,
47
+ });
34
48
 
35
49
  log.info(
36
- `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKB} KB (LMDB v2)`,
50
+ `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
37
51
  );
38
52
  [store] = await versionManager.open();
39
53
  } else {
40
- store = await openTmpStore(name, true, config.dataStoreMapSizeKB, MAX_READERS, log);
54
+ store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
41
55
  }
42
56
 
43
57
  return store;
44
58
  }
45
59
 
60
+ /**
61
+ * Open a persistent on-disk store rooted in OS tmpdir. Caller chooses whether to
62
+ * auto-remove the directory on close. Uses standard durable LMDB flags (fsync on
63
+ * every commit) — for full in-memory / NOSYNC semantics use `openEphemeralStore`.
64
+ */
46
65
  export async function openTmpStore(
47
66
  name: string,
48
- ephemeral: boolean = true,
67
+ cleanupTmpDir: boolean = true,
49
68
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
50
69
  maxReaders = MAX_READERS,
51
- log: Logger = createLogger('kv-store:lmdb-v2:' + name),
70
+ bindings?: LoggerBindings,
52
71
  ): Promise<AztecLMDBStoreV2> {
72
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
53
73
  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
54
74
  log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
55
75
 
56
76
  // pass a cleanup callback because process.on('beforeExit', cleanup) does not work under Jest
57
77
  const cleanup = async () => {
58
- if (ephemeral) {
78
+ if (cleanupTmpDir) {
59
79
  try {
60
80
  await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
61
81
  log.debug(`Deleted temporary data store: ${dataDir}`);
@@ -67,32 +87,93 @@ export async function openTmpStore(
67
87
  }
68
88
  };
69
89
 
70
- // For temporary stores, we don't need to worry about versioning
71
- // as they are ephemeral and get cleaned up after use
72
- return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, log);
90
+ return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, false);
91
+ }
92
+
93
+ /**
94
+ * Open a fully-ephemeral store: a fresh tmpdir backing file, LMDB opened with
95
+ * MDB_NOSYNC | MDB_NOMETASYNC (no fsync, kernel-only writeback), and the directory
96
+ * unconditionally removed on close. Intended for tests and worker-local scratch
97
+ * stores where durability is explicitly not desired.
98
+ */
99
+ export async function openEphemeralStore(
100
+ name: string,
101
+ dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
102
+ maxReaders = MAX_READERS,
103
+ bindings?: LoggerBindings,
104
+ ): Promise<AztecLMDBStoreV2> {
105
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
106
+ const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
107
+ log.debug(`Created ephemeral data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
108
+
109
+ const cleanup = async () => {
110
+ try {
111
+ await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
112
+ log.debug(`Deleted ephemeral data store: ${dataDir}`);
113
+ } catch (err) {
114
+ log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
115
+ }
116
+ };
117
+
118
+ return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
73
119
  }
74
120
 
75
121
  export async function openStoreAt(
76
122
  dataDir: string,
77
123
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
78
124
  maxReaders = MAX_READERS,
79
- log: Logger = createLogger('kv-store:lmdb-v2'),
125
+ bindings?: LoggerBindings,
80
126
  ): Promise<AztecLMDBStoreV2> {
127
+ const log = createLogger('kv-store:lmdb-v2', bindings);
81
128
  log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
82
- return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log);
129
+ return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
130
+ }
131
+
132
+ /**
133
+ * Open a fully-ephemeral store seeded from an existing `data.mdb` file. Creates a fresh tmpdir,
134
+ * copies `srcDataMdbPath` into it as `data.mdb`, opens with MDB_NOSYNC | MDB_NOMETASYNC, and
135
+ * removes the tmpdir on close. Intended for worker clones of a main-thread-built store — the
136
+ * source file is never written to.
137
+ */
138
+ export async function cloneEphemeralStoreFrom(
139
+ srcDataMdbPath: string,
140
+ name: string,
141
+ dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
142
+ maxReaders = MAX_READERS,
143
+ bindings?: LoggerBindings,
144
+ ): Promise<AztecLMDBStoreV2> {
145
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
146
+ const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
147
+ await copyFile(srcDataMdbPath, join(dataDir, 'data.mdb'));
148
+ log.debug(`Cloned ephemeral data store at: ${dataDir} from ${srcDataMdbPath} (LMDB v2)`);
149
+
150
+ const cleanup = async () => {
151
+ try {
152
+ await rm(dataDir, { recursive: true, force: true, maxRetries: 3 });
153
+ log.debug(`Deleted ephemeral data store: ${dataDir}`);
154
+ } catch (err) {
155
+ log.warn(`Failed to delete ephemeral data directory (LMDB v2) ${dataDir}: ${err}`);
156
+ }
157
+ };
158
+
159
+ return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings, true);
83
160
  }
84
161
 
85
162
  export async function openVersionedStoreAt(
86
- dataDir: string,
163
+ dataDirectory: string,
87
164
  schemaVersion: number,
88
165
  rollupAddress: EthAddress,
89
166
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
90
167
  maxReaders = MAX_READERS,
91
- log: Logger = createLogger('kv-store:lmdb-v2'),
168
+ bindings?: LoggerBindings,
92
169
  ): Promise<AztecLMDBStoreV2> {
93
- log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
94
- const [store] = await new DatabaseVersionManager(schemaVersion, rollupAddress, dataDir, dataDir =>
95
- AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log),
96
- ).open();
170
+ const log = createLogger('kv-store:lmdb-v2', bindings);
171
+ log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
172
+ const [store] = await new DatabaseVersionManager({
173
+ schemaVersion,
174
+ rollupAddress,
175
+ dataDirectory,
176
+ onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
177
+ }).open();
97
178
  return store;
98
179
  }