@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.0208eb9

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/dest/config.d.ts +2 -2
  2. package/dest/config.d.ts.map +1 -1
  3. package/dest/config.js +5 -3
  4. package/dest/indexeddb/array.d.ts +3 -2
  5. package/dest/indexeddb/array.d.ts.map +1 -1
  6. package/dest/indexeddb/array.js +24 -7
  7. package/dest/indexeddb/index.d.ts +2 -2
  8. package/dest/indexeddb/index.d.ts.map +1 -1
  9. package/dest/indexeddb/index.js +4 -7
  10. package/dest/indexeddb/map.d.ts +12 -6
  11. package/dest/indexeddb/map.d.ts.map +1 -1
  12. package/dest/indexeddb/map.js +39 -61
  13. package/dest/indexeddb/multi_map.d.ts +14 -0
  14. package/dest/indexeddb/multi_map.d.ts.map +1 -0
  15. package/dest/indexeddb/multi_map.js +107 -0
  16. package/dest/indexeddb/set.d.ts +1 -1
  17. package/dest/indexeddb/set.d.ts.map +1 -1
  18. package/dest/indexeddb/singleton.d.ts +3 -2
  19. package/dest/indexeddb/singleton.d.ts.map +1 -1
  20. package/dest/indexeddb/singleton.js +6 -2
  21. package/dest/indexeddb/store.d.ts +15 -16
  22. package/dest/indexeddb/store.d.ts.map +1 -1
  23. package/dest/indexeddb/store.js +45 -42
  24. package/dest/interfaces/array.d.ts +5 -4
  25. package/dest/interfaces/array.d.ts.map +1 -1
  26. package/dest/interfaces/array.js +1 -3
  27. package/dest/interfaces/array_test_suite.d.ts +1 -1
  28. package/dest/interfaces/common.d.ts +11 -9
  29. package/dest/interfaces/common.d.ts.map +1 -1
  30. package/dest/interfaces/common.js +8 -3
  31. package/dest/interfaces/counter.d.ts +1 -1
  32. package/dest/interfaces/index.d.ts +4 -2
  33. package/dest/interfaces/index.d.ts.map +1 -1
  34. package/dest/interfaces/index.js +2 -0
  35. package/dest/interfaces/map.d.ts +21 -49
  36. package/dest/interfaces/map.d.ts.map +1 -1
  37. package/dest/interfaces/map.js +1 -1
  38. package/dest/interfaces/map_test_suite.d.ts +1 -1
  39. package/dest/interfaces/map_test_suite.d.ts.map +1 -1
  40. package/dest/interfaces/map_test_suite.js +181 -70
  41. package/dest/interfaces/multi_map.d.ts +41 -0
  42. package/dest/interfaces/multi_map.d.ts.map +1 -0
  43. package/dest/interfaces/multi_map.js +3 -0
  44. package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
  45. package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
  46. package/dest/interfaces/multi_map_test_suite.js +364 -0
  47. package/dest/interfaces/set.d.ts +1 -1
  48. package/dest/interfaces/set_test_suite.d.ts +1 -1
  49. package/dest/interfaces/singleton.d.ts +1 -1
  50. package/dest/interfaces/singleton_test_suite.d.ts +1 -1
  51. package/dest/interfaces/store.d.ts +18 -43
  52. package/dest/interfaces/store.d.ts.map +1 -1
  53. package/dest/interfaces/utils.d.ts +3 -1
  54. package/dest/interfaces/utils.d.ts.map +1 -1
  55. package/dest/interfaces/utils.js +3 -1
  56. package/dest/lmdb/array.d.ts +3 -2
  57. package/dest/lmdb/array.d.ts.map +1 -1
  58. package/dest/lmdb/array.js +4 -2
  59. package/dest/lmdb/counter.d.ts +1 -1
  60. package/dest/lmdb/counter.d.ts.map +1 -1
  61. package/dest/lmdb/index.d.ts +2 -2
  62. package/dest/lmdb/index.d.ts.map +1 -1
  63. package/dest/lmdb/index.js +5 -5
  64. package/dest/lmdb/map.d.ts +11 -23
  65. package/dest/lmdb/map.d.ts.map +1 -1
  66. package/dest/lmdb/map.js +15 -81
  67. package/dest/lmdb/multi_map.d.ts +13 -0
  68. package/dest/lmdb/multi_map.d.ts.map +1 -0
  69. package/dest/lmdb/multi_map.js +44 -0
  70. package/dest/lmdb/set.d.ts +1 -1
  71. package/dest/lmdb/set.d.ts.map +1 -1
  72. package/dest/lmdb/singleton.d.ts +1 -1
  73. package/dest/lmdb/singleton.d.ts.map +1 -1
  74. package/dest/lmdb/store.d.ts +9 -25
  75. package/dest/lmdb/store.d.ts.map +1 -1
  76. package/dest/lmdb/store.js +12 -32
  77. package/dest/lmdb-v2/array.d.ts +4 -3
  78. package/dest/lmdb-v2/array.d.ts.map +1 -1
  79. package/dest/lmdb-v2/array.js +4 -2
  80. package/dest/lmdb-v2/factory.d.ts +6 -9
  81. package/dest/lmdb-v2/factory.d.ts.map +1 -1
  82. package/dest/lmdb-v2/factory.js +26 -12
  83. package/dest/lmdb-v2/index.d.ts +1 -1
  84. package/dest/lmdb-v2/map.d.ts +12 -45
  85. package/dest/lmdb-v2/map.d.ts.map +1 -1
  86. package/dest/lmdb-v2/map.js +17 -104
  87. package/dest/lmdb-v2/message.d.ts +24 -5
  88. package/dest/lmdb-v2/message.d.ts.map +1 -1
  89. package/dest/lmdb-v2/message.js +6 -4
  90. package/dest/lmdb-v2/multi_map.d.ts +52 -0
  91. package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
  92. package/dest/lmdb-v2/multi_map.js +123 -0
  93. package/dest/lmdb-v2/read_transaction.d.ts +3 -1
  94. package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
  95. package/dest/lmdb-v2/read_transaction.js +34 -0
  96. package/dest/lmdb-v2/set.d.ts +15 -0
  97. package/dest/lmdb-v2/set.d.ts.map +1 -0
  98. package/dest/lmdb-v2/set.js +23 -0
  99. package/dest/lmdb-v2/singleton.d.ts +2 -2
  100. package/dest/lmdb-v2/singleton.d.ts.map +1 -1
  101. package/dest/lmdb-v2/singleton.js +1 -1
  102. package/dest/lmdb-v2/store.d.ts +13 -13
  103. package/dest/lmdb-v2/store.d.ts.map +1 -1
  104. package/dest/lmdb-v2/store.js +19 -29
  105. package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
  106. package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
  107. package/dest/lmdb-v2/tx-helpers.js +21 -0
  108. package/dest/lmdb-v2/utils.d.ts +3 -14
  109. package/dest/lmdb-v2/utils.d.ts.map +1 -1
  110. package/dest/lmdb-v2/utils.js +0 -94
  111. package/dest/lmdb-v2/write_transaction.d.ts +3 -5
  112. package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
  113. package/dest/lmdb-v2/write_transaction.js +9 -5
  114. package/dest/stores/index.d.ts +2 -1
  115. package/dest/stores/index.d.ts.map +1 -1
  116. package/dest/stores/index.js +1 -0
  117. package/dest/stores/l2_tips_store.d.ts +24 -8
  118. package/dest/stores/l2_tips_store.d.ts.map +1 -1
  119. package/dest/stores/l2_tips_store.js +63 -45
  120. package/dest/utils.d.ts +9 -6
  121. package/dest/utils.d.ts.map +1 -1
  122. package/dest/utils.js +51 -16
  123. package/package.json +25 -16
  124. package/src/config.ts +6 -4
  125. package/src/indexeddb/array.ts +9 -5
  126. package/src/indexeddb/index.ts +10 -8
  127. package/src/indexeddb/map.ts +38 -55
  128. package/src/indexeddb/multi_map.ts +105 -0
  129. package/src/indexeddb/singleton.ts +5 -2
  130. package/src/indexeddb/store.ts +71 -56
  131. package/src/interfaces/array.ts +5 -3
  132. package/src/interfaces/common.ts +20 -9
  133. package/src/interfaces/index.ts +3 -1
  134. package/src/interfaces/map.ts +19 -53
  135. package/src/interfaces/map_test_suite.ts +101 -44
  136. package/src/interfaces/multi_map.ts +45 -0
  137. package/src/interfaces/multi_map_test_suite.ts +374 -0
  138. package/src/interfaces/store.ts +18 -53
  139. package/src/interfaces/utils.ts +2 -0
  140. package/src/lmdb/array.ts +2 -1
  141. package/src/lmdb/index.ts +11 -6
  142. package/src/lmdb/map.ts +23 -94
  143. package/src/lmdb/multi_map.ts +51 -0
  144. package/src/lmdb/store.ts +24 -48
  145. package/src/lmdb-v2/array.ts +8 -3
  146. package/src/lmdb-v2/factory.ts +29 -18
  147. package/src/lmdb-v2/map.ts +30 -127
  148. package/src/lmdb-v2/message.ts +23 -0
  149. package/src/lmdb-v2/multi_map.ts +150 -0
  150. package/src/lmdb-v2/read_transaction.ts +40 -0
  151. package/src/lmdb-v2/set.ts +33 -0
  152. package/src/lmdb-v2/singleton.ts +6 -2
  153. package/src/lmdb-v2/store.ts +24 -42
  154. package/src/lmdb-v2/tx-helpers.ts +29 -0
  155. package/src/lmdb-v2/utils.ts +0 -118
  156. package/src/lmdb-v2/write_transaction.ts +11 -10
  157. package/src/stores/index.ts +2 -0
  158. package/src/stores/l2_tips_store.ts +67 -49
  159. package/src/utils.ts +79 -21
  160. package/dest/interfaces/store_test_suite.d.ts +0 -3
  161. package/dest/interfaces/store_test_suite.d.ts.map +0 -1
  162. package/dest/interfaces/store_test_suite.js +0 -37
  163. package/src/interfaces/store_test_suite.ts +0 -56
@@ -1,6 +1,6 @@
1
1
  import { EthAddress } from '@aztec/foundation/eth-address';
2
- import { type Logger, createLogger } from '@aztec/foundation/log';
3
- import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
2
+ import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
3
+ import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
4
4
 
5
5
  import { mkdir, mkdtemp, rm } from 'fs/promises';
6
6
  import { tmpdir } from 'os';
@@ -15,8 +15,9 @@ export async function createStore(
15
15
  name: string,
16
16
  schemaVersion: number,
17
17
  config: DataStoreConfig,
18
- log: Logger = createLogger('kv-store:lmdb-v2:' + name),
18
+ bindings?: LoggerBindings,
19
19
  ): Promise<AztecLMDBStoreV2> {
20
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
20
21
  const { dataDirectory, l1Contracts } = config;
21
22
 
22
23
  let store: AztecLMDBStoreV2;
@@ -28,16 +29,20 @@ export async function createStore(
28
29
  const rollupAddress = l1Contracts ? l1Contracts.rollupAddress : EthAddress.ZERO;
29
30
 
30
31
  // Create a version manager
31
- const versionManager = new DatabaseVersionManager(schemaVersion, rollupAddress, subDir, dbDirectory =>
32
- AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKB, MAX_READERS, () => Promise.resolve(), log),
33
- );
32
+ const versionManager = new DatabaseVersionManager({
33
+ schemaVersion,
34
+ rollupAddress,
35
+ dataDirectory: subDir,
36
+ onOpen: dbDirectory =>
37
+ AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
38
+ });
34
39
 
35
40
  log.info(
36
- `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKB} KB (LMDB v2)`,
41
+ `Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
37
42
  );
38
43
  [store] = await versionManager.open();
39
44
  } else {
40
- store = await openTmpStore(name, true, config.dataStoreMapSizeKB, MAX_READERS, log);
45
+ store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
41
46
  }
42
47
 
43
48
  return store;
@@ -48,8 +53,9 @@ export async function openTmpStore(
48
53
  ephemeral: boolean = true,
49
54
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
50
55
  maxReaders = MAX_READERS,
51
- log: Logger = createLogger('kv-store:lmdb-v2:' + name),
56
+ bindings?: LoggerBindings,
52
57
  ): Promise<AztecLMDBStoreV2> {
58
+ const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
53
59
  const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
54
60
  log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
55
61
 
@@ -69,30 +75,35 @@ export async function openTmpStore(
69
75
 
70
76
  // For temporary stores, we don't need to worry about versioning
71
77
  // as they are ephemeral and get cleaned up after use
72
- return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, log);
78
+ return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings);
73
79
  }
74
80
 
75
81
  export async function openStoreAt(
76
82
  dataDir: string,
77
83
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
78
84
  maxReaders = MAX_READERS,
79
- log: Logger = createLogger('kv-store:lmdb-v2'),
85
+ bindings?: LoggerBindings,
80
86
  ): Promise<AztecLMDBStoreV2> {
87
+ const log = createLogger('kv-store:lmdb-v2', bindings);
81
88
  log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
82
- return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log);
89
+ return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
83
90
  }
84
91
 
85
92
  export async function openVersionedStoreAt(
86
- dataDir: string,
93
+ dataDirectory: string,
87
94
  schemaVersion: number,
88
95
  rollupAddress: EthAddress,
89
96
  dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
90
97
  maxReaders = MAX_READERS,
91
- log: Logger = createLogger('kv-store:lmdb-v2'),
98
+ bindings?: LoggerBindings,
92
99
  ): Promise<AztecLMDBStoreV2> {
93
- log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
94
- const [store] = await new DatabaseVersionManager(schemaVersion, rollupAddress, dataDir, dataDir =>
95
- AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log),
96
- ).open();
100
+ const log = createLogger('kv-store:lmdb-v2', bindings);
101
+ log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
102
+ const [store] = await new DatabaseVersionManager({
103
+ schemaVersion,
104
+ rollupAddress,
105
+ dataDirectory,
106
+ onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
107
+ }).open();
97
108
  return store;
98
109
  }
@@ -1,16 +1,20 @@
1
1
  import { Encoder } from 'msgpackr';
2
2
 
3
- import type { Key, Range } from '../interfaces/common.js';
4
- import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
3
+ import type { Key, Range, Value } from '../interfaces/common.js';
4
+ import type { AztecAsyncMap } from '../interfaces/map.js';
5
5
  import type { ReadTransaction } from './read_transaction.js';
6
- import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
6
+ import type { AztecLMDBStoreV2 } from './store.js';
7
+ import { execInReadTx, execInWriteTx } from './tx-helpers.js';
7
8
  import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
8
9
 
9
- export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
10
+ export class LMDBMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
10
11
  private prefix: string;
11
12
  private encoder = new Encoder();
12
13
 
13
- constructor(private store: AztecLMDBStoreV2, name: string) {
14
+ constructor(
15
+ private store: AztecLMDBStoreV2,
16
+ name: string,
17
+ ) {
14
18
  this.prefix = `map:${name}`;
15
19
  }
16
20
  /**
@@ -22,6 +26,18 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
22
26
  return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
23
27
  }
24
28
 
29
+ /**
30
+ * Sets the values at the given keys.
31
+ * @param entries - The entries to set
32
+ */
33
+ async setMany(entries: { key: K; value: V }[]): Promise<void> {
34
+ await execInWriteTx(this.store, async tx => {
35
+ for (const { key, value } of entries) {
36
+ await tx.set(serializeKey(this.prefix, key), this.encoder.pack(value));
37
+ }
38
+ });
39
+ }
40
+
25
41
  /**
26
42
  * Sets the value at the given key if it does not already exist.
27
43
  * @param key - The key to set the value at
@@ -58,15 +74,21 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
58
74
  return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
59
75
  }
60
76
 
77
+ sizeAsync(): Promise<number> {
78
+ return execInReadTx(this.store, tx => tx.countEntries(minKey(this.prefix), maxKey(this.prefix), false));
79
+ }
80
+
61
81
  /**
62
82
  * Iterates over the map's key-value entries in the key's natural order
63
83
  * @param range - The range of keys to iterate over
64
84
  */
65
85
  async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
66
86
  const reverse = range?.reverse ?? false;
67
- const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
68
87
 
69
- const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
88
+ const startKey = range?.start !== undefined ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
89
+
90
+ const endKey =
91
+ range?.end !== undefined ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
70
92
 
71
93
  let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
72
94
  const shouldClose = !tx;
@@ -80,7 +102,7 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
80
102
  range?.limit,
81
103
  )) {
82
104
  const deserializedKey = deserializeKey<K>(this.prefix, key);
83
- if (!deserializedKey) {
105
+ if (deserializedKey === false) {
84
106
  break;
85
107
  }
86
108
  yield [deserializedKey, this.encoder.unpack(val)];
@@ -112,122 +134,3 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
112
134
  }
113
135
  }
114
136
  }
115
-
116
- export class LMDBMultiMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
117
- private prefix: string;
118
- private encoder = new Encoder();
119
- constructor(private store: AztecLMDBStoreV2, name: string) {
120
- this.prefix = `multimap:${name}`;
121
- }
122
-
123
- /**
124
- * Sets the value at the given key.
125
- * @param key - The key to set the value at
126
- * @param val - The value to set
127
- */
128
- set(key: K, val: V): Promise<void> {
129
- return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
130
- }
131
-
132
- /**
133
- * Sets the value at the given key if it does not already exist.
134
- * @param key - The key to set the value at
135
- * @param val - The value to set
136
- */
137
- setIfNotExists(key: K, val: V): Promise<boolean> {
138
- return execInWriteTx(this.store, async tx => {
139
- const exists = !!(await this.getAsync(key));
140
- if (!exists) {
141
- await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
142
- return true;
143
- }
144
- return false;
145
- });
146
- }
147
-
148
- /**
149
- * Deletes the value at the given key.
150
- * @param key - The key to delete the value at
151
- */
152
- delete(key: K): Promise<void> {
153
- return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
154
- }
155
-
156
- getAsync(key: K): Promise<V | undefined> {
157
- return execInReadTx(this.store, async tx => {
158
- const val = await tx.getIndex(serializeKey(this.prefix, key));
159
- return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
160
- });
161
- }
162
-
163
- hasAsync(key: K): Promise<boolean> {
164
- return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
165
- }
166
-
167
- /**
168
- * Iterates over the map's key-value entries in the key's natural order
169
- * @param range - The range of keys to iterate over
170
- */
171
- async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
172
- const reverse = range?.reverse ?? false;
173
- const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
174
- const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
175
-
176
- let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
177
- const shouldClose = !tx;
178
- tx ??= this.store.getReadTx();
179
-
180
- try {
181
- for await (const [key, vals] of tx.iterateIndex(
182
- reverse ? endKey! : startKey,
183
- reverse ? startKey : endKey,
184
- reverse,
185
- range?.limit,
186
- )) {
187
- const deserializedKey = deserializeKey<K>(this.prefix, key);
188
- if (!deserializedKey) {
189
- break;
190
- }
191
-
192
- for (const val of vals) {
193
- yield [deserializedKey, this.encoder.unpack(val)];
194
- }
195
- }
196
- } finally {
197
- if (shouldClose) {
198
- tx.close();
199
- }
200
- }
201
- }
202
-
203
- /**
204
- * Iterates over the map's values in the key's natural order
205
- * @param range - The range of keys to iterate over
206
- */
207
- async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
208
- for await (const [_, value] of this.entriesAsync(range)) {
209
- yield value;
210
- }
211
- }
212
-
213
- /**
214
- * Iterates over the map's keys in the key's natural order
215
- * @param range - The range of keys to iterate over
216
- */
217
- async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
218
- for await (const [key, _] of this.entriesAsync(range)) {
219
- yield key;
220
- }
221
- }
222
-
223
- deleteValue(key: K, val: V | undefined): Promise<void> {
224
- return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
225
- }
226
-
227
- async *getValuesAsync(key: K): AsyncIterableIterator<V> {
228
- const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
229
- for (const value of values) {
230
- yield this.encoder.unpack(value);
231
- }
232
- }
233
- }
@@ -12,6 +12,7 @@ export enum LMDBMessageType {
12
12
 
13
13
  START_CURSOR,
14
14
  ADVANCE_CURSOR,
15
+ ADVANCE_CURSOR_COUNT,
15
16
  CLOSE_CURSOR,
16
17
 
17
18
  BATCH,
@@ -19,6 +20,7 @@ export enum LMDBMessageType {
19
20
  STATS,
20
21
 
21
22
  CLOSE,
23
+ COPY_STORE,
22
24
  }
23
25
 
24
26
  type Key = Uint8Array;
@@ -59,10 +61,20 @@ interface AdvanceCursorRequest {
59
61
  count: number | null;
60
62
  }
61
63
 
64
+ interface AdvanceCursorCountRequest {
65
+ cursor: number;
66
+ endKey: Key;
67
+ }
68
+
62
69
  interface CloseCursorRequest {
63
70
  cursor: number;
64
71
  }
65
72
 
73
+ interface CopyStoreRequest {
74
+ dstPath: string;
75
+ compact: boolean;
76
+ }
77
+
66
78
  export interface Batch {
67
79
  addEntries: Array<KeyValues>;
68
80
  removeEntries: Array<KeyOptionalValues>;
@@ -80,6 +92,7 @@ export type LMDBRequestBody = {
80
92
 
81
93
  [LMDBMessageType.START_CURSOR]: StartCursorRequest;
82
94
  [LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorRequest;
95
+ [LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountRequest;
83
96
  [LMDBMessageType.CLOSE_CURSOR]: CloseCursorRequest;
84
97
 
85
98
  [LMDBMessageType.BATCH]: BatchRequest;
@@ -87,6 +100,7 @@ export type LMDBRequestBody = {
87
100
  [LMDBMessageType.STATS]: void;
88
101
 
89
102
  [LMDBMessageType.CLOSE]: void;
103
+ [LMDBMessageType.COPY_STORE]: CopyStoreRequest;
90
104
  };
91
105
 
92
106
  interface GetResponse {
@@ -107,6 +121,11 @@ interface AdvanceCursorResponse {
107
121
  done: boolean;
108
122
  }
109
123
 
124
+ interface AdvanceCursorCountResponse {
125
+ count: number;
126
+ done: boolean;
127
+ }
128
+
110
129
  interface BatchResponse {
111
130
  durationNs: number;
112
131
  }
@@ -122,6 +141,7 @@ interface StatsResponse {
122
141
  totalUsedSize: bigint | number;
123
142
  }>;
124
143
  dbMapSizeBytes: bigint | number;
144
+ dbPhysicalFileSizeBytes: bigint | number;
125
145
  }
126
146
 
127
147
  export type LMDBResponseBody = {
@@ -132,6 +152,7 @@ export type LMDBResponseBody = {
132
152
 
133
153
  [LMDBMessageType.START_CURSOR]: StartCursorResponse;
134
154
  [LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorResponse;
155
+ [LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountResponse;
135
156
  [LMDBMessageType.CLOSE_CURSOR]: BoolResponse;
136
157
 
137
158
  [LMDBMessageType.BATCH]: BatchResponse;
@@ -139,6 +160,8 @@ export type LMDBResponseBody = {
139
160
  [LMDBMessageType.STATS]: StatsResponse;
140
161
 
141
162
  [LMDBMessageType.CLOSE]: BoolResponse;
163
+
164
+ [LMDBMessageType.COPY_STORE]: BoolResponse;
142
165
  };
143
166
 
144
167
  export interface LMDBMessageChannel {
@@ -0,0 +1,150 @@
1
+ import { Encoder } from 'msgpackr/pack';
2
+ import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
3
+
4
+ import type { Key, Range, Value } from '../interfaces/common.js';
5
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
6
+ import type { ReadTransaction } from './read_transaction.js';
7
+ import type { AztecLMDBStoreV2 } from './store.js';
8
+ import { execInReadTx, execInWriteTx } from './tx-helpers.js';
9
+ import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
10
+
11
+ export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncMultiMap<K, V> {
12
+ private prefix: string;
13
+ private encoder = new Encoder();
14
+ constructor(
15
+ private store: AztecLMDBStoreV2,
16
+ name: string,
17
+ ) {
18
+ this.prefix = `multimap:${name}`;
19
+ }
20
+
21
+ /**
22
+ * Sets the value at the given key.
23
+ * @param key - The key to set the value at
24
+ * @param val - The value to set
25
+ */
26
+ set(key: K, val: V): Promise<void> {
27
+ return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
28
+ }
29
+
30
+ async setMany(entries: { key: K; value: V }[]): Promise<void> {
31
+ await execInWriteTx(this.store, async tx => {
32
+ for (const { key, value } of entries) {
33
+ await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(value));
34
+ }
35
+ });
36
+ }
37
+
38
+ /**
39
+ * Sets the value at the given key if it does not already exist.
40
+ * @param key - The key to set the value at
41
+ * @param val - The value to set
42
+ */
43
+ setIfNotExists(key: K, val: V): Promise<boolean> {
44
+ return execInWriteTx(this.store, async tx => {
45
+ const exists = !!(await this.getAsync(key));
46
+ if (!exists) {
47
+ await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
48
+ return true;
49
+ }
50
+ return false;
51
+ });
52
+ }
53
+
54
+ /**
55
+ * Deletes the value at the given key.
56
+ * @param key - The key to delete the value at
57
+ */
58
+ delete(key: K): Promise<void> {
59
+ return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
60
+ }
61
+
62
+ getAsync(key: K): Promise<V | undefined> {
63
+ return execInReadTx(this.store, async tx => {
64
+ const val = await tx.getIndex(serializeKey(this.prefix, key));
65
+ return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
66
+ });
67
+ }
68
+
69
+ hasAsync(key: K): Promise<boolean> {
70
+ return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
71
+ }
72
+
73
+ sizeAsync(): Promise<number> {
74
+ return execInReadTx(this.store, tx => tx.countEntriesIndex(minKey(this.prefix), maxKey(this.prefix), false));
75
+ }
76
+
77
+ /**
78
+ * Iterates over the map's key-value entries in the key's natural order
79
+ * @param range - The range of keys to iterate over
80
+ */
81
+ async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
82
+ const reverse = range?.reverse ?? false;
83
+ const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
84
+ const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
85
+
86
+ let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
87
+ const shouldClose = !tx;
88
+ tx ??= this.store.getReadTx();
89
+
90
+ try {
91
+ for await (const [key, vals] of tx.iterateIndex(
92
+ reverse ? endKey! : startKey,
93
+ reverse ? startKey : endKey,
94
+ reverse,
95
+ range?.limit,
96
+ )) {
97
+ const deserializedKey = deserializeKey<K>(this.prefix, key);
98
+ if (!deserializedKey) {
99
+ break;
100
+ }
101
+
102
+ for (const val of vals) {
103
+ yield [deserializedKey, this.encoder.unpack(val)];
104
+ }
105
+ }
106
+ } finally {
107
+ if (shouldClose) {
108
+ tx.close();
109
+ }
110
+ }
111
+ }
112
+
113
+ /**
114
+ * Iterates over the map's values in the key's natural order
115
+ * @param range - The range of keys to iterate over
116
+ */
117
+ async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
118
+ for await (const [_, value] of this.entriesAsync(range)) {
119
+ yield value;
120
+ }
121
+ }
122
+
123
+ /**
124
+ * Iterates over the map's keys in the key's natural order
125
+ * @param range - The range of keys to iterate over
126
+ */
127
+ async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
128
+ for await (const [key, _] of this.entriesAsync(range)) {
129
+ yield key;
130
+ }
131
+ }
132
+
133
+ deleteValue(key: K, val: V | undefined): Promise<void> {
134
+ return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
135
+ }
136
+
137
+ async *getValuesAsync(key: K): AsyncIterableIterator<V> {
138
+ const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
139
+ for (const value of values) {
140
+ yield this.encoder.unpack(value);
141
+ }
142
+ }
143
+
144
+ getValueCountAsync(key: K): Promise<number> {
145
+ const startKey = serializeKey(this.prefix, key);
146
+ const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
147
+
148
+ return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
149
+ }
150
+ }
@@ -48,6 +48,14 @@ export class ReadTransaction {
48
48
  yield* this.#iterate(Database.INDEX, startKey, endKey, reverse, limit, vals => vals);
49
49
  }
50
50
 
51
+ public countEntries(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
52
+ return this.#countEntries(Database.DATA, startKey, endKey, reverse);
53
+ }
54
+
55
+ public countEntriesIndex(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
56
+ return this.#countEntries(Database.INDEX, startKey, endKey, reverse);
57
+ }
58
+
51
59
  async *#iterate<T>(
52
60
  db: string,
53
61
  startKey: Uint8Array,
@@ -113,4 +121,36 @@ export class ReadTransaction {
113
121
  }
114
122
  }
115
123
  }
124
+
125
+ async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
126
+ this.assertIsOpen();
127
+
128
+ const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
129
+ key: startKey,
130
+ reverse,
131
+ count: 0,
132
+ onePage: false,
133
+ db,
134
+ });
135
+
136
+ const cursor = response.cursor;
137
+
138
+ try {
139
+ if (!cursor) {
140
+ return 0;
141
+ }
142
+
143
+ const advanceResponse = await this.channel.sendMessage(LMDBMessageType.ADVANCE_CURSOR_COUNT, {
144
+ cursor,
145
+ endKey: endKey,
146
+ });
147
+
148
+ return advanceResponse.count;
149
+ } finally {
150
+ // we might not have anything to close
151
+ if (typeof cursor === 'number') {
152
+ await this.channel.sendMessage(LMDBMessageType.CLOSE_CURSOR, { cursor });
153
+ }
154
+ }
155
+ }
116
156
  }
@@ -0,0 +1,33 @@
1
+ import type { Key, Range } from '../interfaces/common.js';
2
+ import type { AztecAsyncSet } from '../interfaces/set.js';
3
+ import { LMDBMap } from './map.js';
4
+ import type { AztecLMDBStoreV2 } from './store.js';
5
+
6
+ /**
7
+ * A set backed by LMDB.
8
+ */
9
+ export class LMDBSet<K extends Key> implements AztecAsyncSet<K> {
10
+ private map: LMDBMap<K, boolean>;
11
+
12
+ constructor(store: AztecLMDBStoreV2, mapName: string) {
13
+ this.map = new LMDBMap(store, mapName);
14
+ }
15
+
16
+ hasAsync(key: K): Promise<boolean> {
17
+ return this.map.hasAsync(key);
18
+ }
19
+
20
+ add(key: K): Promise<void> {
21
+ return this.map.set(key, true);
22
+ }
23
+
24
+ delete(key: K): Promise<void> {
25
+ return this.map.delete(key);
26
+ }
27
+
28
+ async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
29
+ for await (const key of this.map.keysAsync(range)) {
30
+ yield key;
31
+ }
32
+ }
33
+ }
@@ -1,13 +1,17 @@
1
1
  import { Encoder } from 'msgpackr';
2
2
 
3
3
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
4
- import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
4
+ import type { AztecLMDBStoreV2 } from './store.js';
5
+ import { execInReadTx, execInWriteTx } from './tx-helpers.js';
5
6
  import { serializeKey } from './utils.js';
6
7
 
7
8
  export class LMDBSingleValue<T> implements AztecAsyncSingleton<T> {
8
9
  private key: Uint8Array;
9
10
  private encoder = new Encoder();
10
- constructor(private store: AztecLMDBStoreV2, name: string) {
11
+ constructor(
12
+ private store: AztecLMDBStoreV2,
13
+ name: string,
14
+ ) {
11
15
  this.key = serializeKey(`singleton:${name}`, 'value');
12
16
  }
13
17