@aztec/kv-store 0.0.0-test.0 → 0.0.1-commit.001888fc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/indexeddb/array.d.ts +3 -2
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +24 -7
- package/dest/indexeddb/index.d.ts +3 -3
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/index.js +4 -7
- package/dest/indexeddb/map.d.ts +18 -6
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +51 -63
- package/dest/indexeddb/multi_map.d.ts +14 -0
- package/dest/indexeddb/multi_map.d.ts.map +1 -0
- package/dest/indexeddb/multi_map.js +107 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +3 -2
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +6 -2
- package/dest/indexeddb/store.d.ts +15 -16
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +51 -46
- package/dest/interfaces/array.d.ts +5 -4
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array.js +1 -3
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +11 -9
- package/dest/interfaces/common.d.ts.map +1 -1
- package/dest/interfaces/common.js +8 -3
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +4 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/index.js +2 -0
- package/dest/interfaces/map.d.ts +21 -49
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map.js +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +181 -70
- package/dest/interfaces/multi_map.d.ts +41 -0
- package/dest/interfaces/multi_map.d.ts.map +1 -0
- package/dest/interfaces/multi_map.js +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/multi_map_test_suite.js +364 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +18 -43
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/utils.d.ts +3 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +3 -1
- package/dest/lmdb/array.d.ts +3 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +4 -2
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +3 -3
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +5 -5
- package/dest/lmdb/map.d.ts +11 -23
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +15 -81
- package/dest/lmdb/multi_map.d.ts +13 -0
- package/dest/lmdb/multi_map.d.ts.map +1 -0
- package/dest/lmdb/multi_map.js +44 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +11 -27
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +24 -40
- package/dest/lmdb-v2/array.d.ts +4 -3
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +4 -2
- package/dest/lmdb-v2/factory.d.ts +7 -10
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +26 -12
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +12 -45
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +17 -104
- package/dest/lmdb-v2/message.d.ts +24 -5
- package/dest/lmdb-v2/message.d.ts.map +1 -1
- package/dest/lmdb-v2/message.js +6 -4
- package/dest/lmdb-v2/multi_map.d.ts +52 -0
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
- package/dest/lmdb-v2/multi_map.js +123 -0
- package/dest/lmdb-v2/read_transaction.d.ts +3 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +34 -0
- package/dest/lmdb-v2/set.d.ts +15 -0
- package/dest/lmdb-v2/set.d.ts.map +1 -0
- package/dest/lmdb-v2/set.js +23 -0
- package/dest/lmdb-v2/singleton.d.ts +2 -2
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -1
- package/dest/lmdb-v2/store.d.ts +13 -13
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +19 -29
- package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
- package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
- package/dest/lmdb-v2/tx-helpers.js +21 -0
- package/dest/lmdb-v2/utils.d.ts +3 -14
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.js +0 -94
- package/dest/lmdb-v2/write_transaction.d.ts +3 -5
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.js +9 -5
- package/dest/stores/index.d.ts +2 -1
- package/dest/stores/index.d.ts.map +1 -1
- package/dest/stores/index.js +1 -0
- package/dest/stores/l2_tips_store.d.ts +24 -8
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +63 -45
- package/dest/utils.d.ts +9 -6
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +51 -16
- package/package.json +30 -22
- package/src/indexeddb/array.ts +9 -5
- package/src/indexeddb/index.ts +11 -9
- package/src/indexeddb/map.ts +51 -56
- package/src/indexeddb/multi_map.ts +105 -0
- package/src/indexeddb/singleton.ts +5 -2
- package/src/indexeddb/store.ts +77 -60
- package/src/interfaces/array.ts +5 -3
- package/src/interfaces/common.ts +20 -9
- package/src/interfaces/index.ts +3 -1
- package/src/interfaces/map.ts +19 -53
- package/src/interfaces/map_test_suite.ts +101 -44
- package/src/interfaces/multi_map.ts +45 -0
- package/src/interfaces/multi_map_test_suite.ts +374 -0
- package/src/interfaces/store.ts +18 -53
- package/src/interfaces/utils.ts +2 -0
- package/src/lmdb/array.ts +2 -1
- package/src/lmdb/index.ts +12 -7
- package/src/lmdb/map.ts +23 -94
- package/src/lmdb/multi_map.ts +51 -0
- package/src/lmdb/store.ts +36 -56
- package/src/lmdb-v2/array.ts +8 -3
- package/src/lmdb-v2/factory.ts +30 -19
- package/src/lmdb-v2/map.ts +30 -127
- package/src/lmdb-v2/message.ts +23 -0
- package/src/lmdb-v2/multi_map.ts +150 -0
- package/src/lmdb-v2/read_transaction.ts +40 -0
- package/src/lmdb-v2/set.ts +33 -0
- package/src/lmdb-v2/singleton.ts +6 -2
- package/src/lmdb-v2/store.ts +24 -42
- package/src/lmdb-v2/tx-helpers.ts +29 -0
- package/src/lmdb-v2/utils.ts +0 -118
- package/src/lmdb-v2/write_transaction.ts +11 -10
- package/src/stores/index.ts +2 -0
- package/src/stores/l2_tips_store.ts +67 -49
- package/src/utils.ts +79 -21
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -24
- package/dest/interfaces/store_test_suite.d.ts +0 -3
- package/dest/interfaces/store_test_suite.d.ts.map +0 -1
- package/dest/interfaces/store_test_suite.js +0 -37
- package/src/config.ts +0 -34
- package/src/interfaces/store_test_suite.ts +0 -56
package/src/lmdb-v2/map.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
|
-
import type { Key, Range } from '../interfaces/common.js';
|
|
4
|
-
import type { AztecAsyncMap
|
|
3
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
4
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
5
5
|
import type { ReadTransaction } from './read_transaction.js';
|
|
6
|
-
import {
|
|
6
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
7
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
7
8
|
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
8
9
|
|
|
9
|
-
export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
10
|
+
export class LMDBMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
|
|
10
11
|
private prefix: string;
|
|
11
12
|
private encoder = new Encoder();
|
|
12
13
|
|
|
13
|
-
constructor(
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
14
18
|
this.prefix = `map:${name}`;
|
|
15
19
|
}
|
|
16
20
|
/**
|
|
@@ -22,6 +26,18 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
22
26
|
return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
23
27
|
}
|
|
24
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Sets the values at the given keys.
|
|
31
|
+
* @param entries - The entries to set
|
|
32
|
+
*/
|
|
33
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
34
|
+
await execInWriteTx(this.store, async tx => {
|
|
35
|
+
for (const { key, value } of entries) {
|
|
36
|
+
await tx.set(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
25
41
|
/**
|
|
26
42
|
* Sets the value at the given key if it does not already exist.
|
|
27
43
|
* @param key - The key to set the value at
|
|
@@ -58,15 +74,21 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
58
74
|
return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
|
|
59
75
|
}
|
|
60
76
|
|
|
77
|
+
sizeAsync(): Promise<number> {
|
|
78
|
+
return execInReadTx(this.store, tx => tx.countEntries(minKey(this.prefix), maxKey(this.prefix), false));
|
|
79
|
+
}
|
|
80
|
+
|
|
61
81
|
/**
|
|
62
82
|
* Iterates over the map's key-value entries in the key's natural order
|
|
63
83
|
* @param range - The range of keys to iterate over
|
|
64
84
|
*/
|
|
65
85
|
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
66
86
|
const reverse = range?.reverse ?? false;
|
|
67
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
68
87
|
|
|
69
|
-
const
|
|
88
|
+
const startKey = range?.start !== undefined ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
89
|
+
|
|
90
|
+
const endKey =
|
|
91
|
+
range?.end !== undefined ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
70
92
|
|
|
71
93
|
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
72
94
|
const shouldClose = !tx;
|
|
@@ -80,7 +102,7 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
80
102
|
range?.limit,
|
|
81
103
|
)) {
|
|
82
104
|
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
83
|
-
if (
|
|
105
|
+
if (deserializedKey === false) {
|
|
84
106
|
break;
|
|
85
107
|
}
|
|
86
108
|
yield [deserializedKey, this.encoder.unpack(val)];
|
|
@@ -112,122 +134,3 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
112
134
|
}
|
|
113
135
|
}
|
|
114
136
|
}
|
|
115
|
-
|
|
116
|
-
export class LMDBMultiMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
|
|
117
|
-
private prefix: string;
|
|
118
|
-
private encoder = new Encoder();
|
|
119
|
-
constructor(private store: AztecLMDBStoreV2, name: string) {
|
|
120
|
-
this.prefix = `multimap:${name}`;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Sets the value at the given key.
|
|
125
|
-
* @param key - The key to set the value at
|
|
126
|
-
* @param val - The value to set
|
|
127
|
-
*/
|
|
128
|
-
set(key: K, val: V): Promise<void> {
|
|
129
|
-
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Sets the value at the given key if it does not already exist.
|
|
134
|
-
* @param key - The key to set the value at
|
|
135
|
-
* @param val - The value to set
|
|
136
|
-
*/
|
|
137
|
-
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
138
|
-
return execInWriteTx(this.store, async tx => {
|
|
139
|
-
const exists = !!(await this.getAsync(key));
|
|
140
|
-
if (!exists) {
|
|
141
|
-
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
142
|
-
return true;
|
|
143
|
-
}
|
|
144
|
-
return false;
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Deletes the value at the given key.
|
|
150
|
-
* @param key - The key to delete the value at
|
|
151
|
-
*/
|
|
152
|
-
delete(key: K): Promise<void> {
|
|
153
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
getAsync(key: K): Promise<V | undefined> {
|
|
157
|
-
return execInReadTx(this.store, async tx => {
|
|
158
|
-
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
159
|
-
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
hasAsync(key: K): Promise<boolean> {
|
|
164
|
-
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Iterates over the map's key-value entries in the key's natural order
|
|
169
|
-
* @param range - The range of keys to iterate over
|
|
170
|
-
*/
|
|
171
|
-
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
172
|
-
const reverse = range?.reverse ?? false;
|
|
173
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
174
|
-
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
175
|
-
|
|
176
|
-
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
177
|
-
const shouldClose = !tx;
|
|
178
|
-
tx ??= this.store.getReadTx();
|
|
179
|
-
|
|
180
|
-
try {
|
|
181
|
-
for await (const [key, vals] of tx.iterateIndex(
|
|
182
|
-
reverse ? endKey! : startKey,
|
|
183
|
-
reverse ? startKey : endKey,
|
|
184
|
-
reverse,
|
|
185
|
-
range?.limit,
|
|
186
|
-
)) {
|
|
187
|
-
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
188
|
-
if (!deserializedKey) {
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
for (const val of vals) {
|
|
193
|
-
yield [deserializedKey, this.encoder.unpack(val)];
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
} finally {
|
|
197
|
-
if (shouldClose) {
|
|
198
|
-
tx.close();
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/**
|
|
204
|
-
* Iterates over the map's values in the key's natural order
|
|
205
|
-
* @param range - The range of keys to iterate over
|
|
206
|
-
*/
|
|
207
|
-
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
208
|
-
for await (const [_, value] of this.entriesAsync(range)) {
|
|
209
|
-
yield value;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Iterates over the map's keys in the key's natural order
|
|
215
|
-
* @param range - The range of keys to iterate over
|
|
216
|
-
*/
|
|
217
|
-
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
218
|
-
for await (const [key, _] of this.entriesAsync(range)) {
|
|
219
|
-
yield key;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
224
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
228
|
-
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
229
|
-
for (const value of values) {
|
|
230
|
-
yield this.encoder.unpack(value);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
}
|
package/src/lmdb-v2/message.ts
CHANGED
|
@@ -12,6 +12,7 @@ export enum LMDBMessageType {
|
|
|
12
12
|
|
|
13
13
|
START_CURSOR,
|
|
14
14
|
ADVANCE_CURSOR,
|
|
15
|
+
ADVANCE_CURSOR_COUNT,
|
|
15
16
|
CLOSE_CURSOR,
|
|
16
17
|
|
|
17
18
|
BATCH,
|
|
@@ -19,6 +20,7 @@ export enum LMDBMessageType {
|
|
|
19
20
|
STATS,
|
|
20
21
|
|
|
21
22
|
CLOSE,
|
|
23
|
+
COPY_STORE,
|
|
22
24
|
}
|
|
23
25
|
|
|
24
26
|
type Key = Uint8Array;
|
|
@@ -59,10 +61,20 @@ interface AdvanceCursorRequest {
|
|
|
59
61
|
count: number | null;
|
|
60
62
|
}
|
|
61
63
|
|
|
64
|
+
interface AdvanceCursorCountRequest {
|
|
65
|
+
cursor: number;
|
|
66
|
+
endKey: Key;
|
|
67
|
+
}
|
|
68
|
+
|
|
62
69
|
interface CloseCursorRequest {
|
|
63
70
|
cursor: number;
|
|
64
71
|
}
|
|
65
72
|
|
|
73
|
+
interface CopyStoreRequest {
|
|
74
|
+
dstPath: string;
|
|
75
|
+
compact: boolean;
|
|
76
|
+
}
|
|
77
|
+
|
|
66
78
|
export interface Batch {
|
|
67
79
|
addEntries: Array<KeyValues>;
|
|
68
80
|
removeEntries: Array<KeyOptionalValues>;
|
|
@@ -80,6 +92,7 @@ export type LMDBRequestBody = {
|
|
|
80
92
|
|
|
81
93
|
[LMDBMessageType.START_CURSOR]: StartCursorRequest;
|
|
82
94
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorRequest;
|
|
95
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountRequest;
|
|
83
96
|
[LMDBMessageType.CLOSE_CURSOR]: CloseCursorRequest;
|
|
84
97
|
|
|
85
98
|
[LMDBMessageType.BATCH]: BatchRequest;
|
|
@@ -87,6 +100,7 @@ export type LMDBRequestBody = {
|
|
|
87
100
|
[LMDBMessageType.STATS]: void;
|
|
88
101
|
|
|
89
102
|
[LMDBMessageType.CLOSE]: void;
|
|
103
|
+
[LMDBMessageType.COPY_STORE]: CopyStoreRequest;
|
|
90
104
|
};
|
|
91
105
|
|
|
92
106
|
interface GetResponse {
|
|
@@ -107,6 +121,11 @@ interface AdvanceCursorResponse {
|
|
|
107
121
|
done: boolean;
|
|
108
122
|
}
|
|
109
123
|
|
|
124
|
+
interface AdvanceCursorCountResponse {
|
|
125
|
+
count: number;
|
|
126
|
+
done: boolean;
|
|
127
|
+
}
|
|
128
|
+
|
|
110
129
|
interface BatchResponse {
|
|
111
130
|
durationNs: number;
|
|
112
131
|
}
|
|
@@ -122,6 +141,7 @@ interface StatsResponse {
|
|
|
122
141
|
totalUsedSize: bigint | number;
|
|
123
142
|
}>;
|
|
124
143
|
dbMapSizeBytes: bigint | number;
|
|
144
|
+
dbPhysicalFileSizeBytes: bigint | number;
|
|
125
145
|
}
|
|
126
146
|
|
|
127
147
|
export type LMDBResponseBody = {
|
|
@@ -132,6 +152,7 @@ export type LMDBResponseBody = {
|
|
|
132
152
|
|
|
133
153
|
[LMDBMessageType.START_CURSOR]: StartCursorResponse;
|
|
134
154
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorResponse;
|
|
155
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountResponse;
|
|
135
156
|
[LMDBMessageType.CLOSE_CURSOR]: BoolResponse;
|
|
136
157
|
|
|
137
158
|
[LMDBMessageType.BATCH]: BatchResponse;
|
|
@@ -139,6 +160,8 @@ export type LMDBResponseBody = {
|
|
|
139
160
|
[LMDBMessageType.STATS]: StatsResponse;
|
|
140
161
|
|
|
141
162
|
[LMDBMessageType.CLOSE]: BoolResponse;
|
|
163
|
+
|
|
164
|
+
[LMDBMessageType.COPY_STORE]: BoolResponse;
|
|
142
165
|
};
|
|
143
166
|
|
|
144
167
|
export interface LMDBMessageChannel {
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { Encoder } from 'msgpackr/pack';
|
|
2
|
+
import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
|
|
3
|
+
|
|
4
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
6
|
+
import type { ReadTransaction } from './read_transaction.js';
|
|
7
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
8
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
9
|
+
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
10
|
+
|
|
11
|
+
export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncMultiMap<K, V> {
|
|
12
|
+
private prefix: string;
|
|
13
|
+
private encoder = new Encoder();
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
18
|
+
this.prefix = `multimap:${name}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Sets the value at the given key.
|
|
23
|
+
* @param key - The key to set the value at
|
|
24
|
+
* @param val - The value to set
|
|
25
|
+
*/
|
|
26
|
+
set(key: K, val: V): Promise<void> {
|
|
27
|
+
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
31
|
+
await execInWriteTx(this.store, async tx => {
|
|
32
|
+
for (const { key, value } of entries) {
|
|
33
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Sets the value at the given key if it does not already exist.
|
|
40
|
+
* @param key - The key to set the value at
|
|
41
|
+
* @param val - The value to set
|
|
42
|
+
*/
|
|
43
|
+
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
44
|
+
return execInWriteTx(this.store, async tx => {
|
|
45
|
+
const exists = !!(await this.getAsync(key));
|
|
46
|
+
if (!exists) {
|
|
47
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return false;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Deletes the value at the given key.
|
|
56
|
+
* @param key - The key to delete the value at
|
|
57
|
+
*/
|
|
58
|
+
delete(key: K): Promise<void> {
|
|
59
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
getAsync(key: K): Promise<V | undefined> {
|
|
63
|
+
return execInReadTx(this.store, async tx => {
|
|
64
|
+
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
65
|
+
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
hasAsync(key: K): Promise<boolean> {
|
|
70
|
+
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
sizeAsync(): Promise<number> {
|
|
74
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(minKey(this.prefix), maxKey(this.prefix), false));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Iterates over the map's key-value entries in the key's natural order
|
|
79
|
+
* @param range - The range of keys to iterate over
|
|
80
|
+
*/
|
|
81
|
+
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
82
|
+
const reverse = range?.reverse ?? false;
|
|
83
|
+
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
84
|
+
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
85
|
+
|
|
86
|
+
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
87
|
+
const shouldClose = !tx;
|
|
88
|
+
tx ??= this.store.getReadTx();
|
|
89
|
+
|
|
90
|
+
try {
|
|
91
|
+
for await (const [key, vals] of tx.iterateIndex(
|
|
92
|
+
reverse ? endKey! : startKey,
|
|
93
|
+
reverse ? startKey : endKey,
|
|
94
|
+
reverse,
|
|
95
|
+
range?.limit,
|
|
96
|
+
)) {
|
|
97
|
+
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
98
|
+
if (!deserializedKey) {
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const val of vals) {
|
|
103
|
+
yield [deserializedKey, this.encoder.unpack(val)];
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
} finally {
|
|
107
|
+
if (shouldClose) {
|
|
108
|
+
tx.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Iterates over the map's values in the key's natural order
|
|
115
|
+
* @param range - The range of keys to iterate over
|
|
116
|
+
*/
|
|
117
|
+
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
118
|
+
for await (const [_, value] of this.entriesAsync(range)) {
|
|
119
|
+
yield value;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Iterates over the map's keys in the key's natural order
|
|
125
|
+
* @param range - The range of keys to iterate over
|
|
126
|
+
*/
|
|
127
|
+
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
128
|
+
for await (const [key, _] of this.entriesAsync(range)) {
|
|
129
|
+
yield key;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
134
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
138
|
+
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
139
|
+
for (const value of values) {
|
|
140
|
+
yield this.encoder.unpack(value);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
145
|
+
const startKey = serializeKey(this.prefix, key);
|
|
146
|
+
const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
|
|
147
|
+
|
|
148
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -48,6 +48,14 @@ export class ReadTransaction {
|
|
|
48
48
|
yield* this.#iterate(Database.INDEX, startKey, endKey, reverse, limit, vals => vals);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
public countEntries(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
52
|
+
return this.#countEntries(Database.DATA, startKey, endKey, reverse);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
public countEntriesIndex(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
56
|
+
return this.#countEntries(Database.INDEX, startKey, endKey, reverse);
|
|
57
|
+
}
|
|
58
|
+
|
|
51
59
|
async *#iterate<T>(
|
|
52
60
|
db: string,
|
|
53
61
|
startKey: Uint8Array,
|
|
@@ -113,4 +121,36 @@ export class ReadTransaction {
|
|
|
113
121
|
}
|
|
114
122
|
}
|
|
115
123
|
}
|
|
124
|
+
|
|
125
|
+
async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
126
|
+
this.assertIsOpen();
|
|
127
|
+
|
|
128
|
+
const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
|
|
129
|
+
key: startKey,
|
|
130
|
+
reverse,
|
|
131
|
+
count: 0,
|
|
132
|
+
onePage: false,
|
|
133
|
+
db,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const cursor = response.cursor;
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
if (!cursor) {
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const advanceResponse = await this.channel.sendMessage(LMDBMessageType.ADVANCE_CURSOR_COUNT, {
|
|
144
|
+
cursor,
|
|
145
|
+
endKey: endKey,
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
return advanceResponse.count;
|
|
149
|
+
} finally {
|
|
150
|
+
// we might not have anything to close
|
|
151
|
+
if (typeof cursor === 'number') {
|
|
152
|
+
await this.channel.sendMessage(LMDBMessageType.CLOSE_CURSOR, { cursor });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
116
156
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Key, Range } from '../interfaces/common.js';
|
|
2
|
+
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
3
|
+
import { LMDBMap } from './map.js';
|
|
4
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A set backed by LMDB.
|
|
8
|
+
*/
|
|
9
|
+
export class LMDBSet<K extends Key> implements AztecAsyncSet<K> {
|
|
10
|
+
private map: LMDBMap<K, boolean>;
|
|
11
|
+
|
|
12
|
+
constructor(store: AztecLMDBStoreV2, mapName: string) {
|
|
13
|
+
this.map = new LMDBMap(store, mapName);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
hasAsync(key: K): Promise<boolean> {
|
|
17
|
+
return this.map.hasAsync(key);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
add(key: K): Promise<void> {
|
|
21
|
+
return this.map.set(key, true);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
delete(key: K): Promise<void> {
|
|
25
|
+
return this.map.delete(key);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
29
|
+
for await (const key of this.map.keysAsync(range)) {
|
|
30
|
+
yield key;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
package/src/lmdb-v2/singleton.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
3
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
4
|
-
import {
|
|
4
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
5
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
5
6
|
import { serializeKey } from './utils.js';
|
|
6
7
|
|
|
7
8
|
export class LMDBSingleValue<T> implements AztecAsyncSingleton<T> {
|
|
8
9
|
private key: Uint8Array;
|
|
9
10
|
private encoder = new Encoder();
|
|
10
|
-
constructor(
|
|
11
|
+
constructor(
|
|
12
|
+
private store: AztecLMDBStoreV2,
|
|
13
|
+
name: string,
|
|
14
|
+
) {
|
|
11
15
|
this.key = serializeKey(`singleton:${name}`, 'value');
|
|
12
16
|
}
|
|
13
17
|
|
package/src/lmdb-v2/store.ts
CHANGED
|
@@ -1,19 +1,20 @@
|
|
|
1
|
-
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
1
|
+
import { type Logger, type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
2
2
|
import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
|
|
3
3
|
import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
|
|
4
4
|
|
|
5
5
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
6
|
-
import { rm } from 'fs/promises';
|
|
6
|
+
import { mkdir, rm } from 'fs/promises';
|
|
7
7
|
|
|
8
8
|
import type { AztecAsyncArray } from '../interfaces/array.js';
|
|
9
|
-
import type { Key, StoreSize } from '../interfaces/common.js';
|
|
9
|
+
import type { Key, StoreSize, Value } from '../interfaces/common.js';
|
|
10
10
|
import type { AztecAsyncCounter } from '../interfaces/counter.js';
|
|
11
|
-
import type { AztecAsyncMap
|
|
11
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
12
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
12
13
|
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
13
14
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
14
15
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
15
16
|
import { LMDBArray } from './array.js';
|
|
16
|
-
import { LMDBMap
|
|
17
|
+
import { LMDBMap } from './map.js';
|
|
17
18
|
import {
|
|
18
19
|
Database,
|
|
19
20
|
type LMDBMessageChannel,
|
|
@@ -21,10 +22,14 @@ import {
|
|
|
21
22
|
type LMDBRequestBody,
|
|
22
23
|
type LMDBResponseBody,
|
|
23
24
|
} from './message.js';
|
|
25
|
+
import { LMDBMultiMap } from './multi_map.js';
|
|
24
26
|
import { ReadTransaction } from './read_transaction.js';
|
|
27
|
+
import { LMDBSet } from './set.js';
|
|
25
28
|
import { LMDBSingleValue } from './singleton.js';
|
|
26
29
|
import { WriteTransaction } from './write_transaction.js';
|
|
27
30
|
|
|
31
|
+
export { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
32
|
+
|
|
28
33
|
export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
29
34
|
private open = false;
|
|
30
35
|
private channel: MsgpackChannel<LMDBMessageType, LMDBRequestBody, LMDBResponseBody>;
|
|
@@ -70,13 +75,19 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
70
75
|
dbMapSizeKb: number = 10 * 1024 * 1024,
|
|
71
76
|
maxReaders: number = 16,
|
|
72
77
|
cleanup?: () => Promise<void>,
|
|
73
|
-
|
|
78
|
+
bindings?: LoggerBindings,
|
|
74
79
|
) {
|
|
80
|
+
const log = createLogger('kv-store:lmdb-v2', bindings);
|
|
75
81
|
const db = new AztecLMDBStoreV2(dataDir, dbMapSizeKb, maxReaders, log, cleanup);
|
|
76
82
|
await db.start();
|
|
77
83
|
return db;
|
|
78
84
|
}
|
|
79
85
|
|
|
86
|
+
public async backupTo(dstPath: string, compact = true) {
|
|
87
|
+
await mkdir(dstPath, { recursive: true });
|
|
88
|
+
await this.channel.sendMessage(LMDBMessageType.COPY_STORE, { dstPath, compact });
|
|
89
|
+
}
|
|
90
|
+
|
|
80
91
|
public getReadTx(): ReadTransaction {
|
|
81
92
|
if (!this.open) {
|
|
82
93
|
throw new Error('Store is closed');
|
|
@@ -92,24 +103,24 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
92
103
|
return currentWrite;
|
|
93
104
|
}
|
|
94
105
|
|
|
95
|
-
openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
|
|
106
|
+
openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
|
|
96
107
|
return new LMDBMap(this, name);
|
|
97
108
|
}
|
|
98
109
|
|
|
99
|
-
openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
|
|
110
|
+
openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
|
|
100
111
|
return new LMDBMultiMap(this, name);
|
|
101
112
|
}
|
|
102
113
|
|
|
103
|
-
openSingleton<T>(name: string): AztecAsyncSingleton<T> {
|
|
114
|
+
openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
|
|
104
115
|
return new LMDBSingleValue(this, name);
|
|
105
116
|
}
|
|
106
117
|
|
|
107
|
-
openArray<T>(name: string): AztecAsyncArray<T> {
|
|
118
|
+
openArray<T extends Value>(name: string): AztecAsyncArray<T> {
|
|
108
119
|
return new LMDBArray(this, name);
|
|
109
120
|
}
|
|
110
121
|
|
|
111
|
-
openSet<K extends Key>(
|
|
112
|
-
|
|
122
|
+
openSet<K extends Key>(name: string): AztecAsyncSet<K> {
|
|
123
|
+
return new LMDBSet(this, name);
|
|
113
124
|
}
|
|
114
125
|
|
|
115
126
|
openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
|
|
@@ -150,10 +161,6 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
150
161
|
return Promise.resolve();
|
|
151
162
|
}
|
|
152
163
|
|
|
153
|
-
fork(): Promise<AztecAsyncKVStore> {
|
|
154
|
-
throw new Error('Not implemented');
|
|
155
|
-
}
|
|
156
|
-
|
|
157
164
|
async delete(): Promise<void> {
|
|
158
165
|
await this.close();
|
|
159
166
|
await rm(this.dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
@@ -204,34 +211,9 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
204
211
|
const resp = await this.sendMessage(LMDBMessageType.STATS, undefined);
|
|
205
212
|
return {
|
|
206
213
|
mappingSize: Number(resp.dbMapSizeBytes),
|
|
214
|
+
physicalFileSize: Number(resp.dbPhysicalFileSizeBytes),
|
|
207
215
|
actualSize: resp.stats.reduce((s, db) => Number(db.totalUsedSize) + s, 0),
|
|
208
216
|
numItems: resp.stats.reduce((s, db) => Number(db.numDataItems) + s, 0),
|
|
209
217
|
};
|
|
210
218
|
}
|
|
211
219
|
}
|
|
212
|
-
|
|
213
|
-
export function execInWriteTx<T>(store: AztecLMDBStoreV2, fn: (tx: WriteTransaction) => Promise<T>): Promise<T> {
|
|
214
|
-
const currentWrite = store.getCurrentWriteTx();
|
|
215
|
-
if (currentWrite) {
|
|
216
|
-
return fn(currentWrite);
|
|
217
|
-
} else {
|
|
218
|
-
return store.transactionAsync(fn);
|
|
219
|
-
}
|
|
220
|
-
}
|
|
221
|
-
|
|
222
|
-
export async function execInReadTx<T>(
|
|
223
|
-
store: AztecLMDBStoreV2,
|
|
224
|
-
fn: (tx: ReadTransaction) => T | Promise<T>,
|
|
225
|
-
): Promise<T> {
|
|
226
|
-
const currentWrite = store.getCurrentWriteTx();
|
|
227
|
-
if (currentWrite) {
|
|
228
|
-
return await fn(currentWrite);
|
|
229
|
-
} else {
|
|
230
|
-
const tx = store.getReadTx();
|
|
231
|
-
try {
|
|
232
|
-
return await fn(tx);
|
|
233
|
-
} finally {
|
|
234
|
-
tx.close();
|
|
235
|
-
}
|
|
236
|
-
}
|
|
237
|
-
}
|