@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.5daedc8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +2 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -3
- package/dest/indexeddb/array.d.ts +3 -2
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +3 -0
- package/dest/indexeddb/index.d.ts +1 -1
- package/dest/indexeddb/index.js +1 -1
- package/dest/indexeddb/map.d.ts +12 -6
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +38 -60
- package/dest/indexeddb/multi_map.d.ts +13 -0
- package/dest/indexeddb/multi_map.d.ts.map +1 -0
- package/dest/indexeddb/multi_map.js +92 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +3 -2
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +3 -1
- package/dest/indexeddb/store.d.ts +13 -14
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +45 -42
- package/dest/interfaces/array.d.ts +5 -4
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array.js +1 -3
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +11 -9
- package/dest/interfaces/common.d.ts.map +1 -1
- package/dest/interfaces/common.js +8 -3
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +4 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/index.js +2 -0
- package/dest/interfaces/map.d.ts +21 -49
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map.js +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +135 -70
- package/dest/interfaces/multi_map.d.ts +41 -0
- package/dest/interfaces/multi_map.d.ts.map +1 -0
- package/dest/interfaces/multi_map.js +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/multi_map_test_suite.js +339 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +18 -43
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/utils.d.ts +2 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +2 -1
- package/dest/lmdb/array.d.ts +3 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +1 -1
- package/dest/lmdb/index.js +2 -2
- package/dest/lmdb/map.d.ts +11 -23
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +15 -81
- package/dest/lmdb/multi_map.d.ts +13 -0
- package/dest/lmdb/multi_map.d.ts.map +1 -0
- package/dest/lmdb/multi_map.js +44 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +9 -25
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +11 -31
- package/dest/lmdb-v2/array.d.ts +3 -2
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +1 -0
- package/dest/lmdb-v2/factory.d.ts +4 -7
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +16 -6
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +11 -44
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +17 -103
- package/dest/lmdb-v2/message.d.ts +24 -5
- package/dest/lmdb-v2/message.d.ts.map +1 -1
- package/dest/lmdb-v2/message.js +6 -4
- package/dest/lmdb-v2/multi_map.d.ts +52 -0
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
- package/dest/lmdb-v2/multi_map.js +124 -0
- package/dest/lmdb-v2/read_transaction.d.ts +3 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +34 -0
- package/dest/lmdb-v2/set.d.ts +15 -0
- package/dest/lmdb-v2/set.d.ts.map +1 -0
- package/dest/lmdb-v2/set.js +23 -0
- package/dest/lmdb-v2/singleton.d.ts +1 -1
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -0
- package/dest/lmdb-v2/store.d.ts +10 -9
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +19 -7
- package/dest/lmdb-v2/utils.d.ts +3 -5
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.d.ts +3 -5
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/stores/index.d.ts +2 -1
- package/dest/stores/index.d.ts.map +1 -1
- package/dest/stores/index.js +1 -0
- package/dest/stores/l2_tips_store.d.ts +3 -2
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +18 -9
- package/dest/utils.d.ts +1 -1
- package/package.json +21 -16
- package/src/config.ts +6 -4
- package/src/indexeddb/array.ts +5 -1
- package/src/indexeddb/index.ts +2 -2
- package/src/indexeddb/map.ts +35 -53
- package/src/indexeddb/multi_map.ts +91 -0
- package/src/indexeddb/singleton.ts +4 -1
- package/src/indexeddb/store.ts +66 -56
- package/src/interfaces/array.ts +5 -3
- package/src/interfaces/common.ts +20 -9
- package/src/interfaces/index.ts +3 -1
- package/src/interfaces/map.ts +19 -53
- package/src/interfaces/map_test_suite.ts +73 -44
- package/src/interfaces/multi_map.ts +45 -0
- package/src/interfaces/multi_map_test_suite.ts +342 -0
- package/src/interfaces/store.ts +18 -53
- package/src/interfaces/utils.ts +1 -0
- package/src/lmdb/array.ts +2 -1
- package/src/lmdb/index.ts +3 -3
- package/src/lmdb/map.ts +23 -94
- package/src/lmdb/multi_map.ts +51 -0
- package/src/lmdb/store.ts +23 -47
- package/src/lmdb-v2/array.ts +7 -2
- package/src/lmdb-v2/factory.ts +17 -10
- package/src/lmdb-v2/map.ts +29 -126
- package/src/lmdb-v2/message.ts +23 -0
- package/src/lmdb-v2/multi_map.ts +150 -0
- package/src/lmdb-v2/read_transaction.ts +40 -0
- package/src/lmdb-v2/set.ts +33 -0
- package/src/lmdb-v2/singleton.ts +5 -1
- package/src/lmdb-v2/store.ts +22 -14
- package/src/lmdb-v2/write_transaction.ts +2 -2
- package/src/stores/index.ts +2 -0
- package/src/stores/l2_tips_store.ts +18 -9
- package/dest/interfaces/store_test_suite.d.ts +0 -3
- package/dest/interfaces/store_test_suite.d.ts.map +0 -1
- package/dest/interfaces/store_test_suite.js +0 -37
- package/src/interfaces/store_test_suite.ts +0 -56
package/src/lmdb/map.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import type { Database, RangeOptions } from 'lmdb';
|
|
2
2
|
|
|
3
|
-
import type { Key, Range } from '../interfaces/common.js';
|
|
4
|
-
import type {
|
|
3
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
4
|
+
import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js';
|
|
5
5
|
|
|
6
6
|
/** The slot where a key-value entry would be stored */
|
|
7
7
|
type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
|
|
@@ -9,7 +9,7 @@ type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
|
|
|
9
9
|
/**
|
|
10
10
|
* A map backed by LMDB.
|
|
11
11
|
*/
|
|
12
|
-
export class LmdbAztecMap<K extends Key, V> implements
|
|
12
|
+
export class LmdbAztecMap<K extends Key, V extends Value> implements AztecMap<K, V>, AztecAsyncMap<K, V> {
|
|
13
13
|
protected db: Database<[K, V], MapValueSlot<K>>;
|
|
14
14
|
protected name: string;
|
|
15
15
|
|
|
@@ -39,26 +39,6 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
39
39
|
return Promise.resolve(this.get(key));
|
|
40
40
|
}
|
|
41
41
|
|
|
42
|
-
*getValues(key: K): IterableIterator<V> {
|
|
43
|
-
const transaction = this.db.useReadTransaction();
|
|
44
|
-
try {
|
|
45
|
-
const values = this.db.getValues(this.slot(key), {
|
|
46
|
-
transaction,
|
|
47
|
-
});
|
|
48
|
-
for (const value of values) {
|
|
49
|
-
yield value?.[1];
|
|
50
|
-
}
|
|
51
|
-
} finally {
|
|
52
|
-
transaction.done();
|
|
53
|
-
}
|
|
54
|
-
}
|
|
55
|
-
|
|
56
|
-
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
57
|
-
for (const value of this.getValues(key)) {
|
|
58
|
-
yield value;
|
|
59
|
-
}
|
|
60
|
-
}
|
|
61
|
-
|
|
62
42
|
has(key: K): boolean {
|
|
63
43
|
return this.db.doesExist(this.slot(key));
|
|
64
44
|
}
|
|
@@ -71,6 +51,12 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
71
51
|
await this.db.put(this.slot(key), [key, val]);
|
|
72
52
|
}
|
|
73
53
|
|
|
54
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
55
|
+
for (const { key, value } of entries) {
|
|
56
|
+
await this.set(key, value);
|
|
57
|
+
}
|
|
58
|
+
}
|
|
59
|
+
|
|
74
60
|
swap(key: K, fn: (val: V | undefined) => V): Promise<void> {
|
|
75
61
|
return this.db.childTransaction(() => {
|
|
76
62
|
const slot = this.slot(key);
|
|
@@ -90,10 +76,6 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
90
76
|
await this.db.remove(this.slot(key));
|
|
91
77
|
}
|
|
92
78
|
|
|
93
|
-
async deleteValue(key: K, val: V): Promise<void> {
|
|
94
|
-
await this.db.remove(this.slot(key), [key, val]);
|
|
95
|
-
}
|
|
96
|
-
|
|
97
79
|
*entries(range: Range<K> = {}): IterableIterator<[K, V]> {
|
|
98
80
|
const transaction = this.db.useReadTransaction();
|
|
99
81
|
|
|
@@ -106,16 +88,16 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
106
88
|
? this.slot(range.end)
|
|
107
89
|
: this.endSentinel
|
|
108
90
|
: range.start
|
|
109
|
-
|
|
110
|
-
|
|
91
|
+
? this.slot(range.start)
|
|
92
|
+
: this.startSentinel;
|
|
111
93
|
|
|
112
94
|
const end = reverse
|
|
113
95
|
? range.start
|
|
114
96
|
? this.slot(range.start)
|
|
115
97
|
: this.startSentinel
|
|
116
98
|
: range.end
|
|
117
|
-
|
|
118
|
-
|
|
99
|
+
? this.slot(range.end)
|
|
100
|
+
: this.endSentinel;
|
|
119
101
|
|
|
120
102
|
const lmdbRange: RangeOptions = {
|
|
121
103
|
start,
|
|
@@ -137,7 +119,7 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
137
119
|
}
|
|
138
120
|
}
|
|
139
121
|
|
|
140
|
-
async *entriesAsync(range?: Range<K>
|
|
122
|
+
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
141
123
|
for (const entry of this.entries(range)) {
|
|
142
124
|
yield entry;
|
|
143
125
|
}
|
|
@@ -155,6 +137,15 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
155
137
|
}
|
|
156
138
|
}
|
|
157
139
|
|
|
140
|
+
size(): number {
|
|
141
|
+
const iterator = this.db.getRange({ start: this.startSentinel, end: this.endSentinel });
|
|
142
|
+
return iterator.asArray.length;
|
|
143
|
+
}
|
|
144
|
+
|
|
145
|
+
sizeAsync(): Promise<number> {
|
|
146
|
+
return Promise.resolve(this.size());
|
|
147
|
+
}
|
|
148
|
+
|
|
158
149
|
*keys(range: Range<K> = {}): IterableIterator<K> {
|
|
159
150
|
for (const [key, _] of this.entries(range)) {
|
|
160
151
|
yield key;
|
|
@@ -184,65 +175,3 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, Azte
|
|
|
184
175
|
}
|
|
185
176
|
}
|
|
186
177
|
}
|
|
187
|
-
|
|
188
|
-
export class LmdbAztecMapWithSize<K extends Key, V>
|
|
189
|
-
extends LmdbAztecMap<K, V>
|
|
190
|
-
implements AztecMapWithSize<K, V>, AztecAsyncMultiMap<K, V>
|
|
191
|
-
{
|
|
192
|
-
#sizeCache?: number;
|
|
193
|
-
|
|
194
|
-
constructor(rootDb: Database, mapName: string) {
|
|
195
|
-
super(rootDb, mapName);
|
|
196
|
-
}
|
|
197
|
-
|
|
198
|
-
override async set(key: K, val: V): Promise<void> {
|
|
199
|
-
await this.db.childTransaction(() => {
|
|
200
|
-
const exists = this.db.doesExist(this.slot(key));
|
|
201
|
-
this.db.putSync(this.slot(key), [key, val], {
|
|
202
|
-
appendDup: true,
|
|
203
|
-
});
|
|
204
|
-
if (!exists) {
|
|
205
|
-
this.#sizeCache = undefined; // Invalidate cache
|
|
206
|
-
}
|
|
207
|
-
});
|
|
208
|
-
}
|
|
209
|
-
|
|
210
|
-
override async delete(key: K): Promise<void> {
|
|
211
|
-
await this.db.childTransaction(async () => {
|
|
212
|
-
const exists = this.db.doesExist(this.slot(key));
|
|
213
|
-
if (exists) {
|
|
214
|
-
await this.db.remove(this.slot(key));
|
|
215
|
-
this.#sizeCache = undefined; // Invalidate cache
|
|
216
|
-
}
|
|
217
|
-
});
|
|
218
|
-
}
|
|
219
|
-
|
|
220
|
-
override async deleteValue(key: K, val: V): Promise<void> {
|
|
221
|
-
await this.db.childTransaction(async () => {
|
|
222
|
-
const exists = this.db.doesExist(this.slot(key));
|
|
223
|
-
if (exists) {
|
|
224
|
-
await this.db.remove(this.slot(key), [key, val]);
|
|
225
|
-
this.#sizeCache = undefined; // Invalidate cache
|
|
226
|
-
}
|
|
227
|
-
});
|
|
228
|
-
}
|
|
229
|
-
|
|
230
|
-
/**
|
|
231
|
-
* Gets the size of the map by counting entries.
|
|
232
|
-
* @returns The number of entries in the map
|
|
233
|
-
*/
|
|
234
|
-
size(): number {
|
|
235
|
-
if (this.#sizeCache === undefined) {
|
|
236
|
-
this.#sizeCache = this.db.getCount({
|
|
237
|
-
start: this.startSentinel,
|
|
238
|
-
end: this.endSentinel,
|
|
239
|
-
});
|
|
240
|
-
}
|
|
241
|
-
return this.#sizeCache;
|
|
242
|
-
}
|
|
243
|
-
|
|
244
|
-
// Reset cache on clear/drop operations
|
|
245
|
-
clearCache() {
|
|
246
|
-
this.#sizeCache = undefined;
|
|
247
|
-
}
|
|
248
|
-
}
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
import type { Key, Value } from '../interfaces/common.js';
|
|
2
|
+
import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js';
|
|
3
|
+
import { LmdbAztecMap } from './map.js';
|
|
4
|
+
|
|
5
|
+
/**
|
|
6
|
+
* A map backed by LMDB.
|
|
7
|
+
*/
|
|
8
|
+
export class LmdbAztecMultiMap<K extends Key, V extends Value>
|
|
9
|
+
extends LmdbAztecMap<K, V>
|
|
10
|
+
implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V>
|
|
11
|
+
{
|
|
12
|
+
*getValues(key: K): IterableIterator<V> {
|
|
13
|
+
const transaction = this.db.useReadTransaction();
|
|
14
|
+
try {
|
|
15
|
+
const values = this.db.getValues(this.slot(key), {
|
|
16
|
+
transaction,
|
|
17
|
+
});
|
|
18
|
+
for (const value of values) {
|
|
19
|
+
yield value?.[1];
|
|
20
|
+
}
|
|
21
|
+
} finally {
|
|
22
|
+
transaction.done();
|
|
23
|
+
}
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
27
|
+
for (const value of this.getValues(key)) {
|
|
28
|
+
yield value;
|
|
29
|
+
}
|
|
30
|
+
}
|
|
31
|
+
|
|
32
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
33
|
+
const transaction = this.db.useReadTransaction();
|
|
34
|
+
try {
|
|
35
|
+
const values = this.db.getValues(this.slot(key), {
|
|
36
|
+
transaction,
|
|
37
|
+
});
|
|
38
|
+
let count = 0;
|
|
39
|
+
for (const _ of values) {
|
|
40
|
+
count++;
|
|
41
|
+
}
|
|
42
|
+
return Promise.resolve(count);
|
|
43
|
+
} finally {
|
|
44
|
+
transaction.done();
|
|
45
|
+
}
|
|
46
|
+
}
|
|
47
|
+
|
|
48
|
+
async deleteValue(key: K, val: V): Promise<void> {
|
|
49
|
+
await this.db.remove(this.slot(key), [key, val]);
|
|
50
|
+
}
|
|
51
|
+
}
|
package/src/lmdb/store.ts
CHANGED
|
@@ -7,22 +7,17 @@ import { tmpdir } from 'os';
|
|
|
7
7
|
import { join } from 'path';
|
|
8
8
|
|
|
9
9
|
import type { AztecArray, AztecAsyncArray } from '../interfaces/array.js';
|
|
10
|
-
import type { Key, StoreSize } from '../interfaces/common.js';
|
|
10
|
+
import type { Key, StoreSize, Value } from '../interfaces/common.js';
|
|
11
11
|
import type { AztecAsyncCounter, AztecCounter } from '../interfaces/counter.js';
|
|
12
|
-
import type {
|
|
13
|
-
|
|
14
|
-
AztecAsyncMultiMap,
|
|
15
|
-
AztecMap,
|
|
16
|
-
AztecMapWithSize,
|
|
17
|
-
AztecMultiMap,
|
|
18
|
-
AztecMultiMapWithSize,
|
|
19
|
-
} from '../interfaces/map.js';
|
|
12
|
+
import type { AztecAsyncMap, AztecMap } from '../interfaces/map.js';
|
|
13
|
+
import type { AztecAsyncMultiMap, AztecMultiMap } from '../interfaces/multi_map.js';
|
|
20
14
|
import type { AztecAsyncSet, AztecSet } from '../interfaces/set.js';
|
|
21
15
|
import type { AztecAsyncSingleton, AztecSingleton } from '../interfaces/singleton.js';
|
|
22
16
|
import type { AztecAsyncKVStore, AztecKVStore } from '../interfaces/store.js';
|
|
23
17
|
import { LmdbAztecArray } from './array.js';
|
|
24
18
|
import { LmdbAztecCounter } from './counter.js';
|
|
25
|
-
import { LmdbAztecMap
|
|
19
|
+
import { LmdbAztecMap } from './map.js';
|
|
20
|
+
import { LmdbAztecMultiMap } from './multi_map.js';
|
|
26
21
|
import { LmdbAztecSet } from './set.js';
|
|
27
22
|
import { LmdbAztecSingleton } from './singleton.js';
|
|
28
23
|
|
|
@@ -37,7 +32,11 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
37
32
|
#multiMapData: Database<unknown, Key>;
|
|
38
33
|
#log = createLogger('kv-store:lmdb');
|
|
39
34
|
|
|
40
|
-
constructor(
|
|
35
|
+
constructor(
|
|
36
|
+
rootDb: RootDatabase,
|
|
37
|
+
public readonly isEphemeral: boolean,
|
|
38
|
+
private path: string,
|
|
39
|
+
) {
|
|
41
40
|
this.#rootDb = rootDb;
|
|
42
41
|
|
|
43
42
|
// big bucket to store all the data
|
|
@@ -80,27 +79,12 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
80
79
|
return new AztecLmdbStore(rootDb, ephemeral, dbPath);
|
|
81
80
|
}
|
|
82
81
|
|
|
83
|
-
/**
|
|
84
|
-
* Forks the current DB into a new DB by backing it up to a temporary location and opening a new lmdb db.
|
|
85
|
-
* @returns A new AztecLmdbStore.
|
|
86
|
-
*/
|
|
87
|
-
async fork() {
|
|
88
|
-
const baseDir = this.path;
|
|
89
|
-
this.#log.debug(`Forking store with basedir ${baseDir}`);
|
|
90
|
-
const forkPath = await fs.mkdtemp(join(baseDir, 'aztec-store-fork-'));
|
|
91
|
-
this.#log.verbose(`Forking store to ${forkPath}`);
|
|
92
|
-
await this.#rootDb.backup(forkPath, false);
|
|
93
|
-
const forkDb = open(forkPath, { noSync: this.isEphemeral });
|
|
94
|
-
this.#log.debug(`Forked store at ${forkPath} opened successfully`);
|
|
95
|
-
return new AztecLmdbStore(forkDb, this.isEphemeral, forkPath);
|
|
96
|
-
}
|
|
97
|
-
|
|
98
82
|
/**
|
|
99
83
|
* Creates a new AztecMap in the store.
|
|
100
84
|
* @param name - Name of the map
|
|
101
85
|
* @returns A new AztecMap
|
|
102
86
|
*/
|
|
103
|
-
openMap<K extends Key, V>(name: string): AztecMap<K, V> & AztecAsyncMap<K, V> {
|
|
87
|
+
openMap<K extends Key, V extends Value>(name: string): AztecMap<K, V> & AztecAsyncMap<K, V> {
|
|
104
88
|
return new LmdbAztecMap(this.#data, name);
|
|
105
89
|
}
|
|
106
90
|
|
|
@@ -118,37 +102,20 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
118
102
|
* @param name - Name of the map
|
|
119
103
|
* @returns A new AztecMultiMap
|
|
120
104
|
*/
|
|
121
|
-
openMultiMap<K extends Key, V>(name: string): AztecMultiMap<K, V> & AztecAsyncMultiMap<K, V> {
|
|
122
|
-
return new
|
|
105
|
+
openMultiMap<K extends Key, V extends Value>(name: string): AztecMultiMap<K, V> & AztecAsyncMultiMap<K, V> {
|
|
106
|
+
return new LmdbAztecMultiMap(this.#multiMapData, name);
|
|
123
107
|
}
|
|
124
108
|
|
|
125
109
|
openCounter<K extends Key>(name: string): AztecCounter<K> & AztecAsyncCounter<K> {
|
|
126
110
|
return new LmdbAztecCounter(this.#data, name);
|
|
127
111
|
}
|
|
128
|
-
/**
|
|
129
|
-
* Creates a new AztecMultiMapWithSize in the store. A multi-map with size stores multiple values for a single key automatically.
|
|
130
|
-
* @param name - Name of the map
|
|
131
|
-
* @returns A new AztecMultiMapWithSize
|
|
132
|
-
*/
|
|
133
|
-
openMultiMapWithSize<K extends Key, V>(name: string): AztecMultiMapWithSize<K, V> {
|
|
134
|
-
return new LmdbAztecMapWithSize(this.#multiMapData, name);
|
|
135
|
-
}
|
|
136
|
-
|
|
137
|
-
/**
|
|
138
|
-
* Creates a new AztecMapWithSize in the store.
|
|
139
|
-
* @param name - Name of the map
|
|
140
|
-
* @returns A new AztecMapWithSize
|
|
141
|
-
*/
|
|
142
|
-
openMapWithSize<K extends Key, V>(name: string): AztecMapWithSize<K, V> {
|
|
143
|
-
return new LmdbAztecMapWithSize(this.#data, name);
|
|
144
|
-
}
|
|
145
112
|
|
|
146
113
|
/**
|
|
147
114
|
* Creates a new AztecArray in the store.
|
|
148
115
|
* @param name - Name of the array
|
|
149
116
|
* @returns A new AztecArray
|
|
150
117
|
*/
|
|
151
|
-
openArray<T>(name: string): AztecArray<T> & AztecAsyncArray<T> {
|
|
118
|
+
openArray<T extends Value>(name: string): AztecArray<T> & AztecAsyncArray<T> {
|
|
152
119
|
return new LmdbAztecArray(this.#data, name);
|
|
153
120
|
}
|
|
154
121
|
|
|
@@ -224,10 +191,15 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
224
191
|
if ('mapSize' in stats && typeof stats.mapSize === 'number') {
|
|
225
192
|
mapSize = stats.mapSize;
|
|
226
193
|
}
|
|
194
|
+
let physicalFileSize = 0;
|
|
195
|
+
if ('physicalFileSize' in stats && typeof stats.physicalFileSize === 'number') {
|
|
196
|
+
physicalFileSize = stats.physicalFileSize;
|
|
197
|
+
}
|
|
227
198
|
const dataResult = this.estimateSubDBSize(this.#data);
|
|
228
199
|
const multiResult = this.estimateSubDBSize(this.#multiMapData);
|
|
229
200
|
return Promise.resolve({
|
|
230
201
|
mappingSize: mapSize,
|
|
202
|
+
physicalFileSize: physicalFileSize,
|
|
231
203
|
actualSize: dataResult.actualSize + multiResult.actualSize,
|
|
232
204
|
numItems: dataResult.numItems + multiResult.numItems,
|
|
233
205
|
});
|
|
@@ -260,4 +232,8 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
260
232
|
}
|
|
261
233
|
return { actualSize, numItems };
|
|
262
234
|
}
|
|
235
|
+
|
|
236
|
+
backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
|
|
237
|
+
throw new Error('Method not implemented.');
|
|
238
|
+
}
|
|
263
239
|
}
|
package/src/lmdb-v2/array.ts
CHANGED
|
@@ -1,17 +1,22 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr/pack';
|
|
2
2
|
|
|
3
3
|
import type { AztecAsyncArray } from '../interfaces/array.js';
|
|
4
|
+
import type { Value } from '../interfaces/common.js';
|
|
4
5
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
5
6
|
import type { ReadTransaction } from './read_transaction.js';
|
|
7
|
+
// eslint-disable-next-line import/no-cycle
|
|
6
8
|
import { AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
|
|
7
9
|
import { deserializeKey, serializeKey } from './utils.js';
|
|
8
10
|
|
|
9
|
-
export class LMDBArray<T> implements AztecAsyncArray<T> {
|
|
11
|
+
export class LMDBArray<T extends Value> implements AztecAsyncArray<T> {
|
|
10
12
|
private length: AztecAsyncSingleton<number>;
|
|
11
13
|
private encoder = new Encoder();
|
|
12
14
|
private prefix: string;
|
|
13
15
|
|
|
14
|
-
constructor(
|
|
16
|
+
constructor(
|
|
17
|
+
private store: AztecLMDBStoreV2,
|
|
18
|
+
name: string,
|
|
19
|
+
) {
|
|
15
20
|
this.length = store.openSingleton(name + ':length');
|
|
16
21
|
this.prefix = `array:${name}`;
|
|
17
22
|
}
|
package/src/lmdb-v2/factory.ts
CHANGED
|
@@ -28,16 +28,20 @@ export async function createStore(
|
|
|
28
28
|
const rollupAddress = l1Contracts ? l1Contracts.rollupAddress : EthAddress.ZERO;
|
|
29
29
|
|
|
30
30
|
// Create a version manager
|
|
31
|
-
const versionManager = new DatabaseVersionManager(
|
|
32
|
-
|
|
33
|
-
|
|
31
|
+
const versionManager = new DatabaseVersionManager({
|
|
32
|
+
schemaVersion,
|
|
33
|
+
rollupAddress,
|
|
34
|
+
dataDirectory: subDir,
|
|
35
|
+
onOpen: dbDirectory =>
|
|
36
|
+
AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), log),
|
|
37
|
+
});
|
|
34
38
|
|
|
35
39
|
log.info(
|
|
36
|
-
`Creating ${name} data store at directory ${subDir} with map size ${config.
|
|
40
|
+
`Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
|
|
37
41
|
);
|
|
38
42
|
[store] = await versionManager.open();
|
|
39
43
|
} else {
|
|
40
|
-
store = await openTmpStore(name, true, config.
|
|
44
|
+
store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, log);
|
|
41
45
|
}
|
|
42
46
|
|
|
43
47
|
return store;
|
|
@@ -83,16 +87,19 @@ export async function openStoreAt(
|
|
|
83
87
|
}
|
|
84
88
|
|
|
85
89
|
export async function openVersionedStoreAt(
|
|
86
|
-
|
|
90
|
+
dataDirectory: string,
|
|
87
91
|
schemaVersion: number,
|
|
88
92
|
rollupAddress: EthAddress,
|
|
89
93
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
90
94
|
maxReaders = MAX_READERS,
|
|
91
95
|
log: Logger = createLogger('kv-store:lmdb-v2'),
|
|
92
96
|
): Promise<AztecLMDBStoreV2> {
|
|
93
|
-
log.debug(`Opening data store at: ${
|
|
94
|
-
const [store] = await new DatabaseVersionManager(
|
|
95
|
-
|
|
96
|
-
|
|
97
|
+
log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
98
|
+
const [store] = await new DatabaseVersionManager({
|
|
99
|
+
schemaVersion,
|
|
100
|
+
rollupAddress,
|
|
101
|
+
dataDirectory,
|
|
102
|
+
onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, log),
|
|
103
|
+
}).open();
|
|
97
104
|
return store;
|
|
98
105
|
}
|
package/src/lmdb-v2/map.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
|
-
import type { Key, Range } from '../interfaces/common.js';
|
|
4
|
-
import type { AztecAsyncMap
|
|
3
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
4
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
5
5
|
import type { ReadTransaction } from './read_transaction.js';
|
|
6
|
+
// eslint-disable-next-line import/no-cycle
|
|
6
7
|
import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
|
|
7
8
|
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
8
9
|
|
|
9
|
-
export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
10
|
+
export class LMDBMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
|
|
10
11
|
private prefix: string;
|
|
11
12
|
private encoder = new Encoder();
|
|
12
13
|
|
|
13
|
-
constructor(
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
14
18
|
this.prefix = `map:${name}`;
|
|
15
19
|
}
|
|
16
20
|
/**
|
|
@@ -22,6 +26,18 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
22
26
|
return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
23
27
|
}
|
|
24
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Sets the values at the given keys.
|
|
31
|
+
* @param entries - The entries to set
|
|
32
|
+
*/
|
|
33
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
34
|
+
await execInWriteTx(this.store, async tx => {
|
|
35
|
+
for (const { key, value } of entries) {
|
|
36
|
+
await tx.set(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
25
41
|
/**
|
|
26
42
|
* Sets the value at the given key if it does not already exist.
|
|
27
43
|
* @param key - The key to set the value at
|
|
@@ -58,15 +74,21 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
58
74
|
return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
|
|
59
75
|
}
|
|
60
76
|
|
|
77
|
+
sizeAsync(): Promise<number> {
|
|
78
|
+
return execInReadTx(this.store, tx => tx.countEntries(minKey(this.prefix), maxKey(this.prefix), false));
|
|
79
|
+
}
|
|
80
|
+
|
|
61
81
|
/**
|
|
62
82
|
* Iterates over the map's key-value entries in the key's natural order
|
|
63
83
|
* @param range - The range of keys to iterate over
|
|
64
84
|
*/
|
|
65
85
|
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
66
86
|
const reverse = range?.reverse ?? false;
|
|
67
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
68
87
|
|
|
69
|
-
const
|
|
88
|
+
const startKey = range?.start !== undefined ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
89
|
+
|
|
90
|
+
const endKey =
|
|
91
|
+
range?.end !== undefined ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
70
92
|
|
|
71
93
|
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
72
94
|
const shouldClose = !tx;
|
|
@@ -80,7 +102,7 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
80
102
|
range?.limit,
|
|
81
103
|
)) {
|
|
82
104
|
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
83
|
-
if (
|
|
105
|
+
if (deserializedKey === false) {
|
|
84
106
|
break;
|
|
85
107
|
}
|
|
86
108
|
yield [deserializedKey, this.encoder.unpack(val)];
|
|
@@ -112,122 +134,3 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
112
134
|
}
|
|
113
135
|
}
|
|
114
136
|
}
|
|
115
|
-
|
|
116
|
-
export class LMDBMultiMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
|
|
117
|
-
private prefix: string;
|
|
118
|
-
private encoder = new Encoder();
|
|
119
|
-
constructor(private store: AztecLMDBStoreV2, name: string) {
|
|
120
|
-
this.prefix = `multimap:${name}`;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Sets the value at the given key.
|
|
125
|
-
* @param key - The key to set the value at
|
|
126
|
-
* @param val - The value to set
|
|
127
|
-
*/
|
|
128
|
-
set(key: K, val: V): Promise<void> {
|
|
129
|
-
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Sets the value at the given key if it does not already exist.
|
|
134
|
-
* @param key - The key to set the value at
|
|
135
|
-
* @param val - The value to set
|
|
136
|
-
*/
|
|
137
|
-
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
138
|
-
return execInWriteTx(this.store, async tx => {
|
|
139
|
-
const exists = !!(await this.getAsync(key));
|
|
140
|
-
if (!exists) {
|
|
141
|
-
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
142
|
-
return true;
|
|
143
|
-
}
|
|
144
|
-
return false;
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Deletes the value at the given key.
|
|
150
|
-
* @param key - The key to delete the value at
|
|
151
|
-
*/
|
|
152
|
-
delete(key: K): Promise<void> {
|
|
153
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
getAsync(key: K): Promise<V | undefined> {
|
|
157
|
-
return execInReadTx(this.store, async tx => {
|
|
158
|
-
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
159
|
-
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
hasAsync(key: K): Promise<boolean> {
|
|
164
|
-
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Iterates over the map's key-value entries in the key's natural order
|
|
169
|
-
* @param range - The range of keys to iterate over
|
|
170
|
-
*/
|
|
171
|
-
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
172
|
-
const reverse = range?.reverse ?? false;
|
|
173
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
174
|
-
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
175
|
-
|
|
176
|
-
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
177
|
-
const shouldClose = !tx;
|
|
178
|
-
tx ??= this.store.getReadTx();
|
|
179
|
-
|
|
180
|
-
try {
|
|
181
|
-
for await (const [key, vals] of tx.iterateIndex(
|
|
182
|
-
reverse ? endKey! : startKey,
|
|
183
|
-
reverse ? startKey : endKey,
|
|
184
|
-
reverse,
|
|
185
|
-
range?.limit,
|
|
186
|
-
)) {
|
|
187
|
-
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
188
|
-
if (!deserializedKey) {
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
for (const val of vals) {
|
|
193
|
-
yield [deserializedKey, this.encoder.unpack(val)];
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
} finally {
|
|
197
|
-
if (shouldClose) {
|
|
198
|
-
tx.close();
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/**
|
|
204
|
-
* Iterates over the map's values in the key's natural order
|
|
205
|
-
* @param range - The range of keys to iterate over
|
|
206
|
-
*/
|
|
207
|
-
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
208
|
-
for await (const [_, value] of this.entriesAsync(range)) {
|
|
209
|
-
yield value;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Iterates over the map's keys in the key's natural order
|
|
215
|
-
* @param range - The range of keys to iterate over
|
|
216
|
-
*/
|
|
217
|
-
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
218
|
-
for await (const [key, _] of this.entriesAsync(range)) {
|
|
219
|
-
yield key;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
224
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
228
|
-
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
229
|
-
for (const value of values) {
|
|
230
|
-
yield this.encoder.unpack(value);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
}
|