@aztec/kv-store 0.0.0-test.0 → 0.0.1-commit.0208eb9
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +2 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -3
- package/dest/indexeddb/array.d.ts +3 -2
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +24 -7
- package/dest/indexeddb/index.d.ts +2 -2
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/index.js +4 -7
- package/dest/indexeddb/map.d.ts +12 -6
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +39 -61
- package/dest/indexeddb/multi_map.d.ts +14 -0
- package/dest/indexeddb/multi_map.d.ts.map +1 -0
- package/dest/indexeddb/multi_map.js +107 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +3 -2
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +6 -2
- package/dest/indexeddb/store.d.ts +15 -16
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +45 -42
- package/dest/interfaces/array.d.ts +5 -4
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array.js +1 -3
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +11 -9
- package/dest/interfaces/common.d.ts.map +1 -1
- package/dest/interfaces/common.js +8 -3
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +4 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/index.js +2 -0
- package/dest/interfaces/map.d.ts +21 -49
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map.js +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +181 -70
- package/dest/interfaces/multi_map.d.ts +41 -0
- package/dest/interfaces/multi_map.d.ts.map +1 -0
- package/dest/interfaces/multi_map.js +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/multi_map_test_suite.js +364 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +18 -43
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/utils.d.ts +3 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +3 -1
- package/dest/lmdb/array.d.ts +3 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +4 -2
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +2 -2
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +5 -5
- package/dest/lmdb/map.d.ts +11 -23
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +15 -81
- package/dest/lmdb/multi_map.d.ts +13 -0
- package/dest/lmdb/multi_map.d.ts.map +1 -0
- package/dest/lmdb/multi_map.js +44 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +9 -25
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +12 -32
- package/dest/lmdb-v2/array.d.ts +4 -3
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +4 -2
- package/dest/lmdb-v2/factory.d.ts +6 -9
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +26 -12
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +12 -45
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +17 -104
- package/dest/lmdb-v2/message.d.ts +24 -5
- package/dest/lmdb-v2/message.d.ts.map +1 -1
- package/dest/lmdb-v2/message.js +6 -4
- package/dest/lmdb-v2/multi_map.d.ts +52 -0
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
- package/dest/lmdb-v2/multi_map.js +123 -0
- package/dest/lmdb-v2/read_transaction.d.ts +3 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +34 -0
- package/dest/lmdb-v2/set.d.ts +15 -0
- package/dest/lmdb-v2/set.d.ts.map +1 -0
- package/dest/lmdb-v2/set.js +23 -0
- package/dest/lmdb-v2/singleton.d.ts +2 -2
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -1
- package/dest/lmdb-v2/store.d.ts +13 -13
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +19 -29
- package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
- package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
- package/dest/lmdb-v2/tx-helpers.js +21 -0
- package/dest/lmdb-v2/utils.d.ts +3 -14
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.js +0 -94
- package/dest/lmdb-v2/write_transaction.d.ts +3 -5
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.js +9 -5
- package/dest/stores/index.d.ts +2 -1
- package/dest/stores/index.d.ts.map +1 -1
- package/dest/stores/index.js +1 -0
- package/dest/stores/l2_tips_store.d.ts +24 -8
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +63 -45
- package/dest/utils.d.ts +9 -6
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +51 -16
- package/package.json +25 -16
- package/src/config.ts +6 -4
- package/src/indexeddb/array.ts +9 -5
- package/src/indexeddb/index.ts +10 -8
- package/src/indexeddb/map.ts +38 -55
- package/src/indexeddb/multi_map.ts +105 -0
- package/src/indexeddb/singleton.ts +5 -2
- package/src/indexeddb/store.ts +71 -56
- package/src/interfaces/array.ts +5 -3
- package/src/interfaces/common.ts +20 -9
- package/src/interfaces/index.ts +3 -1
- package/src/interfaces/map.ts +19 -53
- package/src/interfaces/map_test_suite.ts +101 -44
- package/src/interfaces/multi_map.ts +45 -0
- package/src/interfaces/multi_map_test_suite.ts +374 -0
- package/src/interfaces/store.ts +18 -53
- package/src/interfaces/utils.ts +2 -0
- package/src/lmdb/array.ts +2 -1
- package/src/lmdb/index.ts +11 -6
- package/src/lmdb/map.ts +23 -94
- package/src/lmdb/multi_map.ts +51 -0
- package/src/lmdb/store.ts +24 -48
- package/src/lmdb-v2/array.ts +8 -3
- package/src/lmdb-v2/factory.ts +29 -18
- package/src/lmdb-v2/map.ts +30 -127
- package/src/lmdb-v2/message.ts +23 -0
- package/src/lmdb-v2/multi_map.ts +150 -0
- package/src/lmdb-v2/read_transaction.ts +40 -0
- package/src/lmdb-v2/set.ts +33 -0
- package/src/lmdb-v2/singleton.ts +6 -2
- package/src/lmdb-v2/store.ts +24 -42
- package/src/lmdb-v2/tx-helpers.ts +29 -0
- package/src/lmdb-v2/utils.ts +0 -118
- package/src/lmdb-v2/write_transaction.ts +11 -10
- package/src/stores/index.ts +2 -0
- package/src/stores/l2_tips_store.ts +67 -49
- package/src/utils.ts +79 -21
- package/dest/interfaces/store_test_suite.d.ts +0 -3
- package/dest/interfaces/store_test_suite.d.ts.map +0 -1
- package/dest/interfaces/store_test_suite.js +0 -37
- package/src/interfaces/store_test_suite.ts +0 -56
package/src/lmdb-v2/factory.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
2
|
-
import { type
|
|
3
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
2
|
+
import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
4
4
|
|
|
5
5
|
import { mkdir, mkdtemp, rm } from 'fs/promises';
|
|
6
6
|
import { tmpdir } from 'os';
|
|
@@ -15,8 +15,9 @@ export async function createStore(
|
|
|
15
15
|
name: string,
|
|
16
16
|
schemaVersion: number,
|
|
17
17
|
config: DataStoreConfig,
|
|
18
|
-
|
|
18
|
+
bindings?: LoggerBindings,
|
|
19
19
|
): Promise<AztecLMDBStoreV2> {
|
|
20
|
+
const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
|
|
20
21
|
const { dataDirectory, l1Contracts } = config;
|
|
21
22
|
|
|
22
23
|
let store: AztecLMDBStoreV2;
|
|
@@ -28,16 +29,20 @@ export async function createStore(
|
|
|
28
29
|
const rollupAddress = l1Contracts ? l1Contracts.rollupAddress : EthAddress.ZERO;
|
|
29
30
|
|
|
30
31
|
// Create a version manager
|
|
31
|
-
const versionManager = new DatabaseVersionManager(
|
|
32
|
-
|
|
33
|
-
|
|
32
|
+
const versionManager = new DatabaseVersionManager({
|
|
33
|
+
schemaVersion,
|
|
34
|
+
rollupAddress,
|
|
35
|
+
dataDirectory: subDir,
|
|
36
|
+
onOpen: dbDirectory =>
|
|
37
|
+
AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
|
|
38
|
+
});
|
|
34
39
|
|
|
35
40
|
log.info(
|
|
36
|
-
`Creating ${name} data store at directory ${subDir} with map size ${config.
|
|
41
|
+
`Creating ${name} data store at directory ${subDir} with map size ${config.dataStoreMapSizeKb} KB (LMDB v2)`,
|
|
37
42
|
);
|
|
38
43
|
[store] = await versionManager.open();
|
|
39
44
|
} else {
|
|
40
|
-
store = await openTmpStore(name, true, config.
|
|
45
|
+
store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
|
|
41
46
|
}
|
|
42
47
|
|
|
43
48
|
return store;
|
|
@@ -48,8 +53,9 @@ export async function openTmpStore(
|
|
|
48
53
|
ephemeral: boolean = true,
|
|
49
54
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
50
55
|
maxReaders = MAX_READERS,
|
|
51
|
-
|
|
56
|
+
bindings?: LoggerBindings,
|
|
52
57
|
): Promise<AztecLMDBStoreV2> {
|
|
58
|
+
const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
|
|
53
59
|
const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
|
|
54
60
|
log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
55
61
|
|
|
@@ -69,30 +75,35 @@ export async function openTmpStore(
|
|
|
69
75
|
|
|
70
76
|
// For temporary stores, we don't need to worry about versioning
|
|
71
77
|
// as they are ephemeral and get cleaned up after use
|
|
72
|
-
return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup,
|
|
78
|
+
return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings);
|
|
73
79
|
}
|
|
74
80
|
|
|
75
81
|
export async function openStoreAt(
|
|
76
82
|
dataDir: string,
|
|
77
83
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
78
84
|
maxReaders = MAX_READERS,
|
|
79
|
-
|
|
85
|
+
bindings?: LoggerBindings,
|
|
80
86
|
): Promise<AztecLMDBStoreV2> {
|
|
87
|
+
const log = createLogger('kv-store:lmdb-v2', bindings);
|
|
81
88
|
log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
82
|
-
return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined,
|
|
89
|
+
return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
|
|
83
90
|
}
|
|
84
91
|
|
|
85
92
|
export async function openVersionedStoreAt(
|
|
86
|
-
|
|
93
|
+
dataDirectory: string,
|
|
87
94
|
schemaVersion: number,
|
|
88
95
|
rollupAddress: EthAddress,
|
|
89
96
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
90
97
|
maxReaders = MAX_READERS,
|
|
91
|
-
|
|
98
|
+
bindings?: LoggerBindings,
|
|
92
99
|
): Promise<AztecLMDBStoreV2> {
|
|
93
|
-
log
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
100
|
+
const log = createLogger('kv-store:lmdb-v2', bindings);
|
|
101
|
+
log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
102
|
+
const [store] = await new DatabaseVersionManager({
|
|
103
|
+
schemaVersion,
|
|
104
|
+
rollupAddress,
|
|
105
|
+
dataDirectory,
|
|
106
|
+
onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
|
|
107
|
+
}).open();
|
|
97
108
|
return store;
|
|
98
109
|
}
|
package/src/lmdb-v2/map.ts
CHANGED
|
@@ -1,16 +1,20 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
|
-
import type { Key, Range } from '../interfaces/common.js';
|
|
4
|
-
import type { AztecAsyncMap
|
|
3
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
4
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
5
5
|
import type { ReadTransaction } from './read_transaction.js';
|
|
6
|
-
import {
|
|
6
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
7
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
7
8
|
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
8
9
|
|
|
9
|
-
export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
10
|
+
export class LMDBMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
|
|
10
11
|
private prefix: string;
|
|
11
12
|
private encoder = new Encoder();
|
|
12
13
|
|
|
13
|
-
constructor(
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
14
18
|
this.prefix = `map:${name}`;
|
|
15
19
|
}
|
|
16
20
|
/**
|
|
@@ -22,6 +26,18 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
22
26
|
return execInWriteTx(this.store, tx => tx.set(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
23
27
|
}
|
|
24
28
|
|
|
29
|
+
/**
|
|
30
|
+
* Sets the values at the given keys.
|
|
31
|
+
* @param entries - The entries to set
|
|
32
|
+
*/
|
|
33
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
34
|
+
await execInWriteTx(this.store, async tx => {
|
|
35
|
+
for (const { key, value } of entries) {
|
|
36
|
+
await tx.set(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
37
|
+
}
|
|
38
|
+
});
|
|
39
|
+
}
|
|
40
|
+
|
|
25
41
|
/**
|
|
26
42
|
* Sets the value at the given key if it does not already exist.
|
|
27
43
|
* @param key - The key to set the value at
|
|
@@ -58,15 +74,21 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
58
74
|
return execInReadTx(this.store, async tx => !!(await tx.get(serializeKey(this.prefix, key))));
|
|
59
75
|
}
|
|
60
76
|
|
|
77
|
+
sizeAsync(): Promise<number> {
|
|
78
|
+
return execInReadTx(this.store, tx => tx.countEntries(minKey(this.prefix), maxKey(this.prefix), false));
|
|
79
|
+
}
|
|
80
|
+
|
|
61
81
|
/**
|
|
62
82
|
* Iterates over the map's key-value entries in the key's natural order
|
|
63
83
|
* @param range - The range of keys to iterate over
|
|
64
84
|
*/
|
|
65
85
|
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
66
86
|
const reverse = range?.reverse ?? false;
|
|
67
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
68
87
|
|
|
69
|
-
const
|
|
88
|
+
const startKey = range?.start !== undefined ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
89
|
+
|
|
90
|
+
const endKey =
|
|
91
|
+
range?.end !== undefined ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
70
92
|
|
|
71
93
|
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
72
94
|
const shouldClose = !tx;
|
|
@@ -80,7 +102,7 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
80
102
|
range?.limit,
|
|
81
103
|
)) {
|
|
82
104
|
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
83
|
-
if (
|
|
105
|
+
if (deserializedKey === false) {
|
|
84
106
|
break;
|
|
85
107
|
}
|
|
86
108
|
yield [deserializedKey, this.encoder.unpack(val)];
|
|
@@ -112,122 +134,3 @@ export class LMDBMap<K extends Key, V> implements AztecAsyncMap<K, V> {
|
|
|
112
134
|
}
|
|
113
135
|
}
|
|
114
136
|
}
|
|
115
|
-
|
|
116
|
-
export class LMDBMultiMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
|
|
117
|
-
private prefix: string;
|
|
118
|
-
private encoder = new Encoder();
|
|
119
|
-
constructor(private store: AztecLMDBStoreV2, name: string) {
|
|
120
|
-
this.prefix = `multimap:${name}`;
|
|
121
|
-
}
|
|
122
|
-
|
|
123
|
-
/**
|
|
124
|
-
* Sets the value at the given key.
|
|
125
|
-
* @param key - The key to set the value at
|
|
126
|
-
* @param val - The value to set
|
|
127
|
-
*/
|
|
128
|
-
set(key: K, val: V): Promise<void> {
|
|
129
|
-
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
130
|
-
}
|
|
131
|
-
|
|
132
|
-
/**
|
|
133
|
-
* Sets the value at the given key if it does not already exist.
|
|
134
|
-
* @param key - The key to set the value at
|
|
135
|
-
* @param val - The value to set
|
|
136
|
-
*/
|
|
137
|
-
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
138
|
-
return execInWriteTx(this.store, async tx => {
|
|
139
|
-
const exists = !!(await this.getAsync(key));
|
|
140
|
-
if (!exists) {
|
|
141
|
-
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
142
|
-
return true;
|
|
143
|
-
}
|
|
144
|
-
return false;
|
|
145
|
-
});
|
|
146
|
-
}
|
|
147
|
-
|
|
148
|
-
/**
|
|
149
|
-
* Deletes the value at the given key.
|
|
150
|
-
* @param key - The key to delete the value at
|
|
151
|
-
*/
|
|
152
|
-
delete(key: K): Promise<void> {
|
|
153
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
154
|
-
}
|
|
155
|
-
|
|
156
|
-
getAsync(key: K): Promise<V | undefined> {
|
|
157
|
-
return execInReadTx(this.store, async tx => {
|
|
158
|
-
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
159
|
-
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
160
|
-
});
|
|
161
|
-
}
|
|
162
|
-
|
|
163
|
-
hasAsync(key: K): Promise<boolean> {
|
|
164
|
-
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
165
|
-
}
|
|
166
|
-
|
|
167
|
-
/**
|
|
168
|
-
* Iterates over the map's key-value entries in the key's natural order
|
|
169
|
-
* @param range - The range of keys to iterate over
|
|
170
|
-
*/
|
|
171
|
-
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
172
|
-
const reverse = range?.reverse ?? false;
|
|
173
|
-
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
174
|
-
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
175
|
-
|
|
176
|
-
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
177
|
-
const shouldClose = !tx;
|
|
178
|
-
tx ??= this.store.getReadTx();
|
|
179
|
-
|
|
180
|
-
try {
|
|
181
|
-
for await (const [key, vals] of tx.iterateIndex(
|
|
182
|
-
reverse ? endKey! : startKey,
|
|
183
|
-
reverse ? startKey : endKey,
|
|
184
|
-
reverse,
|
|
185
|
-
range?.limit,
|
|
186
|
-
)) {
|
|
187
|
-
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
188
|
-
if (!deserializedKey) {
|
|
189
|
-
break;
|
|
190
|
-
}
|
|
191
|
-
|
|
192
|
-
for (const val of vals) {
|
|
193
|
-
yield [deserializedKey, this.encoder.unpack(val)];
|
|
194
|
-
}
|
|
195
|
-
}
|
|
196
|
-
} finally {
|
|
197
|
-
if (shouldClose) {
|
|
198
|
-
tx.close();
|
|
199
|
-
}
|
|
200
|
-
}
|
|
201
|
-
}
|
|
202
|
-
|
|
203
|
-
/**
|
|
204
|
-
* Iterates over the map's values in the key's natural order
|
|
205
|
-
* @param range - The range of keys to iterate over
|
|
206
|
-
*/
|
|
207
|
-
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
208
|
-
for await (const [_, value] of this.entriesAsync(range)) {
|
|
209
|
-
yield value;
|
|
210
|
-
}
|
|
211
|
-
}
|
|
212
|
-
|
|
213
|
-
/**
|
|
214
|
-
* Iterates over the map's keys in the key's natural order
|
|
215
|
-
* @param range - The range of keys to iterate over
|
|
216
|
-
*/
|
|
217
|
-
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
218
|
-
for await (const [key, _] of this.entriesAsync(range)) {
|
|
219
|
-
yield key;
|
|
220
|
-
}
|
|
221
|
-
}
|
|
222
|
-
|
|
223
|
-
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
224
|
-
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
225
|
-
}
|
|
226
|
-
|
|
227
|
-
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
228
|
-
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
229
|
-
for (const value of values) {
|
|
230
|
-
yield this.encoder.unpack(value);
|
|
231
|
-
}
|
|
232
|
-
}
|
|
233
|
-
}
|
package/src/lmdb-v2/message.ts
CHANGED
|
@@ -12,6 +12,7 @@ export enum LMDBMessageType {
|
|
|
12
12
|
|
|
13
13
|
START_CURSOR,
|
|
14
14
|
ADVANCE_CURSOR,
|
|
15
|
+
ADVANCE_CURSOR_COUNT,
|
|
15
16
|
CLOSE_CURSOR,
|
|
16
17
|
|
|
17
18
|
BATCH,
|
|
@@ -19,6 +20,7 @@ export enum LMDBMessageType {
|
|
|
19
20
|
STATS,
|
|
20
21
|
|
|
21
22
|
CLOSE,
|
|
23
|
+
COPY_STORE,
|
|
22
24
|
}
|
|
23
25
|
|
|
24
26
|
type Key = Uint8Array;
|
|
@@ -59,10 +61,20 @@ interface AdvanceCursorRequest {
|
|
|
59
61
|
count: number | null;
|
|
60
62
|
}
|
|
61
63
|
|
|
64
|
+
interface AdvanceCursorCountRequest {
|
|
65
|
+
cursor: number;
|
|
66
|
+
endKey: Key;
|
|
67
|
+
}
|
|
68
|
+
|
|
62
69
|
interface CloseCursorRequest {
|
|
63
70
|
cursor: number;
|
|
64
71
|
}
|
|
65
72
|
|
|
73
|
+
interface CopyStoreRequest {
|
|
74
|
+
dstPath: string;
|
|
75
|
+
compact: boolean;
|
|
76
|
+
}
|
|
77
|
+
|
|
66
78
|
export interface Batch {
|
|
67
79
|
addEntries: Array<KeyValues>;
|
|
68
80
|
removeEntries: Array<KeyOptionalValues>;
|
|
@@ -80,6 +92,7 @@ export type LMDBRequestBody = {
|
|
|
80
92
|
|
|
81
93
|
[LMDBMessageType.START_CURSOR]: StartCursorRequest;
|
|
82
94
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorRequest;
|
|
95
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountRequest;
|
|
83
96
|
[LMDBMessageType.CLOSE_CURSOR]: CloseCursorRequest;
|
|
84
97
|
|
|
85
98
|
[LMDBMessageType.BATCH]: BatchRequest;
|
|
@@ -87,6 +100,7 @@ export type LMDBRequestBody = {
|
|
|
87
100
|
[LMDBMessageType.STATS]: void;
|
|
88
101
|
|
|
89
102
|
[LMDBMessageType.CLOSE]: void;
|
|
103
|
+
[LMDBMessageType.COPY_STORE]: CopyStoreRequest;
|
|
90
104
|
};
|
|
91
105
|
|
|
92
106
|
interface GetResponse {
|
|
@@ -107,6 +121,11 @@ interface AdvanceCursorResponse {
|
|
|
107
121
|
done: boolean;
|
|
108
122
|
}
|
|
109
123
|
|
|
124
|
+
interface AdvanceCursorCountResponse {
|
|
125
|
+
count: number;
|
|
126
|
+
done: boolean;
|
|
127
|
+
}
|
|
128
|
+
|
|
110
129
|
interface BatchResponse {
|
|
111
130
|
durationNs: number;
|
|
112
131
|
}
|
|
@@ -122,6 +141,7 @@ interface StatsResponse {
|
|
|
122
141
|
totalUsedSize: bigint | number;
|
|
123
142
|
}>;
|
|
124
143
|
dbMapSizeBytes: bigint | number;
|
|
144
|
+
dbPhysicalFileSizeBytes: bigint | number;
|
|
125
145
|
}
|
|
126
146
|
|
|
127
147
|
export type LMDBResponseBody = {
|
|
@@ -132,6 +152,7 @@ export type LMDBResponseBody = {
|
|
|
132
152
|
|
|
133
153
|
[LMDBMessageType.START_CURSOR]: StartCursorResponse;
|
|
134
154
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorResponse;
|
|
155
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountResponse;
|
|
135
156
|
[LMDBMessageType.CLOSE_CURSOR]: BoolResponse;
|
|
136
157
|
|
|
137
158
|
[LMDBMessageType.BATCH]: BatchResponse;
|
|
@@ -139,6 +160,8 @@ export type LMDBResponseBody = {
|
|
|
139
160
|
[LMDBMessageType.STATS]: StatsResponse;
|
|
140
161
|
|
|
141
162
|
[LMDBMessageType.CLOSE]: BoolResponse;
|
|
163
|
+
|
|
164
|
+
[LMDBMessageType.COPY_STORE]: BoolResponse;
|
|
142
165
|
};
|
|
143
166
|
|
|
144
167
|
export interface LMDBMessageChannel {
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { Encoder } from 'msgpackr/pack';
|
|
2
|
+
import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
|
|
3
|
+
|
|
4
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
6
|
+
import type { ReadTransaction } from './read_transaction.js';
|
|
7
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
8
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
9
|
+
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
10
|
+
|
|
11
|
+
export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncMultiMap<K, V> {
|
|
12
|
+
private prefix: string;
|
|
13
|
+
private encoder = new Encoder();
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
18
|
+
this.prefix = `multimap:${name}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Sets the value at the given key.
|
|
23
|
+
* @param key - The key to set the value at
|
|
24
|
+
* @param val - The value to set
|
|
25
|
+
*/
|
|
26
|
+
set(key: K, val: V): Promise<void> {
|
|
27
|
+
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
31
|
+
await execInWriteTx(this.store, async tx => {
|
|
32
|
+
for (const { key, value } of entries) {
|
|
33
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Sets the value at the given key if it does not already exist.
|
|
40
|
+
* @param key - The key to set the value at
|
|
41
|
+
* @param val - The value to set
|
|
42
|
+
*/
|
|
43
|
+
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
44
|
+
return execInWriteTx(this.store, async tx => {
|
|
45
|
+
const exists = !!(await this.getAsync(key));
|
|
46
|
+
if (!exists) {
|
|
47
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return false;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Deletes the value at the given key.
|
|
56
|
+
* @param key - The key to delete the value at
|
|
57
|
+
*/
|
|
58
|
+
delete(key: K): Promise<void> {
|
|
59
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
getAsync(key: K): Promise<V | undefined> {
|
|
63
|
+
return execInReadTx(this.store, async tx => {
|
|
64
|
+
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
65
|
+
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
hasAsync(key: K): Promise<boolean> {
|
|
70
|
+
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
sizeAsync(): Promise<number> {
|
|
74
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(minKey(this.prefix), maxKey(this.prefix), false));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Iterates over the map's key-value entries in the key's natural order
|
|
79
|
+
* @param range - The range of keys to iterate over
|
|
80
|
+
*/
|
|
81
|
+
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
82
|
+
const reverse = range?.reverse ?? false;
|
|
83
|
+
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
84
|
+
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
85
|
+
|
|
86
|
+
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
87
|
+
const shouldClose = !tx;
|
|
88
|
+
tx ??= this.store.getReadTx();
|
|
89
|
+
|
|
90
|
+
try {
|
|
91
|
+
for await (const [key, vals] of tx.iterateIndex(
|
|
92
|
+
reverse ? endKey! : startKey,
|
|
93
|
+
reverse ? startKey : endKey,
|
|
94
|
+
reverse,
|
|
95
|
+
range?.limit,
|
|
96
|
+
)) {
|
|
97
|
+
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
98
|
+
if (!deserializedKey) {
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const val of vals) {
|
|
103
|
+
yield [deserializedKey, this.encoder.unpack(val)];
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
} finally {
|
|
107
|
+
if (shouldClose) {
|
|
108
|
+
tx.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Iterates over the map's values in the key's natural order
|
|
115
|
+
* @param range - The range of keys to iterate over
|
|
116
|
+
*/
|
|
117
|
+
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
118
|
+
for await (const [_, value] of this.entriesAsync(range)) {
|
|
119
|
+
yield value;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Iterates over the map's keys in the key's natural order
|
|
125
|
+
* @param range - The range of keys to iterate over
|
|
126
|
+
*/
|
|
127
|
+
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
128
|
+
for await (const [key, _] of this.entriesAsync(range)) {
|
|
129
|
+
yield key;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
134
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
138
|
+
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
139
|
+
for (const value of values) {
|
|
140
|
+
yield this.encoder.unpack(value);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
145
|
+
const startKey = serializeKey(this.prefix, key);
|
|
146
|
+
const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
|
|
147
|
+
|
|
148
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -48,6 +48,14 @@ export class ReadTransaction {
|
|
|
48
48
|
yield* this.#iterate(Database.INDEX, startKey, endKey, reverse, limit, vals => vals);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
public countEntries(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
52
|
+
return this.#countEntries(Database.DATA, startKey, endKey, reverse);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
public countEntriesIndex(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
56
|
+
return this.#countEntries(Database.INDEX, startKey, endKey, reverse);
|
|
57
|
+
}
|
|
58
|
+
|
|
51
59
|
async *#iterate<T>(
|
|
52
60
|
db: string,
|
|
53
61
|
startKey: Uint8Array,
|
|
@@ -113,4 +121,36 @@ export class ReadTransaction {
|
|
|
113
121
|
}
|
|
114
122
|
}
|
|
115
123
|
}
|
|
124
|
+
|
|
125
|
+
async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
126
|
+
this.assertIsOpen();
|
|
127
|
+
|
|
128
|
+
const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
|
|
129
|
+
key: startKey,
|
|
130
|
+
reverse,
|
|
131
|
+
count: 0,
|
|
132
|
+
onePage: false,
|
|
133
|
+
db,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const cursor = response.cursor;
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
if (!cursor) {
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const advanceResponse = await this.channel.sendMessage(LMDBMessageType.ADVANCE_CURSOR_COUNT, {
|
|
144
|
+
cursor,
|
|
145
|
+
endKey: endKey,
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
return advanceResponse.count;
|
|
149
|
+
} finally {
|
|
150
|
+
// we might not have anything to close
|
|
151
|
+
if (typeof cursor === 'number') {
|
|
152
|
+
await this.channel.sendMessage(LMDBMessageType.CLOSE_CURSOR, { cursor });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
116
156
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Key, Range } from '../interfaces/common.js';
|
|
2
|
+
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
3
|
+
import { LMDBMap } from './map.js';
|
|
4
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A set backed by LMDB.
|
|
8
|
+
*/
|
|
9
|
+
export class LMDBSet<K extends Key> implements AztecAsyncSet<K> {
|
|
10
|
+
private map: LMDBMap<K, boolean>;
|
|
11
|
+
|
|
12
|
+
constructor(store: AztecLMDBStoreV2, mapName: string) {
|
|
13
|
+
this.map = new LMDBMap(store, mapName);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
hasAsync(key: K): Promise<boolean> {
|
|
17
|
+
return this.map.hasAsync(key);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
add(key: K): Promise<void> {
|
|
21
|
+
return this.map.set(key, true);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
delete(key: K): Promise<void> {
|
|
25
|
+
return this.map.delete(key);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
29
|
+
for await (const key of this.map.keysAsync(range)) {
|
|
30
|
+
yield key;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
package/src/lmdb-v2/singleton.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
3
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
4
|
-
import {
|
|
4
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
5
|
+
import { execInReadTx, execInWriteTx } from './tx-helpers.js';
|
|
5
6
|
import { serializeKey } from './utils.js';
|
|
6
7
|
|
|
7
8
|
export class LMDBSingleValue<T> implements AztecAsyncSingleton<T> {
|
|
8
9
|
private key: Uint8Array;
|
|
9
10
|
private encoder = new Encoder();
|
|
10
|
-
constructor(
|
|
11
|
+
constructor(
|
|
12
|
+
private store: AztecLMDBStoreV2,
|
|
13
|
+
name: string,
|
|
14
|
+
) {
|
|
11
15
|
this.key = serializeKey(`singleton:${name}`, 'value');
|
|
12
16
|
}
|
|
13
17
|
|