@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.5daedc8
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/config.d.ts +2 -2
- package/dest/config.d.ts.map +1 -1
- package/dest/config.js +5 -3
- package/dest/indexeddb/array.d.ts +3 -2
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +3 -0
- package/dest/indexeddb/index.d.ts +1 -1
- package/dest/indexeddb/index.js +1 -1
- package/dest/indexeddb/map.d.ts +12 -6
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +38 -60
- package/dest/indexeddb/multi_map.d.ts +13 -0
- package/dest/indexeddb/multi_map.d.ts.map +1 -0
- package/dest/indexeddb/multi_map.js +92 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +3 -2
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +3 -1
- package/dest/indexeddb/store.d.ts +13 -14
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +45 -42
- package/dest/interfaces/array.d.ts +5 -4
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array.js +1 -3
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +11 -9
- package/dest/interfaces/common.d.ts.map +1 -1
- package/dest/interfaces/common.js +8 -3
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +4 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/index.js +2 -0
- package/dest/interfaces/map.d.ts +21 -49
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map.js +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +135 -70
- package/dest/interfaces/multi_map.d.ts +41 -0
- package/dest/interfaces/multi_map.d.ts.map +1 -0
- package/dest/interfaces/multi_map.js +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/multi_map_test_suite.js +339 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +18 -43
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/utils.d.ts +2 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +2 -1
- package/dest/lmdb/array.d.ts +3 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +1 -1
- package/dest/lmdb/index.js +2 -2
- package/dest/lmdb/map.d.ts +11 -23
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +15 -81
- package/dest/lmdb/multi_map.d.ts +13 -0
- package/dest/lmdb/multi_map.d.ts.map +1 -0
- package/dest/lmdb/multi_map.js +44 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +9 -25
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +11 -31
- package/dest/lmdb-v2/array.d.ts +3 -2
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +1 -0
- package/dest/lmdb-v2/factory.d.ts +4 -7
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +16 -6
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +11 -44
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +17 -103
- package/dest/lmdb-v2/message.d.ts +24 -5
- package/dest/lmdb-v2/message.d.ts.map +1 -1
- package/dest/lmdb-v2/message.js +6 -4
- package/dest/lmdb-v2/multi_map.d.ts +52 -0
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
- package/dest/lmdb-v2/multi_map.js +124 -0
- package/dest/lmdb-v2/read_transaction.d.ts +3 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +34 -0
- package/dest/lmdb-v2/set.d.ts +15 -0
- package/dest/lmdb-v2/set.d.ts.map +1 -0
- package/dest/lmdb-v2/set.js +23 -0
- package/dest/lmdb-v2/singleton.d.ts +1 -1
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -0
- package/dest/lmdb-v2/store.d.ts +10 -9
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +19 -7
- package/dest/lmdb-v2/utils.d.ts +3 -5
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.d.ts +3 -5
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/stores/index.d.ts +2 -1
- package/dest/stores/index.d.ts.map +1 -1
- package/dest/stores/index.js +1 -0
- package/dest/stores/l2_tips_store.d.ts +3 -2
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +18 -9
- package/dest/utils.d.ts +1 -1
- package/package.json +21 -16
- package/src/config.ts +6 -4
- package/src/indexeddb/array.ts +5 -1
- package/src/indexeddb/index.ts +2 -2
- package/src/indexeddb/map.ts +35 -53
- package/src/indexeddb/multi_map.ts +91 -0
- package/src/indexeddb/singleton.ts +4 -1
- package/src/indexeddb/store.ts +66 -56
- package/src/interfaces/array.ts +5 -3
- package/src/interfaces/common.ts +20 -9
- package/src/interfaces/index.ts +3 -1
- package/src/interfaces/map.ts +19 -53
- package/src/interfaces/map_test_suite.ts +73 -44
- package/src/interfaces/multi_map.ts +45 -0
- package/src/interfaces/multi_map_test_suite.ts +342 -0
- package/src/interfaces/store.ts +18 -53
- package/src/interfaces/utils.ts +1 -0
- package/src/lmdb/array.ts +2 -1
- package/src/lmdb/index.ts +3 -3
- package/src/lmdb/map.ts +23 -94
- package/src/lmdb/multi_map.ts +51 -0
- package/src/lmdb/store.ts +23 -47
- package/src/lmdb-v2/array.ts +7 -2
- package/src/lmdb-v2/factory.ts +17 -10
- package/src/lmdb-v2/map.ts +29 -126
- package/src/lmdb-v2/message.ts +23 -0
- package/src/lmdb-v2/multi_map.ts +150 -0
- package/src/lmdb-v2/read_transaction.ts +40 -0
- package/src/lmdb-v2/set.ts +33 -0
- package/src/lmdb-v2/singleton.ts +5 -1
- package/src/lmdb-v2/store.ts +22 -14
- package/src/lmdb-v2/write_transaction.ts +2 -2
- package/src/stores/index.ts +2 -0
- package/src/stores/l2_tips_store.ts +18 -9
- package/dest/interfaces/store_test_suite.d.ts +0 -3
- package/dest/interfaces/store_test_suite.d.ts.map +0 -1
- package/dest/interfaces/store_test_suite.js +0 -37
- package/src/interfaces/store_test_suite.ts +0 -56
package/src/lmdb-v2/message.ts
CHANGED
|
@@ -12,6 +12,7 @@ export enum LMDBMessageType {
|
|
|
12
12
|
|
|
13
13
|
START_CURSOR,
|
|
14
14
|
ADVANCE_CURSOR,
|
|
15
|
+
ADVANCE_CURSOR_COUNT,
|
|
15
16
|
CLOSE_CURSOR,
|
|
16
17
|
|
|
17
18
|
BATCH,
|
|
@@ -19,6 +20,7 @@ export enum LMDBMessageType {
|
|
|
19
20
|
STATS,
|
|
20
21
|
|
|
21
22
|
CLOSE,
|
|
23
|
+
COPY_STORE,
|
|
22
24
|
}
|
|
23
25
|
|
|
24
26
|
type Key = Uint8Array;
|
|
@@ -59,10 +61,20 @@ interface AdvanceCursorRequest {
|
|
|
59
61
|
count: number | null;
|
|
60
62
|
}
|
|
61
63
|
|
|
64
|
+
interface AdvanceCursorCountRequest {
|
|
65
|
+
cursor: number;
|
|
66
|
+
endKey: Key;
|
|
67
|
+
}
|
|
68
|
+
|
|
62
69
|
interface CloseCursorRequest {
|
|
63
70
|
cursor: number;
|
|
64
71
|
}
|
|
65
72
|
|
|
73
|
+
interface CopyStoreRequest {
|
|
74
|
+
dstPath: string;
|
|
75
|
+
compact: boolean;
|
|
76
|
+
}
|
|
77
|
+
|
|
66
78
|
export interface Batch {
|
|
67
79
|
addEntries: Array<KeyValues>;
|
|
68
80
|
removeEntries: Array<KeyOptionalValues>;
|
|
@@ -80,6 +92,7 @@ export type LMDBRequestBody = {
|
|
|
80
92
|
|
|
81
93
|
[LMDBMessageType.START_CURSOR]: StartCursorRequest;
|
|
82
94
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorRequest;
|
|
95
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountRequest;
|
|
83
96
|
[LMDBMessageType.CLOSE_CURSOR]: CloseCursorRequest;
|
|
84
97
|
|
|
85
98
|
[LMDBMessageType.BATCH]: BatchRequest;
|
|
@@ -87,6 +100,7 @@ export type LMDBRequestBody = {
|
|
|
87
100
|
[LMDBMessageType.STATS]: void;
|
|
88
101
|
|
|
89
102
|
[LMDBMessageType.CLOSE]: void;
|
|
103
|
+
[LMDBMessageType.COPY_STORE]: CopyStoreRequest;
|
|
90
104
|
};
|
|
91
105
|
|
|
92
106
|
interface GetResponse {
|
|
@@ -107,6 +121,11 @@ interface AdvanceCursorResponse {
|
|
|
107
121
|
done: boolean;
|
|
108
122
|
}
|
|
109
123
|
|
|
124
|
+
interface AdvanceCursorCountResponse {
|
|
125
|
+
count: number;
|
|
126
|
+
done: boolean;
|
|
127
|
+
}
|
|
128
|
+
|
|
110
129
|
interface BatchResponse {
|
|
111
130
|
durationNs: number;
|
|
112
131
|
}
|
|
@@ -122,6 +141,7 @@ interface StatsResponse {
|
|
|
122
141
|
totalUsedSize: bigint | number;
|
|
123
142
|
}>;
|
|
124
143
|
dbMapSizeBytes: bigint | number;
|
|
144
|
+
dbPhysicalFileSizeBytes: bigint | number;
|
|
125
145
|
}
|
|
126
146
|
|
|
127
147
|
export type LMDBResponseBody = {
|
|
@@ -132,6 +152,7 @@ export type LMDBResponseBody = {
|
|
|
132
152
|
|
|
133
153
|
[LMDBMessageType.START_CURSOR]: StartCursorResponse;
|
|
134
154
|
[LMDBMessageType.ADVANCE_CURSOR]: AdvanceCursorResponse;
|
|
155
|
+
[LMDBMessageType.ADVANCE_CURSOR_COUNT]: AdvanceCursorCountResponse;
|
|
135
156
|
[LMDBMessageType.CLOSE_CURSOR]: BoolResponse;
|
|
136
157
|
|
|
137
158
|
[LMDBMessageType.BATCH]: BatchResponse;
|
|
@@ -139,6 +160,8 @@ export type LMDBResponseBody = {
|
|
|
139
160
|
[LMDBMessageType.STATS]: StatsResponse;
|
|
140
161
|
|
|
141
162
|
[LMDBMessageType.CLOSE]: BoolResponse;
|
|
163
|
+
|
|
164
|
+
[LMDBMessageType.COPY_STORE]: BoolResponse;
|
|
142
165
|
};
|
|
143
166
|
|
|
144
167
|
export interface LMDBMessageChannel {
|
|
@@ -0,0 +1,150 @@
|
|
|
1
|
+
import { Encoder } from 'msgpackr/pack';
|
|
2
|
+
import { MAXIMUM_KEY, toBufferKey } from 'ordered-binary';
|
|
3
|
+
|
|
4
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
6
|
+
import type { ReadTransaction } from './read_transaction.js';
|
|
7
|
+
// eslint-disable-next-line import/no-cycle
|
|
8
|
+
import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
|
|
9
|
+
import { deserializeKey, maxKey, minKey, serializeKey } from './utils.js';
|
|
10
|
+
|
|
11
|
+
export class LMDBMultiMap<K extends Key, V extends Value> implements AztecAsyncMultiMap<K, V> {
|
|
12
|
+
private prefix: string;
|
|
13
|
+
private encoder = new Encoder();
|
|
14
|
+
constructor(
|
|
15
|
+
private store: AztecLMDBStoreV2,
|
|
16
|
+
name: string,
|
|
17
|
+
) {
|
|
18
|
+
this.prefix = `multimap:${name}`;
|
|
19
|
+
}
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* Sets the value at the given key.
|
|
23
|
+
* @param key - The key to set the value at
|
|
24
|
+
* @param val - The value to set
|
|
25
|
+
*/
|
|
26
|
+
set(key: K, val: V): Promise<void> {
|
|
27
|
+
return execInWriteTx(this.store, tx => tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
31
|
+
await execInWriteTx(this.store, async tx => {
|
|
32
|
+
for (const { key, value } of entries) {
|
|
33
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(value));
|
|
34
|
+
}
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
|
|
38
|
+
/**
|
|
39
|
+
* Sets the value at the given key if it does not already exist.
|
|
40
|
+
* @param key - The key to set the value at
|
|
41
|
+
* @param val - The value to set
|
|
42
|
+
*/
|
|
43
|
+
setIfNotExists(key: K, val: V): Promise<boolean> {
|
|
44
|
+
return execInWriteTx(this.store, async tx => {
|
|
45
|
+
const exists = !!(await this.getAsync(key));
|
|
46
|
+
if (!exists) {
|
|
47
|
+
await tx.setIndex(serializeKey(this.prefix, key), this.encoder.pack(val));
|
|
48
|
+
return true;
|
|
49
|
+
}
|
|
50
|
+
return false;
|
|
51
|
+
});
|
|
52
|
+
}
|
|
53
|
+
|
|
54
|
+
/**
|
|
55
|
+
* Deletes the value at the given key.
|
|
56
|
+
* @param key - The key to delete the value at
|
|
57
|
+
*/
|
|
58
|
+
delete(key: K): Promise<void> {
|
|
59
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key)));
|
|
60
|
+
}
|
|
61
|
+
|
|
62
|
+
getAsync(key: K): Promise<V | undefined> {
|
|
63
|
+
return execInReadTx(this.store, async tx => {
|
|
64
|
+
const val = await tx.getIndex(serializeKey(this.prefix, key));
|
|
65
|
+
return val.length > 0 ? this.encoder.unpack(val[0]) : undefined;
|
|
66
|
+
});
|
|
67
|
+
}
|
|
68
|
+
|
|
69
|
+
hasAsync(key: K): Promise<boolean> {
|
|
70
|
+
return execInReadTx(this.store, async tx => (await tx.getIndex(serializeKey(this.prefix, key))).length > 0);
|
|
71
|
+
}
|
|
72
|
+
|
|
73
|
+
sizeAsync(): Promise<number> {
|
|
74
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(minKey(this.prefix), maxKey(this.prefix), false));
|
|
75
|
+
}
|
|
76
|
+
|
|
77
|
+
/**
|
|
78
|
+
* Iterates over the map's key-value entries in the key's natural order
|
|
79
|
+
* @param range - The range of keys to iterate over
|
|
80
|
+
*/
|
|
81
|
+
async *entriesAsync(range?: Range<K>): AsyncIterableIterator<[K, V]> {
|
|
82
|
+
const reverse = range?.reverse ?? false;
|
|
83
|
+
const startKey = range?.start ? serializeKey(this.prefix, range.start) : minKey(this.prefix);
|
|
84
|
+
const endKey = range?.end ? serializeKey(this.prefix, range.end) : reverse ? maxKey(this.prefix) : undefined;
|
|
85
|
+
|
|
86
|
+
let tx: ReadTransaction | undefined = this.store.getCurrentWriteTx();
|
|
87
|
+
const shouldClose = !tx;
|
|
88
|
+
tx ??= this.store.getReadTx();
|
|
89
|
+
|
|
90
|
+
try {
|
|
91
|
+
for await (const [key, vals] of tx.iterateIndex(
|
|
92
|
+
reverse ? endKey! : startKey,
|
|
93
|
+
reverse ? startKey : endKey,
|
|
94
|
+
reverse,
|
|
95
|
+
range?.limit,
|
|
96
|
+
)) {
|
|
97
|
+
const deserializedKey = deserializeKey<K>(this.prefix, key);
|
|
98
|
+
if (!deserializedKey) {
|
|
99
|
+
break;
|
|
100
|
+
}
|
|
101
|
+
|
|
102
|
+
for (const val of vals) {
|
|
103
|
+
yield [deserializedKey, this.encoder.unpack(val)];
|
|
104
|
+
}
|
|
105
|
+
}
|
|
106
|
+
} finally {
|
|
107
|
+
if (shouldClose) {
|
|
108
|
+
tx.close();
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
}
|
|
112
|
+
|
|
113
|
+
/**
|
|
114
|
+
* Iterates over the map's values in the key's natural order
|
|
115
|
+
* @param range - The range of keys to iterate over
|
|
116
|
+
*/
|
|
117
|
+
async *valuesAsync(range?: Range<K>): AsyncIterableIterator<V> {
|
|
118
|
+
for await (const [_, value] of this.entriesAsync(range)) {
|
|
119
|
+
yield value;
|
|
120
|
+
}
|
|
121
|
+
}
|
|
122
|
+
|
|
123
|
+
/**
|
|
124
|
+
* Iterates over the map's keys in the key's natural order
|
|
125
|
+
* @param range - The range of keys to iterate over
|
|
126
|
+
*/
|
|
127
|
+
async *keysAsync(range?: Range<K>): AsyncIterableIterator<K> {
|
|
128
|
+
for await (const [key, _] of this.entriesAsync(range)) {
|
|
129
|
+
yield key;
|
|
130
|
+
}
|
|
131
|
+
}
|
|
132
|
+
|
|
133
|
+
deleteValue(key: K, val: V | undefined): Promise<void> {
|
|
134
|
+
return execInWriteTx(this.store, tx => tx.removeIndex(serializeKey(this.prefix, key), this.encoder.pack(val)));
|
|
135
|
+
}
|
|
136
|
+
|
|
137
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
138
|
+
const values = await execInReadTx(this.store, tx => tx.getIndex(serializeKey(this.prefix, key)));
|
|
139
|
+
for (const value of values) {
|
|
140
|
+
yield this.encoder.unpack(value);
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
145
|
+
const startKey = serializeKey(this.prefix, key);
|
|
146
|
+
const endKey = toBufferKey([this.prefix, key, MAXIMUM_KEY]);
|
|
147
|
+
|
|
148
|
+
return execInReadTx(this.store, tx => tx.countEntriesIndex(startKey, endKey, false));
|
|
149
|
+
}
|
|
150
|
+
}
|
|
@@ -48,6 +48,14 @@ export class ReadTransaction {
|
|
|
48
48
|
yield* this.#iterate(Database.INDEX, startKey, endKey, reverse, limit, vals => vals);
|
|
49
49
|
}
|
|
50
50
|
|
|
51
|
+
public countEntries(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
52
|
+
return this.#countEntries(Database.DATA, startKey, endKey, reverse);
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
public countEntriesIndex(startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
56
|
+
return this.#countEntries(Database.INDEX, startKey, endKey, reverse);
|
|
57
|
+
}
|
|
58
|
+
|
|
51
59
|
async *#iterate<T>(
|
|
52
60
|
db: string,
|
|
53
61
|
startKey: Uint8Array,
|
|
@@ -113,4 +121,36 @@ export class ReadTransaction {
|
|
|
113
121
|
}
|
|
114
122
|
}
|
|
115
123
|
}
|
|
124
|
+
|
|
125
|
+
async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
126
|
+
this.assertIsOpen();
|
|
127
|
+
|
|
128
|
+
const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
|
|
129
|
+
key: startKey,
|
|
130
|
+
reverse,
|
|
131
|
+
count: 0,
|
|
132
|
+
onePage: false,
|
|
133
|
+
db,
|
|
134
|
+
});
|
|
135
|
+
|
|
136
|
+
const cursor = response.cursor;
|
|
137
|
+
|
|
138
|
+
try {
|
|
139
|
+
if (!cursor) {
|
|
140
|
+
return 0;
|
|
141
|
+
}
|
|
142
|
+
|
|
143
|
+
const advanceResponse = await this.channel.sendMessage(LMDBMessageType.ADVANCE_CURSOR_COUNT, {
|
|
144
|
+
cursor,
|
|
145
|
+
endKey: endKey,
|
|
146
|
+
});
|
|
147
|
+
|
|
148
|
+
return advanceResponse.count;
|
|
149
|
+
} finally {
|
|
150
|
+
// we might not have anything to close
|
|
151
|
+
if (typeof cursor === 'number') {
|
|
152
|
+
await this.channel.sendMessage(LMDBMessageType.CLOSE_CURSOR, { cursor });
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
}
|
|
116
156
|
}
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
import type { Key, Range } from '../interfaces/common.js';
|
|
2
|
+
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
3
|
+
import { LMDBMap } from './map.js';
|
|
4
|
+
import type { AztecLMDBStoreV2 } from './store.js';
|
|
5
|
+
|
|
6
|
+
/**
|
|
7
|
+
* A set backed by LMDB.
|
|
8
|
+
*/
|
|
9
|
+
export class LMDBSet<K extends Key> implements AztecAsyncSet<K> {
|
|
10
|
+
private map: LMDBMap<K, boolean>;
|
|
11
|
+
|
|
12
|
+
constructor(store: AztecLMDBStoreV2, mapName: string) {
|
|
13
|
+
this.map = new LMDBMap(store, mapName);
|
|
14
|
+
}
|
|
15
|
+
|
|
16
|
+
hasAsync(key: K): Promise<boolean> {
|
|
17
|
+
return this.map.hasAsync(key);
|
|
18
|
+
}
|
|
19
|
+
|
|
20
|
+
add(key: K): Promise<void> {
|
|
21
|
+
return this.map.set(key, true);
|
|
22
|
+
}
|
|
23
|
+
|
|
24
|
+
delete(key: K): Promise<void> {
|
|
25
|
+
return this.map.delete(key);
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
29
|
+
for await (const key of this.map.keysAsync(range)) {
|
|
30
|
+
yield key;
|
|
31
|
+
}
|
|
32
|
+
}
|
|
33
|
+
}
|
package/src/lmdb-v2/singleton.ts
CHANGED
|
@@ -1,13 +1,17 @@
|
|
|
1
1
|
import { Encoder } from 'msgpackr';
|
|
2
2
|
|
|
3
3
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
4
|
+
// eslint-disable-next-line import/no-cycle
|
|
4
5
|
import { type AztecLMDBStoreV2, execInReadTx, execInWriteTx } from './store.js';
|
|
5
6
|
import { serializeKey } from './utils.js';
|
|
6
7
|
|
|
7
8
|
export class LMDBSingleValue<T> implements AztecAsyncSingleton<T> {
|
|
8
9
|
private key: Uint8Array;
|
|
9
10
|
private encoder = new Encoder();
|
|
10
|
-
constructor(
|
|
11
|
+
constructor(
|
|
12
|
+
private store: AztecLMDBStoreV2,
|
|
13
|
+
name: string,
|
|
14
|
+
) {
|
|
11
15
|
this.key = serializeKey(`singleton:${name}`, 'value');
|
|
12
16
|
}
|
|
13
17
|
|
package/src/lmdb-v2/store.ts
CHANGED
|
@@ -3,17 +3,20 @@ import { Semaphore, SerialQueue } from '@aztec/foundation/queue';
|
|
|
3
3
|
import { MsgpackChannel, NativeLMDBStore } from '@aztec/native';
|
|
4
4
|
|
|
5
5
|
import { AsyncLocalStorage } from 'async_hooks';
|
|
6
|
-
import { rm } from 'fs/promises';
|
|
6
|
+
import { mkdir, rm } from 'fs/promises';
|
|
7
7
|
|
|
8
8
|
import type { AztecAsyncArray } from '../interfaces/array.js';
|
|
9
|
-
import type { Key, StoreSize } from '../interfaces/common.js';
|
|
9
|
+
import type { Key, StoreSize, Value } from '../interfaces/common.js';
|
|
10
10
|
import type { AztecAsyncCounter } from '../interfaces/counter.js';
|
|
11
|
-
import type { AztecAsyncMap
|
|
11
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
12
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
12
13
|
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
13
14
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
14
15
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
16
|
+
// eslint-disable-next-line import/no-cycle
|
|
15
17
|
import { LMDBArray } from './array.js';
|
|
16
|
-
|
|
18
|
+
// eslint-disable-next-line import/no-cycle
|
|
19
|
+
import { LMDBMap } from './map.js';
|
|
17
20
|
import {
|
|
18
21
|
Database,
|
|
19
22
|
type LMDBMessageChannel,
|
|
@@ -21,7 +24,10 @@ import {
|
|
|
21
24
|
type LMDBRequestBody,
|
|
22
25
|
type LMDBResponseBody,
|
|
23
26
|
} from './message.js';
|
|
27
|
+
import { LMDBMultiMap } from './multi_map.js';
|
|
24
28
|
import { ReadTransaction } from './read_transaction.js';
|
|
29
|
+
import { LMDBSet } from './set.js';
|
|
30
|
+
// eslint-disable-next-line import/no-cycle
|
|
25
31
|
import { LMDBSingleValue } from './singleton.js';
|
|
26
32
|
import { WriteTransaction } from './write_transaction.js';
|
|
27
33
|
|
|
@@ -77,6 +83,11 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
77
83
|
return db;
|
|
78
84
|
}
|
|
79
85
|
|
|
86
|
+
public async backupTo(dstPath: string, compact = true) {
|
|
87
|
+
await mkdir(dstPath, { recursive: true });
|
|
88
|
+
await this.channel.sendMessage(LMDBMessageType.COPY_STORE, { dstPath, compact });
|
|
89
|
+
}
|
|
90
|
+
|
|
80
91
|
public getReadTx(): ReadTransaction {
|
|
81
92
|
if (!this.open) {
|
|
82
93
|
throw new Error('Store is closed');
|
|
@@ -92,24 +103,24 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
92
103
|
return currentWrite;
|
|
93
104
|
}
|
|
94
105
|
|
|
95
|
-
openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
|
|
106
|
+
openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
|
|
96
107
|
return new LMDBMap(this, name);
|
|
97
108
|
}
|
|
98
109
|
|
|
99
|
-
openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
|
|
110
|
+
openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
|
|
100
111
|
return new LMDBMultiMap(this, name);
|
|
101
112
|
}
|
|
102
113
|
|
|
103
|
-
openSingleton<T>(name: string): AztecAsyncSingleton<T> {
|
|
114
|
+
openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
|
|
104
115
|
return new LMDBSingleValue(this, name);
|
|
105
116
|
}
|
|
106
117
|
|
|
107
|
-
openArray<T>(name: string): AztecAsyncArray<T> {
|
|
118
|
+
openArray<T extends Value>(name: string): AztecAsyncArray<T> {
|
|
108
119
|
return new LMDBArray(this, name);
|
|
109
120
|
}
|
|
110
121
|
|
|
111
|
-
openSet<K extends Key>(
|
|
112
|
-
|
|
122
|
+
openSet<K extends Key>(name: string): AztecAsyncSet<K> {
|
|
123
|
+
return new LMDBSet(this, name);
|
|
113
124
|
}
|
|
114
125
|
|
|
115
126
|
openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
|
|
@@ -150,10 +161,6 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
150
161
|
return Promise.resolve();
|
|
151
162
|
}
|
|
152
163
|
|
|
153
|
-
fork(): Promise<AztecAsyncKVStore> {
|
|
154
|
-
throw new Error('Not implemented');
|
|
155
|
-
}
|
|
156
|
-
|
|
157
164
|
async delete(): Promise<void> {
|
|
158
165
|
await this.close();
|
|
159
166
|
await rm(this.dataDir, { recursive: true, force: true, maxRetries: 3 });
|
|
@@ -204,6 +211,7 @@ export class AztecLMDBStoreV2 implements AztecAsyncKVStore, LMDBMessageChannel {
|
|
|
204
211
|
const resp = await this.sendMessage(LMDBMessageType.STATS, undefined);
|
|
205
212
|
return {
|
|
206
213
|
mappingSize: Number(resp.dbMapSizeBytes),
|
|
214
|
+
physicalFileSize: Number(resp.dbPhysicalFileSizeBytes),
|
|
207
215
|
actualSize: resp.stats.reduce((s, db) => Number(db.totalUsedSize) + s, 0),
|
|
208
216
|
numItems: resp.stats.reduce((s, db) => Number(db.numDataItems) + s, 0),
|
|
209
217
|
};
|
|
@@ -163,7 +163,7 @@ export class WriteTransaction extends ReadTransaction {
|
|
|
163
163
|
|
|
164
164
|
public override async *iterate(
|
|
165
165
|
startKey: Uint8Array,
|
|
166
|
-
endKey?: Uint8Array
|
|
166
|
+
endKey?: Uint8Array,
|
|
167
167
|
reverse?: boolean,
|
|
168
168
|
limit?: number,
|
|
169
169
|
): AsyncIterable<[Uint8Array, Uint8Array]> {
|
|
@@ -181,7 +181,7 @@ export class WriteTransaction extends ReadTransaction {
|
|
|
181
181
|
|
|
182
182
|
public override async *iterateIndex(
|
|
183
183
|
startKey: Uint8Array,
|
|
184
|
-
endKey?: Uint8Array
|
|
184
|
+
endKey?: Uint8Array,
|
|
185
185
|
reverse?: boolean,
|
|
186
186
|
limit?: number,
|
|
187
187
|
): AsyncIterable<[Uint8Array, Uint8Array[]]> {
|
package/src/stores/index.ts
CHANGED
|
@@ -11,7 +11,7 @@ import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
|
11
11
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
12
12
|
|
|
13
13
|
/** Stores currently synced L2 tips and unfinalized block hashes. */
|
|
14
|
-
export class
|
|
14
|
+
export class L2TipsKVStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
|
|
15
15
|
private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, number>;
|
|
16
16
|
private readonly l2BlockHashesStore: AztecAsyncMap<number, string>;
|
|
17
17
|
|
|
@@ -47,24 +47,33 @@ export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLoca
|
|
|
47
47
|
|
|
48
48
|
public async handleBlockStreamEvent(event: L2BlockStreamEvent): Promise<void> {
|
|
49
49
|
switch (event.type) {
|
|
50
|
-
case 'blocks-added':
|
|
51
|
-
|
|
52
|
-
|
|
50
|
+
case 'blocks-added': {
|
|
51
|
+
const blocks = event.blocks.map(b => b.block);
|
|
52
|
+
for (const block of blocks) {
|
|
53
|
+
await this.l2BlockHashesStore.set(block.number, (await block.hash()).toString());
|
|
53
54
|
}
|
|
54
|
-
await this.l2TipsStore.set('latest',
|
|
55
|
+
await this.l2TipsStore.set('latest', blocks.at(-1)!.number);
|
|
55
56
|
break;
|
|
57
|
+
}
|
|
56
58
|
case 'chain-pruned':
|
|
57
|
-
await this.
|
|
59
|
+
await this.saveTag('latest', event.block);
|
|
58
60
|
break;
|
|
59
61
|
case 'chain-proven':
|
|
60
|
-
await this.
|
|
62
|
+
await this.saveTag('proven', event.block);
|
|
61
63
|
break;
|
|
62
64
|
case 'chain-finalized':
|
|
63
|
-
await this.
|
|
64
|
-
for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.
|
|
65
|
+
await this.saveTag('finalized', event.block);
|
|
66
|
+
for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.block.number })) {
|
|
65
67
|
await this.l2BlockHashesStore.delete(key);
|
|
66
68
|
}
|
|
67
69
|
break;
|
|
68
70
|
}
|
|
69
71
|
}
|
|
72
|
+
|
|
73
|
+
private async saveTag(name: L2BlockTag, block: L2BlockId) {
|
|
74
|
+
await this.l2TipsStore.set(name, block.number);
|
|
75
|
+
if (block.hash) {
|
|
76
|
+
await this.l2BlockHashesStore.set(block.number, block.hash);
|
|
77
|
+
}
|
|
78
|
+
}
|
|
70
79
|
}
|
|
@@ -1,3 +0,0 @@
|
|
|
1
|
-
import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
|
|
2
|
-
export declare function describeAztecStore(testName: string, getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>, getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>): void;
|
|
3
|
-
//# sourceMappingURL=store_test_suite.d.ts.map
|
|
@@ -1 +0,0 @@
|
|
|
1
|
-
{"version":3,"file":"store_test_suite.d.ts","sourceRoot":"","sources":["../../src/interfaces/store_test_suite.ts"],"names":[],"mappings":"AAGA,OAAO,KAAK,EAAE,iBAAiB,EAAE,YAAY,EAAE,MAAM,YAAY,CAAC;AAGlE,wBAAgB,kBAAkB,CAChC,QAAQ,EAAE,MAAM,EAChB,kBAAkB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACnE,wBAAwB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,EACzE,iBAAiB,EAAE,MAAM,OAAO,CAAC,YAAY,GAAG,iBAAiB,CAAC,QA6CnE"}
|
|
@@ -1,37 +0,0 @@
|
|
|
1
|
-
import { expect } from 'chai';
|
|
2
|
-
import { isSyncStore } from './utils.js';
|
|
3
|
-
export function describeAztecStore(testName, getPersistentStore, getPersistentNoPathStore, getEphemeralStore) {
|
|
4
|
-
describe(testName, ()=>{
|
|
5
|
-
async function get(store, singleton) {
|
|
6
|
-
return isSyncStore(store) ? singleton.get() : await singleton.getAsync();
|
|
7
|
-
}
|
|
8
|
-
const itForks = async (store)=>{
|
|
9
|
-
const singleton = store.openSingleton('singleton');
|
|
10
|
-
await singleton.set('foo');
|
|
11
|
-
const forkedStore = await store.fork();
|
|
12
|
-
const forkedSingleton = forkedStore.openSingleton('singleton');
|
|
13
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
14
|
-
await forkedSingleton.set('bar');
|
|
15
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
16
|
-
expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
|
|
17
|
-
await forkedSingleton.delete();
|
|
18
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
19
|
-
await forkedStore.delete();
|
|
20
|
-
};
|
|
21
|
-
it('forks a persistent store', async ()=>{
|
|
22
|
-
const store = await getPersistentStore();
|
|
23
|
-
await itForks(store);
|
|
24
|
-
await store.delete();
|
|
25
|
-
});
|
|
26
|
-
it('forks a persistent store with no path', async ()=>{
|
|
27
|
-
const store = await getPersistentNoPathStore();
|
|
28
|
-
await itForks(store);
|
|
29
|
-
await store.delete();
|
|
30
|
-
});
|
|
31
|
-
it('forks an ephemeral store', async ()=>{
|
|
32
|
-
const store = await getEphemeralStore();
|
|
33
|
-
await itForks(store);
|
|
34
|
-
await store.delete();
|
|
35
|
-
});
|
|
36
|
-
});
|
|
37
|
-
}
|
|
@@ -1,56 +0,0 @@
|
|
|
1
|
-
import { expect } from 'chai';
|
|
2
|
-
|
|
3
|
-
import type { AztecAsyncSingleton, AztecSingleton } from './singleton.js';
|
|
4
|
-
import type { AztecAsyncKVStore, AztecKVStore } from './store.js';
|
|
5
|
-
import { isSyncStore } from './utils.js';
|
|
6
|
-
|
|
7
|
-
export function describeAztecStore(
|
|
8
|
-
testName: string,
|
|
9
|
-
getPersistentStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
|
|
10
|
-
getPersistentNoPathStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
|
|
11
|
-
getEphemeralStore: () => Promise<AztecKVStore | AztecAsyncKVStore>,
|
|
12
|
-
) {
|
|
13
|
-
describe(testName, () => {
|
|
14
|
-
async function get(
|
|
15
|
-
store: AztecKVStore | AztecAsyncKVStore,
|
|
16
|
-
singleton: AztecSingleton<string> | AztecAsyncSingleton<string>,
|
|
17
|
-
) {
|
|
18
|
-
return isSyncStore(store)
|
|
19
|
-
? (singleton as AztecSingleton<string>).get()
|
|
20
|
-
: await (singleton as AztecAsyncSingleton<string>).getAsync();
|
|
21
|
-
}
|
|
22
|
-
|
|
23
|
-
const itForks = async (store: AztecKVStore | AztecAsyncKVStore) => {
|
|
24
|
-
const singleton = store.openSingleton<string>('singleton');
|
|
25
|
-
await singleton.set('foo');
|
|
26
|
-
|
|
27
|
-
const forkedStore = await store.fork();
|
|
28
|
-
const forkedSingleton = forkedStore.openSingleton<string>('singleton');
|
|
29
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
30
|
-
await forkedSingleton.set('bar');
|
|
31
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
32
|
-
expect(await get(forkedStore, forkedSingleton)).to.equal('bar');
|
|
33
|
-
await forkedSingleton.delete();
|
|
34
|
-
expect(await get(store, singleton)).to.equal('foo');
|
|
35
|
-
await forkedStore.delete();
|
|
36
|
-
};
|
|
37
|
-
|
|
38
|
-
it('forks a persistent store', async () => {
|
|
39
|
-
const store = await getPersistentStore();
|
|
40
|
-
await itForks(store);
|
|
41
|
-
await store.delete();
|
|
42
|
-
});
|
|
43
|
-
|
|
44
|
-
it('forks a persistent store with no path', async () => {
|
|
45
|
-
const store = await getPersistentNoPathStore();
|
|
46
|
-
await itForks(store);
|
|
47
|
-
await store.delete();
|
|
48
|
-
});
|
|
49
|
-
|
|
50
|
-
it('forks an ephemeral store', async () => {
|
|
51
|
-
const store = await getEphemeralStore();
|
|
52
|
-
await itForks(store);
|
|
53
|
-
await store.delete();
|
|
54
|
-
});
|
|
55
|
-
});
|
|
56
|
-
}
|