@aztec/kv-store 0.66.0 → 0.67.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/indexeddb/array.d.ts +21 -0
- package/dest/indexeddb/array.d.ts.map +1 -0
- package/dest/indexeddb/array.js +96 -0
- package/dest/indexeddb/index.d.ts +7 -0
- package/dest/indexeddb/index.d.ts.map +1 -0
- package/dest/indexeddb/index.js +22 -0
- package/dest/indexeddb/map.d.ts +26 -0
- package/dest/indexeddb/map.d.ts.map +1 -0
- package/dest/indexeddb/map.js +104 -0
- package/dest/indexeddb/set.d.ts +17 -0
- package/dest/indexeddb/set.d.ts.map +1 -0
- package/dest/indexeddb/set.js +25 -0
- package/dest/indexeddb/singleton.d.ts +16 -0
- package/dest/indexeddb/singleton.d.ts.map +1 -0
- package/dest/indexeddb/singleton.js +42 -0
- package/dest/indexeddb/store.d.ts +100 -0
- package/dest/indexeddb/store.d.ts.map +1 -0
- package/dest/indexeddb/store.js +156 -0
- package/dest/interfaces/array.d.ts +43 -11
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array_test_suite.d.ts +3 -0
- package/dest/interfaces/array_test_suite.d.ts.map +1 -0
- package/dest/interfaces/array_test_suite.js +97 -0
- package/dest/interfaces/counter.d.ts +21 -1
- package/dest/interfaces/counter.d.ts.map +1 -1
- package/dest/interfaces/map.d.ts +62 -12
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.d.ts +3 -0
- package/dest/interfaces/map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/map_test_suite.js +114 -0
- package/dest/interfaces/set.d.ts +23 -7
- package/dest/interfaces/set.d.ts.map +1 -1
- package/dest/interfaces/set_test_suite.d.ts +3 -0
- package/dest/interfaces/set_test_suite.d.ts.map +1 -0
- package/dest/interfaces/set_test_suite.js +56 -0
- package/dest/interfaces/singleton.d.ts +14 -5
- package/dest/interfaces/singleton.d.ts.map +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +3 -0
- package/dest/interfaces/singleton_test_suite.d.ts.map +1 -0
- package/dest/interfaces/singleton_test_suite.js +30 -0
- package/dest/interfaces/store.d.ts +71 -8
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/store_test_suite.d.ts +3 -0
- package/dest/interfaces/store_test_suite.d.ts.map +1 -0
- package/dest/interfaces/store_test_suite.js +36 -0
- package/dest/interfaces/utils.d.ts +16 -0
- package/dest/interfaces/utils.d.ts.map +1 -0
- package/dest/interfaces/utils.js +19 -0
- package/dest/lmdb/array.d.ts +7 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +20 -1
- package/dest/lmdb/counter.d.ts +5 -2
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/counter.js +10 -1
- package/dest/lmdb/index.d.ts +10 -0
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +28 -1
- package/dest/lmdb/map.d.ts +8 -2
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +27 -1
- package/dest/lmdb/set.d.ts +4 -2
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/set.js +9 -1
- package/dest/lmdb/singleton.d.ts +3 -2
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/singleton.js +4 -1
- package/dest/lmdb/store.d.ts +21 -13
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +23 -19
- package/dest/stores/l2_tips_store.d.ts +2 -2
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +12 -12
- package/dest/utils.d.ts +8 -7
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +6 -29
- package/package.json +54 -37
- package/src/indexeddb/array.ts +118 -0
- package/src/indexeddb/index.ts +29 -0
- package/src/indexeddb/map.ts +142 -0
- package/src/indexeddb/set.ts +37 -0
- package/src/indexeddb/singleton.ts +49 -0
- package/src/indexeddb/store.ts +192 -0
- package/src/interfaces/array.ts +48 -12
- package/src/interfaces/array_test_suite.ts +126 -0
- package/src/interfaces/counter.ts +23 -1
- package/src/interfaces/map.ts +69 -14
- package/src/interfaces/map_test_suite.ts +154 -0
- package/src/interfaces/set.ts +25 -8
- package/src/interfaces/set_test_suite.ts +77 -0
- package/src/interfaces/singleton.ts +14 -6
- package/src/interfaces/singleton_test_suite.ts +42 -0
- package/src/interfaces/store.ts +78 -8
- package/src/interfaces/store_test_suite.ts +52 -0
- package/src/interfaces/utils.ts +21 -0
- package/src/lmdb/array.ts +26 -2
- package/src/lmdb/counter.ts +14 -2
- package/src/lmdb/index.ts +36 -0
- package/src/lmdb/map.ts +34 -2
- package/src/lmdb/set.ts +12 -2
- package/src/lmdb/singleton.ts +6 -2
- package/src/lmdb/store.ts +39 -32
- package/src/stores/l2_tips_store.ts +16 -16
- package/src/utils.ts +8 -37
package/src/lmdb/array.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Database, type Key } from 'lmdb';
|
|
2
2
|
|
|
3
|
-
import { type AztecArray } from '../interfaces/array.js';
|
|
3
|
+
import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
|
|
4
4
|
import { LmdbAztecSingleton } from './singleton.js';
|
|
5
5
|
|
|
6
6
|
/** The shape of a key that stores a value in an array */
|
|
@@ -9,7 +9,7 @@ type ArrayIndexSlot = ['array', string, 'slot', number];
|
|
|
9
9
|
/**
|
|
10
10
|
* An persistent array backed by LMDB.
|
|
11
11
|
*/
|
|
12
|
-
export class LmdbAztecArray<T> implements AztecArray<T> {
|
|
12
|
+
export class LmdbAztecArray<T> implements AztecArray<T>, AztecAsyncArray<T> {
|
|
13
13
|
#db: Database<T, ArrayIndexSlot>;
|
|
14
14
|
#name: string;
|
|
15
15
|
#length: LmdbAztecSingleton<number>;
|
|
@@ -24,6 +24,10 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
|
|
|
24
24
|
return this.#length.get() ?? 0;
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
+
lengthAsync(): Promise<number> {
|
|
28
|
+
return Promise.resolve(this.length);
|
|
29
|
+
}
|
|
30
|
+
|
|
27
31
|
push(...vals: T[]): Promise<number> {
|
|
28
32
|
return this.#db.childTransaction(() => {
|
|
29
33
|
let length = this.length;
|
|
@@ -69,6 +73,10 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
|
|
|
69
73
|
return this.#db.get(this.#slot(index));
|
|
70
74
|
}
|
|
71
75
|
|
|
76
|
+
atAsync(index: number): Promise<T | undefined> {
|
|
77
|
+
return Promise.resolve(this.at(index));
|
|
78
|
+
}
|
|
79
|
+
|
|
72
80
|
setAt(index: number, val: T): Promise<boolean> {
|
|
73
81
|
if (index < 0) {
|
|
74
82
|
index = this.length + index;
|
|
@@ -93,16 +101,32 @@ export class LmdbAztecArray<T> implements AztecArray<T> {
|
|
|
93
101
|
}
|
|
94
102
|
}
|
|
95
103
|
|
|
104
|
+
async *entriesAsync(): AsyncIterableIterator<[number, T]> {
|
|
105
|
+
for (const [key, value] of this.entries()) {
|
|
106
|
+
yield [key, value];
|
|
107
|
+
}
|
|
108
|
+
}
|
|
109
|
+
|
|
96
110
|
*values(): IterableIterator<T> {
|
|
97
111
|
for (const [_, value] of this.entries()) {
|
|
98
112
|
yield value;
|
|
99
113
|
}
|
|
100
114
|
}
|
|
101
115
|
|
|
116
|
+
async *valuesAsync(): AsyncIterableIterator<T> {
|
|
117
|
+
for (const [_, value] of this.entries()) {
|
|
118
|
+
yield value;
|
|
119
|
+
}
|
|
120
|
+
}
|
|
121
|
+
|
|
102
122
|
[Symbol.iterator](): IterableIterator<T> {
|
|
103
123
|
return this.values();
|
|
104
124
|
}
|
|
105
125
|
|
|
126
|
+
[Symbol.asyncIterator](): AsyncIterableIterator<T> {
|
|
127
|
+
return this.valuesAsync();
|
|
128
|
+
}
|
|
129
|
+
|
|
106
130
|
#slot(index: number): ArrayIndexSlot {
|
|
107
131
|
return ['array', this.#name, 'slot', index];
|
|
108
132
|
}
|
package/src/lmdb/counter.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { type Key as BaseKey, type Database } from 'lmdb';
|
|
2
2
|
|
|
3
3
|
import { type Key, type Range } from '../interfaces/common.js';
|
|
4
|
-
import { type AztecCounter } from '../interfaces/counter.js';
|
|
4
|
+
import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
|
|
5
5
|
import { LmdbAztecMap } from './map.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* A counter implementation backed by LMDB
|
|
9
9
|
*/
|
|
10
|
-
export class LmdbAztecCounter<K extends Key> implements AztecCounter<K> {
|
|
10
|
+
export class LmdbAztecCounter<K extends Key> implements AztecCounter<K>, AztecAsyncCounter<K> {
|
|
11
11
|
#db: Database;
|
|
12
12
|
#name: string;
|
|
13
13
|
#map: LmdbAztecMap<K, number>;
|
|
@@ -45,11 +45,23 @@ export class LmdbAztecCounter<K extends Key> implements AztecCounter<K> {
|
|
|
45
45
|
return this.#map.get(key) ?? 0;
|
|
46
46
|
}
|
|
47
47
|
|
|
48
|
+
getAsync(key: K): Promise<number> {
|
|
49
|
+
return Promise.resolve(this.get(key));
|
|
50
|
+
}
|
|
51
|
+
|
|
48
52
|
entries(range: Range<K> = {}): IterableIterator<[K, number]> {
|
|
49
53
|
return this.#map.entries(range);
|
|
50
54
|
}
|
|
51
55
|
|
|
56
|
+
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, number]> {
|
|
57
|
+
yield* this.entries(range);
|
|
58
|
+
}
|
|
59
|
+
|
|
52
60
|
keys(range: Range<K> = {}): IterableIterator<K> {
|
|
53
61
|
return this.#map.keys(range);
|
|
54
62
|
}
|
|
63
|
+
|
|
64
|
+
async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
65
|
+
yield* this.keys(range);
|
|
66
|
+
}
|
|
55
67
|
}
|
package/src/lmdb/index.ts
CHANGED
|
@@ -1 +1,37 @@
|
|
|
1
|
+
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
|
|
3
|
+
import { join } from 'path';
|
|
4
|
+
|
|
5
|
+
import { type DataStoreConfig } from '../config.js';
|
|
6
|
+
import { initStoreForRollup } from '../utils.js';
|
|
7
|
+
import { AztecLmdbStore } from './store.js';
|
|
8
|
+
|
|
1
9
|
export { AztecLmdbStore } from './store.js';
|
|
10
|
+
|
|
11
|
+
export function createStore(name: string, config: DataStoreConfig, log: Logger = createLogger('kv-store')) {
|
|
12
|
+
let { dataDirectory } = config;
|
|
13
|
+
if (typeof dataDirectory !== 'undefined') {
|
|
14
|
+
dataDirectory = join(dataDirectory, name);
|
|
15
|
+
}
|
|
16
|
+
|
|
17
|
+
log.info(
|
|
18
|
+
dataDirectory
|
|
19
|
+
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
|
|
20
|
+
: `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
|
|
21
|
+
);
|
|
22
|
+
|
|
23
|
+
const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false);
|
|
24
|
+
if (config.l1Contracts?.rollupAddress) {
|
|
25
|
+
return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
|
|
26
|
+
}
|
|
27
|
+
return store;
|
|
28
|
+
}
|
|
29
|
+
/**
|
|
30
|
+
* Opens a temporary store for testing purposes.
|
|
31
|
+
* @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
|
|
32
|
+
* @returns A new store
|
|
33
|
+
*/
|
|
34
|
+
export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore {
|
|
35
|
+
const mapSize = 1024 * 1024 * 10; // 10 GB map size
|
|
36
|
+
return AztecLmdbStore.open(undefined, mapSize, ephemeral);
|
|
37
|
+
}
|
package/src/lmdb/map.ts
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { type Database, type RangeOptions } from 'lmdb';
|
|
2
2
|
|
|
3
3
|
import { type Key, type Range } from '../interfaces/common.js';
|
|
4
|
-
import { type AztecMultiMap } from '../interfaces/map.js';
|
|
4
|
+
import { type AztecAsyncMultiMap, type AztecMultiMap } from '../interfaces/map.js';
|
|
5
5
|
|
|
6
6
|
/** The slot where a key-value entry would be stored */
|
|
7
7
|
type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
|
|
@@ -9,7 +9,7 @@ type MapValueSlot<K extends Key | Buffer> = ['map', string, 'slot', K];
|
|
|
9
9
|
/**
|
|
10
10
|
* A map backed by LMDB.
|
|
11
11
|
*/
|
|
12
|
-
export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
|
|
12
|
+
export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V>, AztecAsyncMultiMap<K, V> {
|
|
13
13
|
protected db: Database<[K, V], MapValueSlot<K>>;
|
|
14
14
|
protected name: string;
|
|
15
15
|
|
|
@@ -35,6 +35,10 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
|
|
|
35
35
|
return this.db.get(this.#slot(key))?.[1];
|
|
36
36
|
}
|
|
37
37
|
|
|
38
|
+
getAsync(key: K): Promise<V | undefined> {
|
|
39
|
+
return Promise.resolve(this.get(key));
|
|
40
|
+
}
|
|
41
|
+
|
|
38
42
|
*getValues(key: K): IterableIterator<V> {
|
|
39
43
|
const values = this.db.getValues(this.#slot(key));
|
|
40
44
|
for (const value of values) {
|
|
@@ -42,10 +46,20 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
|
|
|
42
46
|
}
|
|
43
47
|
}
|
|
44
48
|
|
|
49
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
50
|
+
for (const value of this.getValues(key)) {
|
|
51
|
+
yield value;
|
|
52
|
+
}
|
|
53
|
+
}
|
|
54
|
+
|
|
45
55
|
has(key: K): boolean {
|
|
46
56
|
return this.db.doesExist(this.#slot(key));
|
|
47
57
|
}
|
|
48
58
|
|
|
59
|
+
hasAsync(key: K): Promise<boolean> {
|
|
60
|
+
return Promise.resolve(this.has(key));
|
|
61
|
+
}
|
|
62
|
+
|
|
49
63
|
async set(key: K, val: V): Promise<void> {
|
|
50
64
|
await this.db.put(this.#slot(key), [key, val]);
|
|
51
65
|
}
|
|
@@ -109,18 +123,36 @@ export class LmdbAztecMap<K extends Key, V> implements AztecMultiMap<K, V> {
|
|
|
109
123
|
}
|
|
110
124
|
}
|
|
111
125
|
|
|
126
|
+
async *entriesAsync(range?: Range<K> | undefined): AsyncIterableIterator<[K, V]> {
|
|
127
|
+
for (const entry of this.entries(range)) {
|
|
128
|
+
yield entry;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
112
132
|
*values(range: Range<K> = {}): IterableIterator<V> {
|
|
113
133
|
for (const [_, value] of this.entries(range)) {
|
|
114
134
|
yield value;
|
|
115
135
|
}
|
|
116
136
|
}
|
|
117
137
|
|
|
138
|
+
async *valuesAsync(range: Range<K> = {}): AsyncIterableIterator<V> {
|
|
139
|
+
for await (const [_, value] of this.entriesAsync(range)) {
|
|
140
|
+
yield value;
|
|
141
|
+
}
|
|
142
|
+
}
|
|
143
|
+
|
|
118
144
|
*keys(range: Range<K> = {}): IterableIterator<K> {
|
|
119
145
|
for (const [key, _] of this.entries(range)) {
|
|
120
146
|
yield key;
|
|
121
147
|
}
|
|
122
148
|
}
|
|
123
149
|
|
|
150
|
+
async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
151
|
+
for await (const [key, _] of this.entriesAsync(range)) {
|
|
152
|
+
yield key;
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
124
156
|
#slot(key: K): MapValueSlot<K> {
|
|
125
157
|
return ['map', this.name, 'slot', key];
|
|
126
158
|
}
|
package/src/lmdb/set.ts
CHANGED
|
@@ -1,13 +1,13 @@
|
|
|
1
1
|
import { type Database } from 'lmdb';
|
|
2
2
|
|
|
3
3
|
import { type Key, type Range } from '../interfaces/common.js';
|
|
4
|
-
import { type AztecSet } from '../interfaces/set.js';
|
|
4
|
+
import { type AztecAsyncSet, type AztecSet } from '../interfaces/set.js';
|
|
5
5
|
import { LmdbAztecMap } from './map.js';
|
|
6
6
|
|
|
7
7
|
/**
|
|
8
8
|
* A set backed by LMDB.
|
|
9
9
|
*/
|
|
10
|
-
export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
|
|
10
|
+
export class LmdbAztecSet<K extends Key> implements AztecSet<K>, AztecAsyncSet<K> {
|
|
11
11
|
private map: LmdbAztecMap<K, boolean>;
|
|
12
12
|
constructor(rootDb: Database, mapName: string) {
|
|
13
13
|
this.map = new LmdbAztecMap(rootDb, mapName);
|
|
@@ -21,6 +21,10 @@ export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
|
|
|
21
21
|
return this.map.has(key);
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
hasAsync(key: K): Promise<boolean> {
|
|
25
|
+
return Promise.resolve(this.has(key));
|
|
26
|
+
}
|
|
27
|
+
|
|
24
28
|
add(key: K): Promise<void> {
|
|
25
29
|
return this.map.set(key, true);
|
|
26
30
|
}
|
|
@@ -32,4 +36,10 @@ export class LmdbAztecSet<K extends Key> implements AztecSet<K> {
|
|
|
32
36
|
entries(range: Range<K> = {}): IterableIterator<K> {
|
|
33
37
|
return this.map.keys(range);
|
|
34
38
|
}
|
|
39
|
+
|
|
40
|
+
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
41
|
+
for await (const key of this.map.keysAsync(range)) {
|
|
42
|
+
yield key;
|
|
43
|
+
}
|
|
44
|
+
}
|
|
35
45
|
}
|
package/src/lmdb/singleton.ts
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
import { type Database, type Key } from 'lmdb';
|
|
2
2
|
|
|
3
|
-
import { type AztecSingleton } from '../interfaces/singleton.js';
|
|
3
|
+
import { type AztecAsyncSingleton, type AztecSingleton } from '../interfaces/singleton.js';
|
|
4
4
|
|
|
5
5
|
/** The slot where this singleton will store its value */
|
|
6
6
|
type ValueSlot = ['singleton', string, 'value'];
|
|
@@ -8,7 +8,7 @@ type ValueSlot = ['singleton', string, 'value'];
|
|
|
8
8
|
/**
|
|
9
9
|
* Stores a single value in LMDB.
|
|
10
10
|
*/
|
|
11
|
-
export class LmdbAztecSingleton<T> implements AztecSingleton<T> {
|
|
11
|
+
export class LmdbAztecSingleton<T> implements AztecSingleton<T>, AztecAsyncSingleton<T> {
|
|
12
12
|
#db: Database<T, ValueSlot>;
|
|
13
13
|
#slot: ValueSlot;
|
|
14
14
|
|
|
@@ -21,6 +21,10 @@ export class LmdbAztecSingleton<T> implements AztecSingleton<T> {
|
|
|
21
21
|
return this.#db.get(this.#slot);
|
|
22
22
|
}
|
|
23
23
|
|
|
24
|
+
getAsync(): Promise<T | undefined> {
|
|
25
|
+
return Promise.resolve(this.get());
|
|
26
|
+
}
|
|
27
|
+
|
|
24
28
|
set(val: T): Promise<boolean> {
|
|
25
29
|
return this.#db.put(this.#slot, val);
|
|
26
30
|
}
|
package/src/lmdb/store.ts
CHANGED
|
@@ -1,17 +1,17 @@
|
|
|
1
|
-
import {
|
|
1
|
+
import { createLogger } from '@aztec/foundation/log';
|
|
2
2
|
|
|
3
|
-
import { mkdirSync } from 'fs';
|
|
4
|
-
import {
|
|
5
|
-
import { type Database, type Key, type RootDatabase, open } from 'lmdb';
|
|
3
|
+
import { promises as fs, mkdirSync } from 'fs';
|
|
4
|
+
import { type Database, type RootDatabase, open } from 'lmdb';
|
|
6
5
|
import { tmpdir } from 'os';
|
|
7
6
|
import { dirname, join } from 'path';
|
|
8
7
|
|
|
9
|
-
import { type AztecArray } from '../interfaces/array.js';
|
|
10
|
-
import { type
|
|
11
|
-
import { type
|
|
12
|
-
import { type
|
|
13
|
-
import { type
|
|
14
|
-
import { type
|
|
8
|
+
import { type AztecArray, type AztecAsyncArray } from '../interfaces/array.js';
|
|
9
|
+
import { type Key } from '../interfaces/common.js';
|
|
10
|
+
import { type AztecAsyncCounter, type AztecCounter } from '../interfaces/counter.js';
|
|
11
|
+
import { type AztecAsyncMap, type AztecAsyncMultiMap, type AztecMap, type AztecMultiMap } from '../interfaces/map.js';
|
|
12
|
+
import { type AztecAsyncSet, type AztecSet } from '../interfaces/set.js';
|
|
13
|
+
import { type AztecAsyncSingleton, type AztecSingleton } from '../interfaces/singleton.js';
|
|
14
|
+
import { type AztecAsyncKVStore, type AztecKVStore } from '../interfaces/store.js';
|
|
15
15
|
import { LmdbAztecArray } from './array.js';
|
|
16
16
|
import { LmdbAztecCounter } from './counter.js';
|
|
17
17
|
import { LmdbAztecMap } from './map.js';
|
|
@@ -21,11 +21,13 @@ import { LmdbAztecSingleton } from './singleton.js';
|
|
|
21
21
|
/**
|
|
22
22
|
* A key-value store backed by LMDB.
|
|
23
23
|
*/
|
|
24
|
-
export class AztecLmdbStore implements AztecKVStore {
|
|
24
|
+
export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
25
|
+
syncGetters = true as const;
|
|
26
|
+
|
|
25
27
|
#rootDb: RootDatabase;
|
|
26
28
|
#data: Database<unknown, Key>;
|
|
27
29
|
#multiMapData: Database<unknown, Key>;
|
|
28
|
-
#log =
|
|
30
|
+
#log = createLogger('kv-store:lmdb');
|
|
29
31
|
|
|
30
32
|
constructor(rootDb: RootDatabase, public readonly isEphemeral: boolean, private path?: string) {
|
|
31
33
|
this.#rootDb = rootDb;
|
|
@@ -60,7 +62,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
60
62
|
path?: string,
|
|
61
63
|
mapSizeKb = 1 * 1024 * 1024, // defaults to 1 GB map size
|
|
62
64
|
ephemeral: boolean = false,
|
|
63
|
-
log =
|
|
65
|
+
log = createLogger('kv-store:lmdb'),
|
|
64
66
|
): AztecLmdbStore {
|
|
65
67
|
if (path) {
|
|
66
68
|
mkdirSync(path, { recursive: true });
|
|
@@ -79,7 +81,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
79
81
|
const baseDir = this.path ? dirname(this.path) : tmpdir();
|
|
80
82
|
this.#log.debug(`Forking store with basedir ${baseDir}`);
|
|
81
83
|
const forkPath =
|
|
82
|
-
(await mkdtemp(join(baseDir, 'aztec-store-fork-'))) + (this.isEphemeral || !this.path ? '/data.mdb' : '');
|
|
84
|
+
(await fs.mkdtemp(join(baseDir, 'aztec-store-fork-'))) + (this.isEphemeral || !this.path ? '/data.mdb' : '');
|
|
83
85
|
this.#log.verbose(`Forking store to ${forkPath}`);
|
|
84
86
|
await this.#rootDb.backup(forkPath, false);
|
|
85
87
|
const forkDb = open(forkPath, { noSync: this.isEphemeral });
|
|
@@ -92,7 +94,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
92
94
|
* @param name - Name of the map
|
|
93
95
|
* @returns A new AztecMap
|
|
94
96
|
*/
|
|
95
|
-
openMap<K extends
|
|
97
|
+
openMap<K extends Key, V>(name: string): AztecMap<K, V> & AztecAsyncMap<K, V> {
|
|
96
98
|
return new LmdbAztecMap(this.#data, name);
|
|
97
99
|
}
|
|
98
100
|
|
|
@@ -101,7 +103,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
101
103
|
* @param name - Name of the set
|
|
102
104
|
* @returns A new AztecSet
|
|
103
105
|
*/
|
|
104
|
-
openSet<K extends
|
|
106
|
+
openSet<K extends Key>(name: string): AztecSet<K> & AztecAsyncSet<K> {
|
|
105
107
|
return new LmdbAztecSet(this.#data, name);
|
|
106
108
|
}
|
|
107
109
|
|
|
@@ -110,11 +112,11 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
110
112
|
* @param name - Name of the map
|
|
111
113
|
* @returns A new AztecMultiMap
|
|
112
114
|
*/
|
|
113
|
-
openMultiMap<K extends
|
|
115
|
+
openMultiMap<K extends Key, V>(name: string): AztecMultiMap<K, V> & AztecAsyncMultiMap<K, V> {
|
|
114
116
|
return new LmdbAztecMap(this.#multiMapData, name);
|
|
115
117
|
}
|
|
116
118
|
|
|
117
|
-
openCounter<K extends
|
|
119
|
+
openCounter<K extends Key>(name: string): AztecCounter<K> & AztecAsyncCounter<K> {
|
|
118
120
|
return new LmdbAztecCounter(this.#data, name);
|
|
119
121
|
}
|
|
120
122
|
|
|
@@ -123,7 +125,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
123
125
|
* @param name - Name of the array
|
|
124
126
|
* @returns A new AztecArray
|
|
125
127
|
*/
|
|
126
|
-
openArray<T>(name: string): AztecArray<T> {
|
|
128
|
+
openArray<T>(name: string): AztecArray<T> & AztecAsyncArray<T> {
|
|
127
129
|
return new LmdbAztecArray(this.#data, name);
|
|
128
130
|
}
|
|
129
131
|
|
|
@@ -132,7 +134,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
132
134
|
* @param name - Name of the singleton
|
|
133
135
|
* @returns A new AztecSingleton
|
|
134
136
|
*/
|
|
135
|
-
openSingleton<T>(name: string): AztecSingleton<T> {
|
|
137
|
+
openSingleton<T>(name: string): AztecSingleton<T> & AztecAsyncSingleton<T> {
|
|
136
138
|
return new LmdbAztecSingleton(this.#data, name);
|
|
137
139
|
}
|
|
138
140
|
|
|
@@ -145,6 +147,15 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
145
147
|
return this.#rootDb.transaction(callback);
|
|
146
148
|
}
|
|
147
149
|
|
|
150
|
+
/**
|
|
151
|
+
* Runs a callback in a transaction.
|
|
152
|
+
* @param callback - Function to execute in a transaction
|
|
153
|
+
* @returns A promise that resolves to the return value of the callback
|
|
154
|
+
*/
|
|
155
|
+
async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
|
|
156
|
+
return await this.#rootDb.transaction(callback);
|
|
157
|
+
}
|
|
158
|
+
|
|
148
159
|
/**
|
|
149
160
|
* Clears all entries in the store & sub DBs.
|
|
150
161
|
*/
|
|
@@ -177,7 +188,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
177
188
|
await this.drop();
|
|
178
189
|
await this.close();
|
|
179
190
|
if (this.path) {
|
|
180
|
-
await rm(this.path, { recursive: true, force: true });
|
|
191
|
+
await fs.rm(this.path, { recursive: true, force: true });
|
|
181
192
|
this.#log.verbose(`Deleted database files at ${this.path}`);
|
|
182
193
|
}
|
|
183
194
|
}
|
|
@@ -201,11 +212,7 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
201
212
|
|
|
202
213
|
private estimateSubDBSize(db: Database<unknown, Key>): { actualSize: number; numItems: number } {
|
|
203
214
|
const stats = db.getStats();
|
|
204
|
-
let
|
|
205
|
-
let leafPages = 0;
|
|
206
|
-
let overflowPages = 0;
|
|
207
|
-
let pageSize = 0;
|
|
208
|
-
let totalSize = 0;
|
|
215
|
+
let actualSize = 0;
|
|
209
216
|
let numItems = 0;
|
|
210
217
|
// This is the total number of key/value pairs present in the DB
|
|
211
218
|
if ('entryCount' in stats && typeof stats.entryCount === 'number') {
|
|
@@ -222,12 +229,12 @@ export class AztecLmdbStore implements AztecKVStore {
|
|
|
222
229
|
'pageSize' in stats &&
|
|
223
230
|
typeof stats.pageSize === 'number'
|
|
224
231
|
) {
|
|
225
|
-
branchPages = stats.treeBranchPageCount;
|
|
226
|
-
leafPages = stats.treeLeafPageCount;
|
|
227
|
-
overflowPages = stats.overflowPages;
|
|
228
|
-
pageSize = stats.pageSize;
|
|
229
|
-
|
|
232
|
+
const branchPages = stats.treeBranchPageCount;
|
|
233
|
+
const leafPages = stats.treeLeafPageCount;
|
|
234
|
+
const overflowPages = stats.overflowPages;
|
|
235
|
+
const pageSize = stats.pageSize;
|
|
236
|
+
actualSize = (branchPages + leafPages + overflowPages) * pageSize;
|
|
230
237
|
}
|
|
231
|
-
return { actualSize
|
|
238
|
+
return { actualSize, numItems };
|
|
232
239
|
}
|
|
233
240
|
}
|
|
@@ -7,37 +7,37 @@ import {
|
|
|
7
7
|
type L2Tips,
|
|
8
8
|
} from '@aztec/circuit-types';
|
|
9
9
|
|
|
10
|
-
import { type
|
|
11
|
-
import { type
|
|
10
|
+
import { type AztecAsyncMap } from '../interfaces/map.js';
|
|
11
|
+
import { type AztecAsyncKVStore } from '../interfaces/store.js';
|
|
12
12
|
|
|
13
13
|
/** Stores currently synced L2 tips and unfinalized block hashes. */
|
|
14
14
|
export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLocalDataProvider {
|
|
15
|
-
private readonly l2TipsStore:
|
|
16
|
-
private readonly l2BlockHashesStore:
|
|
15
|
+
private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, number>;
|
|
16
|
+
private readonly l2BlockHashesStore: AztecAsyncMap<number, string>;
|
|
17
17
|
|
|
18
|
-
constructor(store:
|
|
18
|
+
constructor(store: AztecAsyncKVStore, namespace: string) {
|
|
19
19
|
this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_'));
|
|
20
20
|
this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_'));
|
|
21
21
|
}
|
|
22
22
|
|
|
23
23
|
public getL2BlockHash(number: number): Promise<string | undefined> {
|
|
24
|
-
return
|
|
24
|
+
return this.l2BlockHashesStore.getAsync(number);
|
|
25
25
|
}
|
|
26
26
|
|
|
27
|
-
public getL2Tips(): Promise<L2Tips> {
|
|
28
|
-
return
|
|
29
|
-
latest: this.getL2Tip('latest'),
|
|
30
|
-
finalized: this.getL2Tip('finalized'),
|
|
31
|
-
proven: this.getL2Tip('proven'),
|
|
32
|
-
}
|
|
27
|
+
public async getL2Tips(): Promise<L2Tips> {
|
|
28
|
+
return {
|
|
29
|
+
latest: await this.getL2Tip('latest'),
|
|
30
|
+
finalized: await this.getL2Tip('finalized'),
|
|
31
|
+
proven: await this.getL2Tip('proven'),
|
|
32
|
+
};
|
|
33
33
|
}
|
|
34
34
|
|
|
35
|
-
private getL2Tip(tag: L2BlockTag): L2BlockId {
|
|
36
|
-
const blockNumber = this.l2TipsStore.
|
|
35
|
+
private async getL2Tip(tag: L2BlockTag): Promise<L2BlockId> {
|
|
36
|
+
const blockNumber = await this.l2TipsStore.getAsync(tag);
|
|
37
37
|
if (blockNumber === undefined || blockNumber === 0) {
|
|
38
38
|
return { number: 0, hash: undefined };
|
|
39
39
|
}
|
|
40
|
-
const blockHash = this.l2BlockHashesStore.
|
|
40
|
+
const blockHash = await this.l2BlockHashesStore.getAsync(blockNumber);
|
|
41
41
|
if (!blockHash) {
|
|
42
42
|
throw new Error(`Block hash not found for block number ${blockNumber}`);
|
|
43
43
|
}
|
|
@@ -60,7 +60,7 @@ export class L2TipsStore implements L2BlockStreamEventHandler, L2BlockStreamLoca
|
|
|
60
60
|
break;
|
|
61
61
|
case 'chain-finalized':
|
|
62
62
|
await this.l2TipsStore.set('finalized', event.blockNumber);
|
|
63
|
-
for (const key of this.l2BlockHashesStore.
|
|
63
|
+
for await (const key of this.l2BlockHashesStore.keysAsync({ end: event.blockNumber })) {
|
|
64
64
|
await this.l2BlockHashesStore.delete(key);
|
|
65
65
|
}
|
|
66
66
|
break;
|
package/src/utils.ts
CHANGED
|
@@ -1,30 +1,9 @@
|
|
|
1
1
|
import { type EthAddress } from '@aztec/foundation/eth-address';
|
|
2
|
-
import { type Logger
|
|
2
|
+
import { type Logger } from '@aztec/foundation/log';
|
|
3
3
|
|
|
4
|
-
import {
|
|
5
|
-
|
|
6
|
-
import {
|
|
7
|
-
import { type AztecKVStore } from './interfaces/store.js';
|
|
8
|
-
import { AztecLmdbStore } from './lmdb/store.js';
|
|
9
|
-
|
|
10
|
-
export function createStore(name: string, config: DataStoreConfig, log: Logger = createDebugLogger('aztec:kv-store')) {
|
|
11
|
-
let { dataDirectory } = config;
|
|
12
|
-
if (typeof dataDirectory !== 'undefined') {
|
|
13
|
-
dataDirectory = join(dataDirectory, name);
|
|
14
|
-
}
|
|
15
|
-
|
|
16
|
-
log.info(
|
|
17
|
-
dataDirectory
|
|
18
|
-
? `Creating ${name} data store at directory ${dataDirectory} with map size ${config.dataStoreMapSizeKB} KB`
|
|
19
|
-
: `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKB} KB`,
|
|
20
|
-
);
|
|
21
|
-
|
|
22
|
-
const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKB, false);
|
|
23
|
-
if (config.l1Contracts?.rollupAddress) {
|
|
24
|
-
return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
|
|
25
|
-
}
|
|
26
|
-
return store;
|
|
27
|
-
}
|
|
4
|
+
import { type AztecAsyncSingleton, type AztecSingleton } from './interfaces/singleton.js';
|
|
5
|
+
import { type AztecAsyncKVStore, type AztecKVStore } from './interfaces/store.js';
|
|
6
|
+
import { isSyncStore } from './interfaces/utils.js';
|
|
28
7
|
|
|
29
8
|
/**
|
|
30
9
|
* Clears the store if the rollup address does not match the one stored in the database.
|
|
@@ -33,7 +12,7 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger =
|
|
|
33
12
|
* @param rollupAddress - The ETH address of the rollup contract
|
|
34
13
|
* @returns A promise that resolves when the store is cleared, or rejects if the rollup address does not match
|
|
35
14
|
*/
|
|
36
|
-
async function initStoreForRollup<T extends AztecKVStore>(
|
|
15
|
+
export async function initStoreForRollup<T extends AztecKVStore | AztecAsyncKVStore>(
|
|
37
16
|
store: T,
|
|
38
17
|
rollupAddress: EthAddress,
|
|
39
18
|
log?: Logger,
|
|
@@ -43,7 +22,9 @@ async function initStoreForRollup<T extends AztecKVStore>(
|
|
|
43
22
|
}
|
|
44
23
|
const rollupAddressValue = store.openSingleton<ReturnType<EthAddress['toString']>>('rollupAddress');
|
|
45
24
|
const rollupAddressString = rollupAddress.toString();
|
|
46
|
-
const storedRollupAddressString =
|
|
25
|
+
const storedRollupAddressString = isSyncStore(store)
|
|
26
|
+
? (rollupAddressValue as AztecSingleton<ReturnType<EthAddress['toString']>>).get()
|
|
27
|
+
: await (rollupAddressValue as AztecAsyncSingleton<ReturnType<EthAddress['toString']>>).getAsync();
|
|
47
28
|
|
|
48
29
|
if (typeof storedRollupAddressString !== 'undefined' && storedRollupAddressString !== rollupAddressString) {
|
|
49
30
|
log?.warn(`Rollup address mismatch. Clearing entire database...`, {
|
|
@@ -57,13 +38,3 @@ async function initStoreForRollup<T extends AztecKVStore>(
|
|
|
57
38
|
await rollupAddressValue.set(rollupAddressString);
|
|
58
39
|
return store;
|
|
59
40
|
}
|
|
60
|
-
|
|
61
|
-
/**
|
|
62
|
-
* Opens a temporary store for testing purposes.
|
|
63
|
-
* @param ephemeral - true if the store should only exist in memory and not automatically be flushed to disk. Optional
|
|
64
|
-
* @returns A new store
|
|
65
|
-
*/
|
|
66
|
-
export function openTmpStore(ephemeral: boolean = false): AztecLmdbStore {
|
|
67
|
-
const mapSize = 1024 * 1024 * 10; // 10 GB map size
|
|
68
|
-
return AztecLmdbStore.open(undefined, mapSize, ephemeral);
|
|
69
|
-
}
|