@aztec/kv-store 0.0.1-commit.d1f2d6c → 0.0.1-commit.d20b825a7
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/bench/shared_map_bench.d.ts +19 -0
- package/dest/bench/shared_map_bench.d.ts.map +1 -0
- package/dest/bench/shared_map_bench.js +91 -0
- package/dest/indexeddb/array.js +18 -6
- package/dest/indexeddb/index.d.ts +3 -3
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/index.js +3 -6
- package/dest/indexeddb/map.d.ts +8 -2
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +23 -13
- package/dest/indexeddb/multi_map.d.ts +2 -1
- package/dest/indexeddb/multi_map.d.ts.map +1 -1
- package/dest/indexeddb/multi_map.js +16 -1
- package/dest/indexeddb/singleton.js +3 -1
- package/dest/indexeddb/store.d.ts +3 -3
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +6 -4
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +48 -2
- package/dest/interfaces/multi_map_test_suite.d.ts +1 -1
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/multi_map_test_suite.js +25 -0
- package/dest/interfaces/utils.d.ts +2 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +2 -1
- package/dest/lmdb/index.d.ts +3 -3
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +3 -3
- package/dest/lmdb/store.d.ts +3 -3
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +12 -8
- package/dest/lmdb-v2/factory.d.ts +7 -7
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +14 -10
- package/dest/lmdb-v2/read_transaction.js +21 -19
- package/dest/lmdb-v2/store.d.ts +3 -3
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +2 -1
- package/dest/sqlite-opfs/array.d.ts +21 -0
- package/dest/sqlite-opfs/array.d.ts.map +1 -0
- package/dest/sqlite-opfs/array.js +128 -0
- package/dest/sqlite-opfs/index.d.ts +7 -0
- package/dest/sqlite-opfs/index.d.ts.map +1 -0
- package/dest/sqlite-opfs/index.js +13 -0
- package/dest/sqlite-opfs/map.d.ts +35 -0
- package/dest/sqlite-opfs/map.d.ts.map +1 -0
- package/dest/sqlite-opfs/map.js +163 -0
- package/dest/sqlite-opfs/messages.d.ts +58 -0
- package/dest/sqlite-opfs/messages.d.ts.map +1 -0
- package/dest/sqlite-opfs/messages.js +5 -0
- package/dest/sqlite-opfs/multi_map.d.ts +16 -0
- package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
- package/dest/sqlite-opfs/multi_map.js +67 -0
- package/dest/sqlite-opfs/set.d.ts +13 -0
- package/dest/sqlite-opfs/set.d.ts.map +1 -0
- package/dest/sqlite-opfs/set.js +19 -0
- package/dest/sqlite-opfs/singleton.d.ts +13 -0
- package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
- package/dest/sqlite-opfs/singleton.js +48 -0
- package/dest/sqlite-opfs/store.d.ts +70 -0
- package/dest/sqlite-opfs/store.d.ts.map +1 -0
- package/dest/sqlite-opfs/store.js +242 -0
- package/dest/sqlite-opfs/worker.d.ts +2 -0
- package/dest/sqlite-opfs/worker.d.ts.map +1 -0
- package/dest/sqlite-opfs/worker.js +194 -0
- package/dest/utils.d.ts +9 -6
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +51 -16
- package/package.json +17 -13
- package/src/bench/shared_map_bench.ts +111 -0
- package/src/indexeddb/array.ts +4 -4
- package/src/indexeddb/index.ts +9 -7
- package/src/indexeddb/map.ts +24 -11
- package/src/indexeddb/multi_map.ts +15 -1
- package/src/indexeddb/singleton.ts +1 -1
- package/src/indexeddb/store.ts +13 -6
- package/src/interfaces/map_test_suite.ts +30 -2
- package/src/interfaces/multi_map_test_suite.ts +32 -0
- package/src/interfaces/utils.ts +1 -0
- package/src/lmdb/index.ts +9 -4
- package/src/lmdb/store.ts +12 -8
- package/src/lmdb-v2/factory.ts +16 -12
- package/src/lmdb-v2/read_transaction.ts +23 -23
- package/src/lmdb-v2/store.ts +3 -2
- package/src/sqlite-opfs/array.ts +124 -0
- package/src/sqlite-opfs/index.ts +27 -0
- package/src/sqlite-opfs/map.ts +163 -0
- package/src/sqlite-opfs/messages.ts +28 -0
- package/src/sqlite-opfs/multi_map.ts +74 -0
- package/src/sqlite-opfs/set.ts +29 -0
- package/src/sqlite-opfs/singleton.ts +48 -0
- package/src/sqlite-opfs/store.ts +248 -0
- package/src/sqlite-opfs/worker.ts +162 -0
- package/src/utils.ts +79 -21
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -26
- package/src/config.ts +0 -36
|
@@ -0,0 +1,111 @@
|
|
|
1
|
+
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { Timer } from '@aztec/foundation/timer';
|
|
3
|
+
|
|
4
|
+
import type { Key } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
6
|
+
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
7
|
+
|
|
8
|
+
/** One benchmark measurement. */
|
|
9
|
+
export type BenchResult = {
|
|
10
|
+
/** Benchmark name (includes the backend prefix for disambiguation). */
|
|
11
|
+
name: string;
|
|
12
|
+
value: number;
|
|
13
|
+
unit: 'ms' | 'us';
|
|
14
|
+
};
|
|
15
|
+
|
|
16
|
+
export type BenchReporter = (results: BenchResult[]) => void | Promise<void>;
|
|
17
|
+
|
|
18
|
+
/**
|
|
19
|
+
* Runs the standard Map benchmark suite against any `AztecAsyncKVStore` backend,
|
|
20
|
+
* populates `results`, and calls `reporter` in `afterAll`.
|
|
21
|
+
*
|
|
22
|
+
* Kept free of Node-only deps (`fs`, `path`) so the same runner works under
|
|
23
|
+
* vitest-browser for IndexedDB and SQLite-OPFS.
|
|
24
|
+
*/
|
|
25
|
+
export function describeAztecMapBench(
|
|
26
|
+
backendPrefix: string,
|
|
27
|
+
getStore: () => Promise<AztecAsyncKVStore>,
|
|
28
|
+
logger: Logger,
|
|
29
|
+
reporter: BenchReporter,
|
|
30
|
+
) {
|
|
31
|
+
describe(`${backendPrefix} Map benchmarks`, () => {
|
|
32
|
+
let store: AztecAsyncKVStore;
|
|
33
|
+
let map: AztecAsyncMap<Key, string>;
|
|
34
|
+
|
|
35
|
+
const results: BenchResult[] = [];
|
|
36
|
+
|
|
37
|
+
const generateKeyValuePairs = (count: number, offset = 0) => {
|
|
38
|
+
const keys = Array.from({ length: count }, (_, i) => `key-${i + offset}`);
|
|
39
|
+
const values = Array.from({ length: count }, (_, i) => `value-${i + offset}`);
|
|
40
|
+
return keys.map((key, i) => ({ key, value: values[i] }));
|
|
41
|
+
};
|
|
42
|
+
|
|
43
|
+
const record = (name: string, value: number, unit: BenchResult['unit']) => {
|
|
44
|
+
results.push({ name: `${backendPrefix}/Map/${name}`, value, unit });
|
|
45
|
+
};
|
|
46
|
+
|
|
47
|
+
beforeEach(async () => {
|
|
48
|
+
store = await getStore();
|
|
49
|
+
map = store.openMap<Key, string>('test');
|
|
50
|
+
});
|
|
51
|
+
|
|
52
|
+
afterEach(async () => {
|
|
53
|
+
await store.delete();
|
|
54
|
+
});
|
|
55
|
+
|
|
56
|
+
afterAll(async () => {
|
|
57
|
+
const pretty = results.map(r => `${r.name}: ${r.value.toFixed(2)} ${r.unit}`).join('\n');
|
|
58
|
+
logger.info(`\n${pretty}\n`);
|
|
59
|
+
await reporter(results);
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
it('adds individual values', async () => {
|
|
63
|
+
const pairs = generateKeyValuePairs(1000);
|
|
64
|
+
const timer = new Timer();
|
|
65
|
+
for (const pair of pairs) {
|
|
66
|
+
await map.set(pair.key, pair.value);
|
|
67
|
+
}
|
|
68
|
+
record('Individual insertion', timer.ms() / pairs.length, 'ms');
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
it('adds batched values', async () => {
|
|
72
|
+
const batches = Array.from({ length: 100 }, (_, i) => generateKeyValuePairs(1000, i * 1000));
|
|
73
|
+
const timer = new Timer();
|
|
74
|
+
for (const batch of batches) {
|
|
75
|
+
await map.setMany(batch);
|
|
76
|
+
}
|
|
77
|
+
record(`Batch insertion of ${batches[0].length} items`, timer.ms() / batches.length, 'ms');
|
|
78
|
+
});
|
|
79
|
+
|
|
80
|
+
it('reads individual values', async () => {
|
|
81
|
+
const pairs = generateKeyValuePairs(10000);
|
|
82
|
+
await map.setMany(pairs);
|
|
83
|
+
const timer = new Timer();
|
|
84
|
+
for (const pair of pairs) {
|
|
85
|
+
await map.getAsync(pair.key);
|
|
86
|
+
}
|
|
87
|
+
record('Individual read', (timer.ms() * 1000) / pairs.length, 'us');
|
|
88
|
+
});
|
|
89
|
+
|
|
90
|
+
it('reads via a cursor', async () => {
|
|
91
|
+
const pairs = generateKeyValuePairs(10000);
|
|
92
|
+
await map.setMany(pairs);
|
|
93
|
+
const timer = new Timer();
|
|
94
|
+
for await (const _ of map.entriesAsync()) {
|
|
95
|
+
// consume
|
|
96
|
+
}
|
|
97
|
+
record(`Iterator per item read of ${pairs.length} items`, (timer.ms() * 1000) / pairs.length, 'us');
|
|
98
|
+
});
|
|
99
|
+
|
|
100
|
+
it('reads the size of the map', async () => {
|
|
101
|
+
const numIterations = 1000;
|
|
102
|
+
const pairs = generateKeyValuePairs(10000);
|
|
103
|
+
await map.setMany(pairs);
|
|
104
|
+
const timer = new Timer();
|
|
105
|
+
for (let i = 0; i < numIterations; i++) {
|
|
106
|
+
await map.sizeAsync();
|
|
107
|
+
}
|
|
108
|
+
record(`Read size of ${pairs.length} items`, (timer.ms() * 1000) / numIterations, 'us');
|
|
109
|
+
});
|
|
110
|
+
});
|
|
111
|
+
}
|
package/src/indexeddb/array.ts
CHANGED
|
@@ -32,7 +32,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
32
32
|
return (
|
|
33
33
|
(await this.db
|
|
34
34
|
.index('key')
|
|
35
|
-
.count(IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]))) ?? 0
|
|
35
|
+
.count(IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]))) ?? 0
|
|
36
36
|
);
|
|
37
37
|
}
|
|
38
38
|
|
|
@@ -43,7 +43,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
43
43
|
value: val,
|
|
44
44
|
hash: hash(val),
|
|
45
45
|
container: this.#container,
|
|
46
|
-
key: this.#name,
|
|
46
|
+
key: [this.#name],
|
|
47
47
|
keyCount: length + 1,
|
|
48
48
|
slot: this.#slot(length),
|
|
49
49
|
});
|
|
@@ -91,7 +91,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
91
91
|
value: val,
|
|
92
92
|
hash: hash(val),
|
|
93
93
|
container: this.#container,
|
|
94
|
-
key: this.#name,
|
|
94
|
+
key: [this.#name],
|
|
95
95
|
keyCount: index + 1,
|
|
96
96
|
slot: this.#slot(index),
|
|
97
97
|
});
|
|
@@ -100,7 +100,7 @@ export class IndexedDBAztecArray<T extends Value> implements AztecAsyncArray<T>
|
|
|
100
100
|
|
|
101
101
|
async *entriesAsync(): AsyncIterableIterator<[number, T]> {
|
|
102
102
|
const index = this.db.index('key');
|
|
103
|
-
const rangeQuery = IDBKeyRange.bound([this.#container, this.#name], [this.#container, this.#name]);
|
|
103
|
+
const rangeQuery = IDBKeyRange.bound([this.#container, [this.#name]], [this.#container, [this.#name]]);
|
|
104
104
|
for await (const cursor of index.iterate(rangeQuery)) {
|
|
105
105
|
yield [cursor.value.keyCount - 1, cursor.value.value] as [number, T];
|
|
106
106
|
}
|
package/src/indexeddb/index.ts
CHANGED
|
@@ -1,12 +1,17 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
2
3
|
|
|
3
|
-
import
|
|
4
|
-
import { initStoreForRollup } from '../utils.js';
|
|
4
|
+
import { initStoreForRollupAndSchemaVersion } from '../utils.js';
|
|
5
5
|
import { AztecIndexedDBStore } from './store.js';
|
|
6
6
|
|
|
7
7
|
export { AztecIndexedDBStore } from './store.js';
|
|
8
8
|
|
|
9
|
-
export async function createStore(
|
|
9
|
+
export async function createStore(
|
|
10
|
+
name: string,
|
|
11
|
+
config: DataStoreConfig,
|
|
12
|
+
schemaVersion: number | undefined = undefined,
|
|
13
|
+
log: Logger = createLogger('kv-store'),
|
|
14
|
+
) {
|
|
10
15
|
let { dataDirectory } = config;
|
|
11
16
|
if (typeof dataDirectory !== 'undefined') {
|
|
12
17
|
dataDirectory = `${dataDirectory}/${name}`;
|
|
@@ -18,10 +23,7 @@ export async function createStore(name: string, config: DataStoreConfig, log: Lo
|
|
|
18
23
|
: `Creating ${name} ephemeral data store with map size ${config.dataStoreMapSizeKb} KB`,
|
|
19
24
|
);
|
|
20
25
|
const store = await AztecIndexedDBStore.open(createLogger('kv-store:indexeddb'), dataDirectory ?? '', false);
|
|
21
|
-
|
|
22
|
-
return initStoreForRollup(store, config.l1Contracts.rollupAddress, log);
|
|
23
|
-
}
|
|
24
|
-
return store;
|
|
26
|
+
return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts?.rollupAddress, log);
|
|
25
27
|
}
|
|
26
28
|
|
|
27
29
|
export function openTmpStore(ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
|
package/src/indexeddb/map.ts
CHANGED
|
@@ -31,7 +31,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
31
31
|
|
|
32
32
|
async getAsync(key: K): Promise<V | undefined> {
|
|
33
33
|
const data = await this.db.get(this.slot(key));
|
|
34
|
-
return data
|
|
34
|
+
return data ? this.restoreBuffers(data.value as V) : undefined;
|
|
35
35
|
}
|
|
36
36
|
|
|
37
37
|
async hasAsync(key: K): Promise<boolean> {
|
|
@@ -41,7 +41,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
41
41
|
|
|
42
42
|
async sizeAsync(): Promise<number> {
|
|
43
43
|
const index = this.db.index('key');
|
|
44
|
-
const rangeQuery = IDBKeyRange.bound([this.container
|
|
44
|
+
const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
|
|
45
45
|
return await index.count(rangeQuery);
|
|
46
46
|
}
|
|
47
47
|
|
|
@@ -80,9 +80,12 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
80
80
|
|
|
81
81
|
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
|
|
82
82
|
const index = this.db.index('key');
|
|
83
|
+
const startKey = range.start ? this.normalizeKey(range.start) : [];
|
|
84
|
+
const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
|
|
85
|
+
|
|
83
86
|
const rangeQuery = IDBKeyRange.bound(
|
|
84
|
-
[this.container,
|
|
85
|
-
[this.container,
|
|
87
|
+
[this.container, startKey],
|
|
88
|
+
[this.container, endKey],
|
|
86
89
|
!!range.reverse,
|
|
87
90
|
!range.reverse,
|
|
88
91
|
);
|
|
@@ -91,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
91
94
|
if (range.limit && count >= range.limit) {
|
|
92
95
|
return;
|
|
93
96
|
}
|
|
94
|
-
yield [this.#denormalizeKey(cursor.value.key), cursor.value.value] as [K, V];
|
|
97
|
+
yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
|
|
95
98
|
count++;
|
|
96
99
|
}
|
|
97
100
|
}
|
|
@@ -108,14 +111,24 @@ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecA
|
|
|
108
111
|
}
|
|
109
112
|
}
|
|
110
113
|
|
|
111
|
-
|
|
112
|
-
|
|
113
|
-
|
|
114
|
+
/**
|
|
115
|
+
* IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
|
|
116
|
+
* This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
|
|
117
|
+
* comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
|
|
118
|
+
*/
|
|
119
|
+
protected restoreBuffers(val: V): V {
|
|
120
|
+
if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
|
|
121
|
+
return Buffer.from(val) as V;
|
|
122
|
+
}
|
|
123
|
+
return val;
|
|
124
|
+
}
|
|
125
|
+
|
|
126
|
+
#denormalizeKey(key: (string | number | Uint8Array)[]): K {
|
|
127
|
+
return (key.length > 1 ? key : key[0]) as K;
|
|
114
128
|
}
|
|
115
129
|
|
|
116
|
-
protected normalizeKey(key: K): string {
|
|
117
|
-
|
|
118
|
-
return (arrayKey as K[]).map((element: K) => (typeof element === 'number' ? `n_${element}` : element)).join(',');
|
|
130
|
+
protected normalizeKey(key: K): (string | number | Uint8Array)[] {
|
|
131
|
+
return Array.isArray(key) ? key : [key];
|
|
119
132
|
}
|
|
120
133
|
|
|
121
134
|
protected slot(key: K, index: number = 0): string {
|
|
@@ -57,7 +57,7 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
|
57
57
|
false,
|
|
58
58
|
);
|
|
59
59
|
for await (const cursor of index.iterate(rangeQuery)) {
|
|
60
|
-
yield cursor.value.value as V;
|
|
60
|
+
yield this.restoreBuffers(cursor.value.value as V);
|
|
61
61
|
}
|
|
62
62
|
}
|
|
63
63
|
|
|
@@ -88,4 +88,18 @@ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
|
88
88
|
await this.db.delete(fullKey);
|
|
89
89
|
}
|
|
90
90
|
}
|
|
91
|
+
|
|
92
|
+
override async delete(key: K): Promise<void> {
|
|
93
|
+
const index = this.db.index('keyCount');
|
|
94
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
95
|
+
[this.container, this.normalizeKey(key), 0],
|
|
96
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
97
|
+
false,
|
|
98
|
+
false,
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
for await (const cursor of index.iterate(rangeQuery)) {
|
|
102
|
+
await cursor.delete();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
91
105
|
}
|
|
@@ -37,7 +37,7 @@ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingl
|
|
|
37
37
|
const result = await this.db.put({
|
|
38
38
|
container: this.#container,
|
|
39
39
|
slot: this.#slot,
|
|
40
|
-
key: this.#slot,
|
|
40
|
+
key: [this.#slot],
|
|
41
41
|
keyCount: 1,
|
|
42
42
|
value: val,
|
|
43
43
|
hash: hash(val),
|
package/src/indexeddb/store.ts
CHANGED
|
@@ -20,7 +20,7 @@ import { IndexedDBAztecSingleton } from './singleton.js';
|
|
|
20
20
|
export type StoredData<V extends Value> = {
|
|
21
21
|
value: V;
|
|
22
22
|
container: string;
|
|
23
|
-
key: string;
|
|
23
|
+
key: (string | number | Uint8Array)[];
|
|
24
24
|
keyCount: number;
|
|
25
25
|
slot: string;
|
|
26
26
|
hash: string;
|
|
@@ -30,7 +30,12 @@ export interface AztecIDBSchema extends DBSchema {
|
|
|
30
30
|
data: {
|
|
31
31
|
value: StoredData<any>;
|
|
32
32
|
key: string;
|
|
33
|
-
indexes: {
|
|
33
|
+
indexes: {
|
|
34
|
+
container: string;
|
|
35
|
+
key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
|
|
36
|
+
keyCount: number;
|
|
37
|
+
hash: string;
|
|
38
|
+
};
|
|
34
39
|
};
|
|
35
40
|
}
|
|
36
41
|
|
|
@@ -187,18 +192,20 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
187
192
|
}
|
|
188
193
|
|
|
189
194
|
/** Deletes this store and removes the database */
|
|
190
|
-
delete() {
|
|
195
|
+
async delete() {
|
|
191
196
|
this.#containers.clear();
|
|
197
|
+
await this.#txQueue.end();
|
|
192
198
|
this.#rootDB.close();
|
|
193
|
-
|
|
199
|
+
await deleteDB(this.#name);
|
|
194
200
|
}
|
|
195
201
|
|
|
196
202
|
estimateSize(): Promise<StoreSize> {
|
|
197
203
|
return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
|
|
198
204
|
}
|
|
199
205
|
|
|
200
|
-
close(): Promise<void> {
|
|
201
|
-
|
|
206
|
+
async close(): Promise<void> {
|
|
207
|
+
await this.#txQueue.end();
|
|
208
|
+
this.#rootDB.close();
|
|
202
209
|
}
|
|
203
210
|
|
|
204
211
|
backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
|
|
@@ -107,6 +107,33 @@ export function describeAztecMap(
|
|
|
107
107
|
expect(await size()).to.equal(1);
|
|
108
108
|
});
|
|
109
109
|
|
|
110
|
+
it('returns 0 for empty map size', async () => {
|
|
111
|
+
expect(await size()).to.equal(0);
|
|
112
|
+
});
|
|
113
|
+
|
|
114
|
+
it('calculates size correctly across multiple operations', async () => {
|
|
115
|
+
expect(await size()).to.equal(0);
|
|
116
|
+
|
|
117
|
+
// Add items
|
|
118
|
+
await map.set('a', 'value1');
|
|
119
|
+
await map.set('b', 'value2');
|
|
120
|
+
await map.set('c', 'value3');
|
|
121
|
+
expect(await size()).to.equal(3);
|
|
122
|
+
|
|
123
|
+
// Update existing (size should not change)
|
|
124
|
+
await map.set('b', 'updated');
|
|
125
|
+
expect(await size()).to.equal(3);
|
|
126
|
+
|
|
127
|
+
// Delete some
|
|
128
|
+
await map.delete('a');
|
|
129
|
+
expect(await size()).to.equal(2);
|
|
130
|
+
|
|
131
|
+
// Delete all
|
|
132
|
+
await map.delete('b');
|
|
133
|
+
await map.delete('c');
|
|
134
|
+
expect(await size()).to.equal(0);
|
|
135
|
+
});
|
|
136
|
+
|
|
110
137
|
it('should be able to iterate over entries when there are no keys', async () => {
|
|
111
138
|
expect(await entries()).to.deep.equal([]);
|
|
112
139
|
});
|
|
@@ -145,10 +172,11 @@ export function describeAztecMap(
|
|
|
145
172
|
for (const [name, data] of [
|
|
146
173
|
['chars', ['a', 'b', 'c', 'd']],
|
|
147
174
|
['numbers', [1, 2, 3, 4]],
|
|
148
|
-
|
|
149
|
-
// ['negative numbers', [-4, -3, -2, -1]],
|
|
175
|
+
['negative numbers', [-4, -3, -2, -1]],
|
|
150
176
|
['strings', ['aaa', 'bbb', 'ccc', 'ddd']],
|
|
151
177
|
['zero-based numbers', [0, 1, 2, 3]],
|
|
178
|
+
['large numbers', [100, 999, 1000, 1001]],
|
|
179
|
+
['mixed negative and positive', [-1000, -1, 1, 1000]],
|
|
152
180
|
]) {
|
|
153
181
|
it(`supports range queries over ${name} keys`, async () => {
|
|
154
182
|
const [a, b, c, d] = data;
|
|
@@ -104,6 +104,38 @@ export function describeAztecMultiMap(
|
|
|
104
104
|
expect(await size()).to.equal(1);
|
|
105
105
|
});
|
|
106
106
|
|
|
107
|
+
it('returns 0 for empty multimap size', async () => {
|
|
108
|
+
expect(await size()).to.equal(0);
|
|
109
|
+
});
|
|
110
|
+
|
|
111
|
+
it('calculates size correctly with multiple values per key', async () => {
|
|
112
|
+
expect(await size()).to.equal(0);
|
|
113
|
+
|
|
114
|
+
// Add multiple values for same key
|
|
115
|
+
await multiMap.set('key1', 'value1');
|
|
116
|
+
expect(await size()).to.equal(1);
|
|
117
|
+
await multiMap.set('key1', 'value2');
|
|
118
|
+
expect(await size()).to.equal(2);
|
|
119
|
+
await multiMap.set('key1', 'value3');
|
|
120
|
+
expect(await size()).to.equal(3);
|
|
121
|
+
|
|
122
|
+
// Add values for different key
|
|
123
|
+
await multiMap.set('key2', 'value4');
|
|
124
|
+
expect(await size()).to.equal(4);
|
|
125
|
+
|
|
126
|
+
// Delete one value from key1
|
|
127
|
+
await multiMap.deleteValue('key1', 'value2');
|
|
128
|
+
expect(await size()).to.equal(3);
|
|
129
|
+
|
|
130
|
+
// Delete entire key
|
|
131
|
+
await multiMap.delete('key1');
|
|
132
|
+
expect(await size()).to.equal(1);
|
|
133
|
+
|
|
134
|
+
// Delete last key
|
|
135
|
+
await multiMap.delete('key2');
|
|
136
|
+
expect(await size()).to.equal(0);
|
|
137
|
+
});
|
|
138
|
+
|
|
107
139
|
it('should be able to iterate over entries when there are no keys', async () => {
|
|
108
140
|
expect(await entries()).to.deep.equal([]);
|
|
109
141
|
});
|
package/src/interfaces/utils.ts
CHANGED
package/src/lmdb/index.ts
CHANGED
|
@@ -1,14 +1,19 @@
|
|
|
1
1
|
import { type Logger, createLogger } from '@aztec/foundation/log';
|
|
2
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
2
3
|
|
|
3
4
|
import { join } from 'path';
|
|
4
5
|
|
|
5
|
-
import
|
|
6
|
-
import { initStoreForRollup } from '../utils.js';
|
|
6
|
+
import { initStoreForRollupAndSchemaVersion } from '../utils.js';
|
|
7
7
|
import { AztecLmdbStore } from './store.js';
|
|
8
8
|
|
|
9
9
|
export { AztecLmdbStore } from './store.js';
|
|
10
10
|
|
|
11
|
-
export function createStore(
|
|
11
|
+
export function createStore(
|
|
12
|
+
name: string,
|
|
13
|
+
config: DataStoreConfig,
|
|
14
|
+
schemaVersion: number | undefined = undefined,
|
|
15
|
+
log: Logger = createLogger('kv-store'),
|
|
16
|
+
) {
|
|
12
17
|
let { dataDirectory } = config;
|
|
13
18
|
if (typeof dataDirectory !== 'undefined') {
|
|
14
19
|
dataDirectory = join(dataDirectory, name);
|
|
@@ -22,7 +27,7 @@ export function createStore(name: string, config: DataStoreConfig, log: Logger =
|
|
|
22
27
|
|
|
23
28
|
const store = AztecLmdbStore.open(dataDirectory, config.dataStoreMapSizeKb, false);
|
|
24
29
|
if (config.l1Contracts?.rollupAddress) {
|
|
25
|
-
return
|
|
30
|
+
return initStoreForRollupAndSchemaVersion(store, schemaVersion, config.l1Contracts.rollupAddress, log);
|
|
26
31
|
}
|
|
27
32
|
return store;
|
|
28
33
|
}
|
package/src/lmdb/store.ts
CHANGED
|
@@ -147,21 +147,25 @@ export class AztecLmdbStore implements AztecKVStore, AztecAsyncKVStore {
|
|
|
147
147
|
}
|
|
148
148
|
|
|
149
149
|
/**
|
|
150
|
-
* Clears all entries in the store & sub DBs.
|
|
150
|
+
* Clears all entries in the store & sub DBs atomically within a single transaction.
|
|
151
151
|
*/
|
|
152
152
|
async clear() {
|
|
153
|
-
await this.#
|
|
154
|
-
|
|
155
|
-
|
|
153
|
+
await this.#rootDb.transaction(async () => {
|
|
154
|
+
await this.#data.clearAsync();
|
|
155
|
+
await this.#multiMapData.clearAsync();
|
|
156
|
+
await this.#rootDb.clearAsync();
|
|
157
|
+
});
|
|
156
158
|
}
|
|
157
159
|
|
|
158
160
|
/**
|
|
159
|
-
* Drops the database & sub DBs.
|
|
161
|
+
* Drops the database & sub DBs atomically within a single transaction.
|
|
160
162
|
*/
|
|
161
163
|
async drop() {
|
|
162
|
-
await this.#
|
|
163
|
-
|
|
164
|
-
|
|
164
|
+
await this.#rootDb.transaction(async () => {
|
|
165
|
+
await this.#data.drop();
|
|
166
|
+
await this.#multiMapData.drop();
|
|
167
|
+
await this.#rootDb.drop();
|
|
168
|
+
});
|
|
165
169
|
}
|
|
166
170
|
|
|
167
171
|
/**
|
package/src/lmdb-v2/factory.ts
CHANGED
|
@@ -1,12 +1,12 @@
|
|
|
1
1
|
import { EthAddress } from '@aztec/foundation/eth-address';
|
|
2
|
-
import { type
|
|
3
|
-
import { DatabaseVersionManager } from '@aztec/stdlib/database-version';
|
|
2
|
+
import { type LoggerBindings, createLogger } from '@aztec/foundation/log';
|
|
3
|
+
import { DatabaseVersionManager } from '@aztec/stdlib/database-version/manager';
|
|
4
|
+
import type { DataStoreConfig } from '@aztec/stdlib/kv-store';
|
|
4
5
|
|
|
5
6
|
import { mkdir, mkdtemp, rm } from 'fs/promises';
|
|
6
7
|
import { tmpdir } from 'os';
|
|
7
8
|
import { join } from 'path';
|
|
8
9
|
|
|
9
|
-
import type { DataStoreConfig } from '../config.js';
|
|
10
10
|
import { AztecLMDBStoreV2 } from './store.js';
|
|
11
11
|
|
|
12
12
|
const MAX_READERS = 16;
|
|
@@ -15,8 +15,9 @@ export async function createStore(
|
|
|
15
15
|
name: string,
|
|
16
16
|
schemaVersion: number,
|
|
17
17
|
config: DataStoreConfig,
|
|
18
|
-
|
|
18
|
+
bindings?: LoggerBindings,
|
|
19
19
|
): Promise<AztecLMDBStoreV2> {
|
|
20
|
+
const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
|
|
20
21
|
const { dataDirectory, l1Contracts } = config;
|
|
21
22
|
|
|
22
23
|
let store: AztecLMDBStoreV2;
|
|
@@ -33,7 +34,7 @@ export async function createStore(
|
|
|
33
34
|
rollupAddress,
|
|
34
35
|
dataDirectory: subDir,
|
|
35
36
|
onOpen: dbDirectory =>
|
|
36
|
-
AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(),
|
|
37
|
+
AztecLMDBStoreV2.new(dbDirectory, config.dataStoreMapSizeKb, MAX_READERS, () => Promise.resolve(), bindings),
|
|
37
38
|
});
|
|
38
39
|
|
|
39
40
|
log.info(
|
|
@@ -41,7 +42,7 @@ export async function createStore(
|
|
|
41
42
|
);
|
|
42
43
|
[store] = await versionManager.open();
|
|
43
44
|
} else {
|
|
44
|
-
store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS,
|
|
45
|
+
store = await openTmpStore(name, true, config.dataStoreMapSizeKb, MAX_READERS, bindings);
|
|
45
46
|
}
|
|
46
47
|
|
|
47
48
|
return store;
|
|
@@ -52,8 +53,9 @@ export async function openTmpStore(
|
|
|
52
53
|
ephemeral: boolean = true,
|
|
53
54
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
54
55
|
maxReaders = MAX_READERS,
|
|
55
|
-
|
|
56
|
+
bindings?: LoggerBindings,
|
|
56
57
|
): Promise<AztecLMDBStoreV2> {
|
|
58
|
+
const log = createLogger('kv-store:lmdb-v2:' + name, bindings);
|
|
57
59
|
const dataDir = await mkdtemp(join(tmpdir(), name + '-'));
|
|
58
60
|
log.debug(`Created temporary data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
59
61
|
|
|
@@ -73,17 +75,18 @@ export async function openTmpStore(
|
|
|
73
75
|
|
|
74
76
|
// For temporary stores, we don't need to worry about versioning
|
|
75
77
|
// as they are ephemeral and get cleaned up after use
|
|
76
|
-
return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup,
|
|
78
|
+
return AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, cleanup, bindings);
|
|
77
79
|
}
|
|
78
80
|
|
|
79
81
|
export async function openStoreAt(
|
|
80
82
|
dataDir: string,
|
|
81
83
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
82
84
|
maxReaders = MAX_READERS,
|
|
83
|
-
|
|
85
|
+
bindings?: LoggerBindings,
|
|
84
86
|
): Promise<AztecLMDBStoreV2> {
|
|
87
|
+
const log = createLogger('kv-store:lmdb-v2', bindings);
|
|
85
88
|
log.debug(`Opening data store at: ${dataDir} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
86
|
-
return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined,
|
|
89
|
+
return await AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings);
|
|
87
90
|
}
|
|
88
91
|
|
|
89
92
|
export async function openVersionedStoreAt(
|
|
@@ -92,14 +95,15 @@ export async function openVersionedStoreAt(
|
|
|
92
95
|
rollupAddress: EthAddress,
|
|
93
96
|
dbMapSizeKb = 10 * 1_024 * 1_024, // 10GB
|
|
94
97
|
maxReaders = MAX_READERS,
|
|
95
|
-
|
|
98
|
+
bindings?: LoggerBindings,
|
|
96
99
|
): Promise<AztecLMDBStoreV2> {
|
|
100
|
+
const log = createLogger('kv-store:lmdb-v2', bindings);
|
|
97
101
|
log.debug(`Opening data store at: ${dataDirectory} with size: ${dbMapSizeKb} KB (LMDB v2)`);
|
|
98
102
|
const [store] = await new DatabaseVersionManager({
|
|
99
103
|
schemaVersion,
|
|
100
104
|
rollupAddress,
|
|
101
105
|
dataDirectory,
|
|
102
|
-
onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined,
|
|
106
|
+
onOpen: dataDir => AztecLMDBStoreV2.new(dataDir, dbMapSizeKb, maxReaders, undefined, bindings),
|
|
103
107
|
}).open();
|
|
104
108
|
return store;
|
|
105
109
|
}
|
|
@@ -66,20 +66,20 @@ export class ReadTransaction {
|
|
|
66
66
|
): AsyncIterable<[Uint8Array, T]> {
|
|
67
67
|
this.assertIsOpen();
|
|
68
68
|
|
|
69
|
-
|
|
70
|
-
key: startKey,
|
|
71
|
-
reverse,
|
|
72
|
-
count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
|
|
73
|
-
onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
|
|
74
|
-
db,
|
|
75
|
-
});
|
|
76
|
-
|
|
77
|
-
const cursor = response.cursor;
|
|
78
|
-
let entries = response.entries;
|
|
79
|
-
let done = typeof cursor !== 'number';
|
|
80
|
-
let count = 0;
|
|
81
|
-
|
|
69
|
+
let cursor: number | undefined;
|
|
82
70
|
try {
|
|
71
|
+
const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
|
|
72
|
+
key: startKey,
|
|
73
|
+
reverse,
|
|
74
|
+
count: typeof limit === 'number' ? Math.min(limit, CURSOR_PAGE_SIZE) : CURSOR_PAGE_SIZE,
|
|
75
|
+
onePage: typeof limit === 'number' && limit < CURSOR_PAGE_SIZE,
|
|
76
|
+
db,
|
|
77
|
+
});
|
|
78
|
+
|
|
79
|
+
cursor = response.cursor ?? undefined;
|
|
80
|
+
let entries = response.entries;
|
|
81
|
+
let done = typeof cursor !== 'number';
|
|
82
|
+
let count = 0;
|
|
83
83
|
// emit the first page and any subsequent pages in a while loop
|
|
84
84
|
// NB: end contition is in the middle of the while loop
|
|
85
85
|
while (entries.length > 0) {
|
|
@@ -125,17 +125,17 @@ export class ReadTransaction {
|
|
|
125
125
|
async #countEntries(db: string, startKey: Uint8Array, endKey: Uint8Array, reverse: boolean): Promise<number> {
|
|
126
126
|
this.assertIsOpen();
|
|
127
127
|
|
|
128
|
-
|
|
129
|
-
key: startKey,
|
|
130
|
-
reverse,
|
|
131
|
-
count: 0,
|
|
132
|
-
onePage: false,
|
|
133
|
-
db,
|
|
134
|
-
});
|
|
135
|
-
|
|
136
|
-
const cursor = response.cursor;
|
|
137
|
-
|
|
128
|
+
let cursor: number | undefined;
|
|
138
129
|
try {
|
|
130
|
+
const response = await this.channel.sendMessage(LMDBMessageType.START_CURSOR, {
|
|
131
|
+
key: startKey,
|
|
132
|
+
reverse,
|
|
133
|
+
count: 0,
|
|
134
|
+
onePage: false,
|
|
135
|
+
db,
|
|
136
|
+
});
|
|
137
|
+
|
|
138
|
+
cursor = response.cursor ?? undefined;
|
|
139
139
|
if (!cursor) {
|
|
140
140
|
return 0;
|
|
141
141
|
}
|