@aztec/kv-store 0.0.0-test.1 → 0.0.1-commit.001888fc
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/indexeddb/array.d.ts +3 -2
- package/dest/indexeddb/array.d.ts.map +1 -1
- package/dest/indexeddb/array.js +24 -7
- package/dest/indexeddb/index.d.ts +3 -3
- package/dest/indexeddb/index.d.ts.map +1 -1
- package/dest/indexeddb/index.js +4 -7
- package/dest/indexeddb/map.d.ts +18 -6
- package/dest/indexeddb/map.d.ts.map +1 -1
- package/dest/indexeddb/map.js +51 -63
- package/dest/indexeddb/multi_map.d.ts +14 -0
- package/dest/indexeddb/multi_map.d.ts.map +1 -0
- package/dest/indexeddb/multi_map.js +107 -0
- package/dest/indexeddb/set.d.ts +1 -1
- package/dest/indexeddb/set.d.ts.map +1 -1
- package/dest/indexeddb/singleton.d.ts +3 -2
- package/dest/indexeddb/singleton.d.ts.map +1 -1
- package/dest/indexeddb/singleton.js +6 -2
- package/dest/indexeddb/store.d.ts +15 -16
- package/dest/indexeddb/store.d.ts.map +1 -1
- package/dest/indexeddb/store.js +51 -46
- package/dest/interfaces/array.d.ts +5 -4
- package/dest/interfaces/array.d.ts.map +1 -1
- package/dest/interfaces/array.js +1 -3
- package/dest/interfaces/array_test_suite.d.ts +1 -1
- package/dest/interfaces/common.d.ts +11 -9
- package/dest/interfaces/common.d.ts.map +1 -1
- package/dest/interfaces/common.js +8 -3
- package/dest/interfaces/counter.d.ts +1 -1
- package/dest/interfaces/index.d.ts +4 -2
- package/dest/interfaces/index.d.ts.map +1 -1
- package/dest/interfaces/index.js +2 -0
- package/dest/interfaces/map.d.ts +21 -49
- package/dest/interfaces/map.d.ts.map +1 -1
- package/dest/interfaces/map.js +1 -1
- package/dest/interfaces/map_test_suite.d.ts +1 -1
- package/dest/interfaces/map_test_suite.d.ts.map +1 -1
- package/dest/interfaces/map_test_suite.js +181 -70
- package/dest/interfaces/multi_map.d.ts +41 -0
- package/dest/interfaces/multi_map.d.ts.map +1 -0
- package/dest/interfaces/multi_map.js +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
- package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
- package/dest/interfaces/multi_map_test_suite.js +364 -0
- package/dest/interfaces/set.d.ts +1 -1
- package/dest/interfaces/set_test_suite.d.ts +1 -1
- package/dest/interfaces/singleton.d.ts +1 -1
- package/dest/interfaces/singleton_test_suite.d.ts +1 -1
- package/dest/interfaces/store.d.ts +18 -43
- package/dest/interfaces/store.d.ts.map +1 -1
- package/dest/interfaces/utils.d.ts +3 -1
- package/dest/interfaces/utils.d.ts.map +1 -1
- package/dest/interfaces/utils.js +3 -1
- package/dest/lmdb/array.d.ts +3 -2
- package/dest/lmdb/array.d.ts.map +1 -1
- package/dest/lmdb/array.js +4 -2
- package/dest/lmdb/counter.d.ts +1 -1
- package/dest/lmdb/counter.d.ts.map +1 -1
- package/dest/lmdb/index.d.ts +3 -3
- package/dest/lmdb/index.d.ts.map +1 -1
- package/dest/lmdb/index.js +5 -5
- package/dest/lmdb/map.d.ts +11 -23
- package/dest/lmdb/map.d.ts.map +1 -1
- package/dest/lmdb/map.js +15 -81
- package/dest/lmdb/multi_map.d.ts +13 -0
- package/dest/lmdb/multi_map.d.ts.map +1 -0
- package/dest/lmdb/multi_map.js +44 -0
- package/dest/lmdb/set.d.ts +1 -1
- package/dest/lmdb/set.d.ts.map +1 -1
- package/dest/lmdb/singleton.d.ts +1 -1
- package/dest/lmdb/singleton.d.ts.map +1 -1
- package/dest/lmdb/store.d.ts +11 -27
- package/dest/lmdb/store.d.ts.map +1 -1
- package/dest/lmdb/store.js +24 -40
- package/dest/lmdb-v2/array.d.ts +4 -3
- package/dest/lmdb-v2/array.d.ts.map +1 -1
- package/dest/lmdb-v2/array.js +4 -2
- package/dest/lmdb-v2/factory.d.ts +7 -10
- package/dest/lmdb-v2/factory.d.ts.map +1 -1
- package/dest/lmdb-v2/factory.js +26 -12
- package/dest/lmdb-v2/index.d.ts +1 -1
- package/dest/lmdb-v2/map.d.ts +12 -45
- package/dest/lmdb-v2/map.d.ts.map +1 -1
- package/dest/lmdb-v2/map.js +17 -104
- package/dest/lmdb-v2/message.d.ts +24 -5
- package/dest/lmdb-v2/message.d.ts.map +1 -1
- package/dest/lmdb-v2/message.js +6 -4
- package/dest/lmdb-v2/multi_map.d.ts +52 -0
- package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
- package/dest/lmdb-v2/multi_map.js +123 -0
- package/dest/lmdb-v2/read_transaction.d.ts +3 -1
- package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/read_transaction.js +34 -0
- package/dest/lmdb-v2/set.d.ts +15 -0
- package/dest/lmdb-v2/set.d.ts.map +1 -0
- package/dest/lmdb-v2/set.js +23 -0
- package/dest/lmdb-v2/singleton.d.ts +2 -2
- package/dest/lmdb-v2/singleton.d.ts.map +1 -1
- package/dest/lmdb-v2/singleton.js +1 -1
- package/dest/lmdb-v2/store.d.ts +13 -13
- package/dest/lmdb-v2/store.d.ts.map +1 -1
- package/dest/lmdb-v2/store.js +19 -29
- package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
- package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
- package/dest/lmdb-v2/tx-helpers.js +21 -0
- package/dest/lmdb-v2/utils.d.ts +3 -14
- package/dest/lmdb-v2/utils.d.ts.map +1 -1
- package/dest/lmdb-v2/utils.js +0 -94
- package/dest/lmdb-v2/write_transaction.d.ts +3 -5
- package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
- package/dest/lmdb-v2/write_transaction.js +9 -5
- package/dest/stores/index.d.ts +2 -1
- package/dest/stores/index.d.ts.map +1 -1
- package/dest/stores/index.js +1 -0
- package/dest/stores/l2_tips_store.d.ts +24 -8
- package/dest/stores/l2_tips_store.d.ts.map +1 -1
- package/dest/stores/l2_tips_store.js +63 -45
- package/dest/utils.d.ts +9 -6
- package/dest/utils.d.ts.map +1 -1
- package/dest/utils.js +51 -16
- package/package.json +30 -22
- package/src/indexeddb/array.ts +9 -5
- package/src/indexeddb/index.ts +11 -9
- package/src/indexeddb/map.ts +51 -56
- package/src/indexeddb/multi_map.ts +105 -0
- package/src/indexeddb/singleton.ts +5 -2
- package/src/indexeddb/store.ts +77 -60
- package/src/interfaces/array.ts +5 -3
- package/src/interfaces/common.ts +20 -9
- package/src/interfaces/index.ts +3 -1
- package/src/interfaces/map.ts +19 -53
- package/src/interfaces/map_test_suite.ts +101 -44
- package/src/interfaces/multi_map.ts +45 -0
- package/src/interfaces/multi_map_test_suite.ts +374 -0
- package/src/interfaces/store.ts +18 -53
- package/src/interfaces/utils.ts +2 -0
- package/src/lmdb/array.ts +2 -1
- package/src/lmdb/index.ts +12 -7
- package/src/lmdb/map.ts +23 -94
- package/src/lmdb/multi_map.ts +51 -0
- package/src/lmdb/store.ts +36 -56
- package/src/lmdb-v2/array.ts +8 -3
- package/src/lmdb-v2/factory.ts +30 -19
- package/src/lmdb-v2/map.ts +30 -127
- package/src/lmdb-v2/message.ts +23 -0
- package/src/lmdb-v2/multi_map.ts +150 -0
- package/src/lmdb-v2/read_transaction.ts +40 -0
- package/src/lmdb-v2/set.ts +33 -0
- package/src/lmdb-v2/singleton.ts +6 -2
- package/src/lmdb-v2/store.ts +24 -42
- package/src/lmdb-v2/tx-helpers.ts +29 -0
- package/src/lmdb-v2/utils.ts +0 -118
- package/src/lmdb-v2/write_transaction.ts +11 -10
- package/src/stores/index.ts +2 -0
- package/src/stores/l2_tips_store.ts +67 -49
- package/src/utils.ts +79 -21
- package/dest/config.d.ts +0 -17
- package/dest/config.d.ts.map +0 -1
- package/dest/config.js +0 -24
- package/dest/interfaces/store_test_suite.d.ts +0 -3
- package/dest/interfaces/store_test_suite.d.ts.map +0 -1
- package/dest/interfaces/store_test_suite.js +0 -37
- package/src/config.ts +0 -34
- package/src/interfaces/store_test_suite.ts +0 -56
package/src/indexeddb/map.ts
CHANGED
|
@@ -1,22 +1,23 @@
|
|
|
1
1
|
import type { IDBPDatabase, IDBPObjectStore } from 'idb';
|
|
2
|
+
import { hash } from 'ohash';
|
|
2
3
|
|
|
3
|
-
import type { Key, Range } from '../interfaces/common.js';
|
|
4
|
-
import type {
|
|
4
|
+
import type { Key, Range, Value } from '../interfaces/common.js';
|
|
5
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
5
6
|
import type { AztecIDBSchema } from './store.js';
|
|
6
7
|
|
|
7
8
|
/**
|
|
8
9
|
* A map backed by IndexedDB.
|
|
9
10
|
*/
|
|
10
|
-
export class IndexedDBAztecMap<K extends Key, V> implements
|
|
11
|
+
export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
|
|
11
12
|
protected name: string;
|
|
12
|
-
|
|
13
|
+
protected container: string;
|
|
13
14
|
|
|
14
15
|
#_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
|
|
15
16
|
#rootDB: IDBPDatabase<AztecIDBSchema>;
|
|
16
17
|
|
|
17
18
|
constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
|
|
18
19
|
this.name = mapName;
|
|
19
|
-
this
|
|
20
|
+
this.container = `map:${mapName}`;
|
|
20
21
|
this.#rootDB = rootDB;
|
|
21
22
|
}
|
|
22
23
|
|
|
@@ -29,21 +30,8 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
|
|
|
29
30
|
}
|
|
30
31
|
|
|
31
32
|
async getAsync(key: K): Promise<V | undefined> {
|
|
32
|
-
const data = await this.db.get(this
|
|
33
|
-
return data
|
|
34
|
-
}
|
|
35
|
-
|
|
36
|
-
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
37
|
-
const index = this.db.index('keyCount');
|
|
38
|
-
const rangeQuery = IDBKeyRange.bound(
|
|
39
|
-
[this.#container, this.#normalizeKey(key), 0],
|
|
40
|
-
[this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
41
|
-
false,
|
|
42
|
-
false,
|
|
43
|
-
);
|
|
44
|
-
for await (const cursor of index.iterate(rangeQuery)) {
|
|
45
|
-
yield cursor.value.value as V;
|
|
46
|
-
}
|
|
33
|
+
const data = await this.db.get(this.slot(key));
|
|
34
|
+
return data ? this.restoreBuffers(data.value as V) : undefined;
|
|
47
35
|
}
|
|
48
36
|
|
|
49
37
|
async hasAsync(key: K): Promise<boolean> {
|
|
@@ -51,19 +39,29 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
|
|
|
51
39
|
return result;
|
|
52
40
|
}
|
|
53
41
|
|
|
42
|
+
async sizeAsync(): Promise<number> {
|
|
43
|
+
const index = this.db.index('key');
|
|
44
|
+
const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
|
|
45
|
+
return await index.count(rangeQuery);
|
|
46
|
+
}
|
|
47
|
+
|
|
54
48
|
async set(key: K, val: V): Promise<void> {
|
|
55
|
-
const count = await this.db
|
|
56
|
-
.index('key')
|
|
57
|
-
.count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
|
|
58
49
|
await this.db.put({
|
|
59
50
|
value: val,
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
|
|
63
|
-
|
|
51
|
+
hash: hash(val),
|
|
52
|
+
container: this.container,
|
|
53
|
+
key: this.normalizeKey(key),
|
|
54
|
+
keyCount: 1,
|
|
55
|
+
slot: this.slot(key),
|
|
64
56
|
});
|
|
65
57
|
}
|
|
66
58
|
|
|
59
|
+
async setMany(entries: { key: K; value: V }[]): Promise<void> {
|
|
60
|
+
for (const { key, value } of entries) {
|
|
61
|
+
await this.set(key, value);
|
|
62
|
+
}
|
|
63
|
+
}
|
|
64
|
+
|
|
67
65
|
swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
|
|
68
66
|
throw new Error('Not implemented');
|
|
69
67
|
}
|
|
@@ -77,30 +75,17 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
|
|
|
77
75
|
}
|
|
78
76
|
|
|
79
77
|
async delete(key: K): Promise<void> {
|
|
80
|
-
await this.db.delete(this
|
|
81
|
-
}
|
|
82
|
-
|
|
83
|
-
async deleteValue(key: K, val: V): Promise<void> {
|
|
84
|
-
const index = this.db.index('keyCount');
|
|
85
|
-
const rangeQuery = IDBKeyRange.bound(
|
|
86
|
-
[this.#container, this.#normalizeKey(key), 0],
|
|
87
|
-
[this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
88
|
-
false,
|
|
89
|
-
false,
|
|
90
|
-
);
|
|
91
|
-
for await (const cursor of index.iterate(rangeQuery)) {
|
|
92
|
-
if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
|
|
93
|
-
await cursor.delete();
|
|
94
|
-
return;
|
|
95
|
-
}
|
|
96
|
-
}
|
|
78
|
+
await this.db.delete(this.slot(key));
|
|
97
79
|
}
|
|
98
80
|
|
|
99
81
|
async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
|
|
100
82
|
const index = this.db.index('key');
|
|
83
|
+
const startKey = range.start ? this.normalizeKey(range.start) : [];
|
|
84
|
+
const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
|
|
85
|
+
|
|
101
86
|
const rangeQuery = IDBKeyRange.bound(
|
|
102
|
-
[this
|
|
103
|
-
[this
|
|
87
|
+
[this.container, startKey],
|
|
88
|
+
[this.container, endKey],
|
|
104
89
|
!!range.reverse,
|
|
105
90
|
!range.reverse,
|
|
106
91
|
);
|
|
@@ -109,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
|
|
|
109
94
|
if (range.limit && count >= range.limit) {
|
|
110
95
|
return;
|
|
111
96
|
}
|
|
112
|
-
yield [cursor.value.key, cursor.value.value] as [K, V];
|
|
97
|
+
yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
|
|
113
98
|
count++;
|
|
114
99
|
}
|
|
115
100
|
}
|
|
@@ -122,21 +107,31 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
|
|
|
122
107
|
|
|
123
108
|
async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
|
|
124
109
|
for await (const [key, _] of this.entriesAsync(range)) {
|
|
125
|
-
yield
|
|
110
|
+
yield key;
|
|
111
|
+
}
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
|
|
116
|
+
* This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
|
|
117
|
+
* comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
|
|
118
|
+
*/
|
|
119
|
+
protected restoreBuffers(val: V): V {
|
|
120
|
+
if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
|
|
121
|
+
return Buffer.from(val) as V;
|
|
126
122
|
}
|
|
123
|
+
return val;
|
|
127
124
|
}
|
|
128
125
|
|
|
129
|
-
#denormalizeKey(key: string): K {
|
|
130
|
-
|
|
131
|
-
return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
|
|
126
|
+
#denormalizeKey(key: (string | number | Uint8Array)[]): K {
|
|
127
|
+
return (key.length > 1 ? key : key[0]) as K;
|
|
132
128
|
}
|
|
133
129
|
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
return arrayKey.join(',');
|
|
130
|
+
protected normalizeKey(key: K): (string | number | Uint8Array)[] {
|
|
131
|
+
return Array.isArray(key) ? key : [key];
|
|
137
132
|
}
|
|
138
133
|
|
|
139
|
-
|
|
140
|
-
return `map:${this.name}:slot:${this
|
|
134
|
+
protected slot(key: K, index: number = 0): string {
|
|
135
|
+
return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
|
|
141
136
|
}
|
|
142
137
|
}
|
|
@@ -0,0 +1,105 @@
|
|
|
1
|
+
import { hash } from 'ohash';
|
|
2
|
+
|
|
3
|
+
import type { Key, Value } from '../interfaces/common.js';
|
|
4
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
5
|
+
import { IndexedDBAztecMap } from './map.js';
|
|
6
|
+
|
|
7
|
+
/**
|
|
8
|
+
* A multi map backed by IndexedDB.
|
|
9
|
+
*/
|
|
10
|
+
export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
|
|
11
|
+
extends IndexedDBAztecMap<K, V>
|
|
12
|
+
implements AztecAsyncMultiMap<K, V>
|
|
13
|
+
{
|
|
14
|
+
override async set(key: K, val: V): Promise<void> {
|
|
15
|
+
// Inserting repeated values is a no-op
|
|
16
|
+
const exists = !!(await this.db
|
|
17
|
+
.index('hash')
|
|
18
|
+
.get(
|
|
19
|
+
IDBKeyRange.bound(
|
|
20
|
+
[this.container, this.normalizeKey(key), hash(val)],
|
|
21
|
+
[this.container, this.normalizeKey(key), hash(val)],
|
|
22
|
+
),
|
|
23
|
+
));
|
|
24
|
+
if (exists) {
|
|
25
|
+
return;
|
|
26
|
+
}
|
|
27
|
+
// Get the maximum keyCount for the given key
|
|
28
|
+
// In order to support sparse multimaps, we cannot rely
|
|
29
|
+
// on just counting the number of entries for the key, since we would repeat slots
|
|
30
|
+
// if we delete an entry
|
|
31
|
+
// set -> container:key:0 (keyCount = 1)
|
|
32
|
+
// set -> container:key:1 (keyCount = 2)
|
|
33
|
+
// delete -> container:key:0 (keyCount = 1)
|
|
34
|
+
// set -> container:key:1 <--- already exists!
|
|
35
|
+
// Instead, we iterate in reverse order to get the last inserted entry
|
|
36
|
+
const index = this.db.index('keyCount');
|
|
37
|
+
const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
|
|
38
|
+
const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
|
|
39
|
+
const count = maxEntry?.value?.keyCount ?? 0;
|
|
40
|
+
await this.db.put({
|
|
41
|
+
value: val,
|
|
42
|
+
hash: hash(val),
|
|
43
|
+
container: this.container,
|
|
44
|
+
key: this.normalizeKey(key),
|
|
45
|
+
keyCount: count + 1,
|
|
46
|
+
slot: this.slot(key, count),
|
|
47
|
+
});
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
async *getValuesAsync(key: K): AsyncIterableIterator<V> {
|
|
51
|
+
// Iterate over the whole range of keyCount for the given key
|
|
52
|
+
const index = this.db.index('keyCount');
|
|
53
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
54
|
+
[this.container, this.normalizeKey(key), 0],
|
|
55
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
56
|
+
false,
|
|
57
|
+
false,
|
|
58
|
+
);
|
|
59
|
+
for await (const cursor of index.iterate(rangeQuery)) {
|
|
60
|
+
yield this.restoreBuffers(cursor.value.value as V);
|
|
61
|
+
}
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
getValueCountAsync(key: K): Promise<number> {
|
|
65
|
+
// Count entries over the keyCount index range for this key
|
|
66
|
+
const index = this.db.index('keyCount');
|
|
67
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
68
|
+
[this.container, this.normalizeKey(key), 0],
|
|
69
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
70
|
+
false,
|
|
71
|
+
false,
|
|
72
|
+
);
|
|
73
|
+
return index.count(rangeQuery);
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
async deleteValue(key: K, val: V): Promise<void> {
|
|
77
|
+
// Since we know the value, we can hash it and directly query the "hash" index
|
|
78
|
+
// to avoid having to iterate over all the values
|
|
79
|
+
const fullKey = await this.db
|
|
80
|
+
.index('hash')
|
|
81
|
+
.getKey(
|
|
82
|
+
IDBKeyRange.bound(
|
|
83
|
+
[this.container, this.normalizeKey(key), hash(val)],
|
|
84
|
+
[this.container, this.normalizeKey(key), hash(val)],
|
|
85
|
+
),
|
|
86
|
+
);
|
|
87
|
+
if (fullKey) {
|
|
88
|
+
await this.db.delete(fullKey);
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
|
|
92
|
+
override async delete(key: K): Promise<void> {
|
|
93
|
+
const index = this.db.index('keyCount');
|
|
94
|
+
const rangeQuery = IDBKeyRange.bound(
|
|
95
|
+
[this.container, this.normalizeKey(key), 0],
|
|
96
|
+
[this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
|
|
97
|
+
false,
|
|
98
|
+
false,
|
|
99
|
+
);
|
|
100
|
+
|
|
101
|
+
for await (const cursor of index.iterate(rangeQuery)) {
|
|
102
|
+
await cursor.delete();
|
|
103
|
+
}
|
|
104
|
+
}
|
|
105
|
+
}
|
|
@@ -1,12 +1,14 @@
|
|
|
1
1
|
import type { IDBPDatabase, IDBPObjectStore } from 'idb';
|
|
2
|
+
import { hash } from 'ohash';
|
|
2
3
|
|
|
4
|
+
import type { Value } from '../interfaces/common.js';
|
|
3
5
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
4
6
|
import type { AztecIDBSchema } from './store.js';
|
|
5
7
|
|
|
6
8
|
/**
|
|
7
9
|
* Stores a single value in IndexedDB.
|
|
8
10
|
*/
|
|
9
|
-
export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
|
|
11
|
+
export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
|
|
10
12
|
#_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
|
|
11
13
|
#rootDB: IDBPDatabase<AztecIDBSchema>;
|
|
12
14
|
#container: string;
|
|
@@ -35,9 +37,10 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
|
|
|
35
37
|
const result = await this.db.put({
|
|
36
38
|
container: this.#container,
|
|
37
39
|
slot: this.#slot,
|
|
38
|
-
key: this.#slot,
|
|
40
|
+
key: [this.#slot],
|
|
39
41
|
keyCount: 1,
|
|
40
42
|
value: val,
|
|
43
|
+
hash: hash(val),
|
|
41
44
|
});
|
|
42
45
|
return result !== undefined;
|
|
43
46
|
}
|
package/src/indexeddb/store.ts
CHANGED
|
@@ -1,26 +1,41 @@
|
|
|
1
1
|
import type { Logger } from '@aztec/foundation/log';
|
|
2
|
+
import { SerialQueue } from '@aztec/foundation/queue';
|
|
2
3
|
|
|
3
4
|
import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
|
|
4
5
|
|
|
5
6
|
import type { AztecAsyncArray } from '../interfaces/array.js';
|
|
6
|
-
import type { Key, StoreSize } from '../interfaces/common.js';
|
|
7
|
+
import type { Key, StoreSize, Value } from '../interfaces/common.js';
|
|
7
8
|
import type { AztecAsyncCounter } from '../interfaces/counter.js';
|
|
8
|
-
import type { AztecAsyncMap
|
|
9
|
+
import type { AztecAsyncMap } from '../interfaces/map.js';
|
|
10
|
+
import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
|
|
9
11
|
import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
10
12
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
11
13
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
12
14
|
import { IndexedDBAztecArray } from './array.js';
|
|
13
15
|
import { IndexedDBAztecMap } from './map.js';
|
|
16
|
+
import { IndexedDBAztecMultiMap } from './multi_map.js';
|
|
14
17
|
import { IndexedDBAztecSet } from './set.js';
|
|
15
18
|
import { IndexedDBAztecSingleton } from './singleton.js';
|
|
16
19
|
|
|
17
|
-
export type StoredData<V> = {
|
|
20
|
+
export type StoredData<V extends Value> = {
|
|
21
|
+
value: V;
|
|
22
|
+
container: string;
|
|
23
|
+
key: (string | number | Uint8Array)[];
|
|
24
|
+
keyCount: number;
|
|
25
|
+
slot: string;
|
|
26
|
+
hash: string;
|
|
27
|
+
};
|
|
18
28
|
|
|
19
29
|
export interface AztecIDBSchema extends DBSchema {
|
|
20
30
|
data: {
|
|
21
31
|
value: StoredData<any>;
|
|
22
32
|
key: string;
|
|
23
|
-
indexes: {
|
|
33
|
+
indexes: {
|
|
34
|
+
container: string;
|
|
35
|
+
key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
|
|
36
|
+
keyCount: number;
|
|
37
|
+
hash: string;
|
|
38
|
+
};
|
|
24
39
|
};
|
|
25
40
|
}
|
|
26
41
|
|
|
@@ -29,18 +44,27 @@ export interface AztecIDBSchema extends DBSchema {
|
|
|
29
44
|
*/
|
|
30
45
|
|
|
31
46
|
export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
32
|
-
#log: Logger;
|
|
33
47
|
#rootDB: IDBPDatabase<AztecIDBSchema>;
|
|
34
48
|
#name: string;
|
|
49
|
+
#txQueue: SerialQueue;
|
|
35
50
|
|
|
36
51
|
#containers = new Set<
|
|
37
|
-
|
|
52
|
+
| IndexedDBAztecArray<any>
|
|
53
|
+
| IndexedDBAztecMap<any, any>
|
|
54
|
+
| IndexedDBAztecMultiMap<any, any>
|
|
55
|
+
| IndexedDBAztecSet<any>
|
|
56
|
+
| IndexedDBAztecSingleton<any>
|
|
38
57
|
>();
|
|
39
58
|
|
|
40
|
-
constructor(
|
|
59
|
+
constructor(
|
|
60
|
+
rootDB: IDBPDatabase<AztecIDBSchema>,
|
|
61
|
+
public readonly isEphemeral: boolean,
|
|
62
|
+
name: string,
|
|
63
|
+
) {
|
|
41
64
|
this.#rootDB = rootDB;
|
|
42
|
-
this.#log = log;
|
|
43
65
|
this.#name = name;
|
|
66
|
+
this.#txQueue = new SerialQueue();
|
|
67
|
+
this.#txQueue.start();
|
|
44
68
|
}
|
|
45
69
|
/**
|
|
46
70
|
* Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
|
|
@@ -53,50 +77,32 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
53
77
|
* @returns The store
|
|
54
78
|
*/
|
|
55
79
|
static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
|
|
56
|
-
name = name && !ephemeral ? name :
|
|
80
|
+
name = name && !ephemeral ? name : globalThis.crypto.getRandomValues(new Uint8Array(16)).join('');
|
|
57
81
|
log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
|
|
58
82
|
const rootDB = await openDB<AztecIDBSchema>(name, 1, {
|
|
59
83
|
upgrade(db) {
|
|
60
84
|
const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
|
|
61
85
|
|
|
62
86
|
objectStore.createIndex('key', ['container', 'key'], { unique: false });
|
|
63
|
-
|
|
87
|
+
// Keep count of the maximum number of keys ever inserted in the container
|
|
88
|
+
// This allows unique slots for repeated keys, which is useful for multi-maps
|
|
89
|
+
objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: true });
|
|
90
|
+
// Keep an index on the pair key-hash for a given container, allowing us to efficiently
|
|
91
|
+
// delete unique values from multi-maps
|
|
92
|
+
objectStore.createIndex('hash', ['container', 'key', 'hash'], { unique: true });
|
|
64
93
|
},
|
|
65
94
|
});
|
|
66
95
|
|
|
67
|
-
const kvStore = new AztecIndexedDBStore(rootDB, ephemeral,
|
|
96
|
+
const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, name);
|
|
68
97
|
return kvStore;
|
|
69
98
|
}
|
|
70
99
|
|
|
71
|
-
/**
|
|
72
|
-
* Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
|
|
73
|
-
* @returns A new AztecIndexedDBStore.
|
|
74
|
-
*/
|
|
75
|
-
async fork(): Promise<AztecAsyncKVStore> {
|
|
76
|
-
const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
|
|
77
|
-
this.#log.verbose(`Forking store to ${forkedStore.#name}`);
|
|
78
|
-
|
|
79
|
-
// Copy old data to new store
|
|
80
|
-
const oldData = this.#rootDB.transaction('data').store;
|
|
81
|
-
const dataToWrite = [];
|
|
82
|
-
for await (const cursor of oldData.iterate()) {
|
|
83
|
-
dataToWrite.push(cursor.value);
|
|
84
|
-
}
|
|
85
|
-
const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
|
|
86
|
-
for (const data of dataToWrite) {
|
|
87
|
-
await tx.add(data);
|
|
88
|
-
}
|
|
89
|
-
|
|
90
|
-
this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
|
|
91
|
-
return forkedStore;
|
|
92
|
-
}
|
|
93
|
-
|
|
94
100
|
/**
|
|
95
101
|
* Creates a new AztecMap in the store.
|
|
96
102
|
* @param name - Name of the map
|
|
97
103
|
* @returns A new AztecMap
|
|
98
104
|
*/
|
|
99
|
-
openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
|
|
105
|
+
openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
|
|
100
106
|
const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
|
|
101
107
|
this.#containers.add(map);
|
|
102
108
|
return map;
|
|
@@ -118,8 +124,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
118
124
|
* @param name - Name of the map
|
|
119
125
|
* @returns A new AztecMultiMap
|
|
120
126
|
*/
|
|
121
|
-
openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
|
|
122
|
-
const multimap = new
|
|
127
|
+
openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
|
|
128
|
+
const multimap = new IndexedDBAztecMultiMap<K, V>(this.#rootDB, name);
|
|
123
129
|
this.#containers.add(multimap);
|
|
124
130
|
return multimap;
|
|
125
131
|
}
|
|
@@ -133,7 +139,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
133
139
|
* @param name - Name of the array
|
|
134
140
|
* @returns A new AztecArray
|
|
135
141
|
*/
|
|
136
|
-
openArray<T>(name: string): AztecAsyncArray<T> {
|
|
142
|
+
openArray<T extends Value>(name: string): AztecAsyncArray<T> {
|
|
137
143
|
const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
|
|
138
144
|
this.#containers.add(array);
|
|
139
145
|
return array;
|
|
@@ -144,7 +150,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
144
150
|
* @param name - Name of the singleton
|
|
145
151
|
* @returns A new AztecSingleton
|
|
146
152
|
*/
|
|
147
|
-
openSingleton<T>(name: string): AztecAsyncSingleton<T> {
|
|
153
|
+
openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
|
|
148
154
|
const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
|
|
149
155
|
this.#containers.add(singleton);
|
|
150
156
|
return singleton;
|
|
@@ -155,22 +161,27 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
155
161
|
* @param callback - Function to execute in a transaction
|
|
156
162
|
* @returns A promise that resolves to the return value of the callback
|
|
157
163
|
*/
|
|
158
|
-
|
|
159
|
-
|
|
160
|
-
for
|
|
161
|
-
|
|
162
|
-
|
|
163
|
-
|
|
164
|
-
|
|
165
|
-
|
|
166
|
-
|
|
167
|
-
|
|
168
|
-
|
|
169
|
-
|
|
170
|
-
|
|
171
|
-
|
|
172
|
-
|
|
173
|
-
|
|
164
|
+
transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
|
|
165
|
+
// We can only have one transaction at a time for the same store
|
|
166
|
+
// So we need to wait for the current one to finish
|
|
167
|
+
return this.#txQueue.put(async () => {
|
|
168
|
+
const tx = this.#rootDB.transaction('data', 'readwrite');
|
|
169
|
+
for (const container of this.#containers) {
|
|
170
|
+
container.db = tx.store;
|
|
171
|
+
}
|
|
172
|
+
// Avoid awaiting this promise so it doesn't get scheduled in the next microtask
|
|
173
|
+
// By then, the tx would be closed
|
|
174
|
+
const runningPromise = callback();
|
|
175
|
+
// Wait for the transaction to finish
|
|
176
|
+
await tx.done;
|
|
177
|
+
for (const container of this.#containers) {
|
|
178
|
+
container.db = undefined;
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
// Return the result of the callback.
|
|
182
|
+
// Tx is guaranteed to already be closed, so the await doesn't hurt anything here
|
|
183
|
+
return await runningPromise;
|
|
184
|
+
});
|
|
174
185
|
}
|
|
175
186
|
|
|
176
187
|
/**
|
|
@@ -181,17 +192,23 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
|
|
|
181
192
|
}
|
|
182
193
|
|
|
183
194
|
/** Deletes this store and removes the database */
|
|
184
|
-
delete() {
|
|
195
|
+
async delete() {
|
|
185
196
|
this.#containers.clear();
|
|
197
|
+
await this.#txQueue.end();
|
|
186
198
|
this.#rootDB.close();
|
|
187
|
-
|
|
199
|
+
await deleteDB(this.#name);
|
|
188
200
|
}
|
|
189
201
|
|
|
190
202
|
estimateSize(): Promise<StoreSize> {
|
|
191
|
-
return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
|
|
203
|
+
return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
|
|
192
204
|
}
|
|
193
205
|
|
|
194
|
-
close(): Promise<void> {
|
|
195
|
-
|
|
206
|
+
async close(): Promise<void> {
|
|
207
|
+
await this.#txQueue.end();
|
|
208
|
+
this.#rootDB.close();
|
|
209
|
+
}
|
|
210
|
+
|
|
211
|
+
backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
|
|
212
|
+
throw new Error('Method not implemented.');
|
|
196
213
|
}
|
|
197
214
|
}
|
package/src/interfaces/array.ts
CHANGED
|
@@ -1,7 +1,9 @@
|
|
|
1
|
+
import type { Value } from './common.js';
|
|
2
|
+
|
|
1
3
|
/**
|
|
2
4
|
* An array backed by a persistent store. Can not have any holes in it.
|
|
3
5
|
*/
|
|
4
|
-
interface BaseAztecArray<T> {
|
|
6
|
+
interface BaseAztecArray<T extends Value> {
|
|
5
7
|
/**
|
|
6
8
|
* Pushes values to the end of the array
|
|
7
9
|
* @param vals - The values to push to the end of the array
|
|
@@ -27,7 +29,7 @@ interface BaseAztecArray<T> {
|
|
|
27
29
|
/**
|
|
28
30
|
* An array backed by a persistent store. Can not have any holes in it.
|
|
29
31
|
*/
|
|
30
|
-
export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
|
|
32
|
+
export interface AztecAsyncArray<T extends Value> extends BaseAztecArray<T> {
|
|
31
33
|
/**
|
|
32
34
|
* The size of the array
|
|
33
35
|
*/
|
|
@@ -58,7 +60,7 @@ export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
|
|
|
58
60
|
[Symbol.asyncIterator](): AsyncIterableIterator<T>;
|
|
59
61
|
}
|
|
60
62
|
|
|
61
|
-
export interface AztecArray<T> extends BaseAztecArray<T> {
|
|
63
|
+
export interface AztecArray<T extends Value> extends BaseAztecArray<T> {
|
|
62
64
|
/**
|
|
63
65
|
* The size of the array
|
|
64
66
|
*/
|
package/src/interfaces/common.ts
CHANGED
|
@@ -1,12 +1,10 @@
|
|
|
1
|
-
/**
|
|
2
|
-
|
|
3
|
-
*/
|
|
4
|
-
export type Key = string | number | Array<string | number>;
|
|
1
|
+
/** The key type for use with the kv-store */
|
|
2
|
+
export type Key = string | number | Uint8Array | Array<string | number>;
|
|
5
3
|
|
|
6
|
-
|
|
7
|
-
|
|
8
|
-
*/
|
|
9
|
-
export type
|
|
4
|
+
export type Value = NonNullable<any>;
|
|
5
|
+
|
|
6
|
+
/** A range of keys of arbitrary type. */
|
|
7
|
+
export type CustomRange<K> = {
|
|
10
8
|
/** The key of the first item to include */
|
|
11
9
|
start?: K;
|
|
12
10
|
/** The key of the last item to include */
|
|
@@ -17,4 +15,17 @@ export type Range<K extends Key = Key> = {
|
|
|
17
15
|
limit?: number;
|
|
18
16
|
};
|
|
19
17
|
|
|
20
|
-
|
|
18
|
+
/** Maps a custom range into a range of valid key types to iterate over. */
|
|
19
|
+
export function mapRange<CK, K extends Key = Key>(range: CustomRange<CK>, mapFn: (key: CK) => K): Range<K> {
|
|
20
|
+
return {
|
|
21
|
+
start: range.start ? mapFn(range.start) : undefined,
|
|
22
|
+
end: range.end ? mapFn(range.end) : undefined,
|
|
23
|
+
reverse: range.reverse,
|
|
24
|
+
limit: range.limit,
|
|
25
|
+
};
|
|
26
|
+
}
|
|
27
|
+
|
|
28
|
+
/** A range of keys to iterate over. */
|
|
29
|
+
export type Range<K extends Key = Key> = CustomRange<K>;
|
|
30
|
+
|
|
31
|
+
export type StoreSize = { mappingSize: number; physicalFileSize: number; actualSize: number; numItems: number };
|
package/src/interfaces/index.ts
CHANGED
|
@@ -4,4 +4,6 @@ export * from './counter.js';
|
|
|
4
4
|
export * from './singleton.js';
|
|
5
5
|
export * from './store.js';
|
|
6
6
|
export * from './set.js';
|
|
7
|
-
export
|
|
7
|
+
export * from './multi_map.js';
|
|
8
|
+
export { mapRange } from './common.js';
|
|
9
|
+
export type { CustomRange, Range, StoreSize } from './common.js';
|