@aztec/kv-store 0.0.0-test.0 → 0.0.1-commit.001888fc

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/dest/indexeddb/array.d.ts +3 -2
  2. package/dest/indexeddb/array.d.ts.map +1 -1
  3. package/dest/indexeddb/array.js +24 -7
  4. package/dest/indexeddb/index.d.ts +3 -3
  5. package/dest/indexeddb/index.d.ts.map +1 -1
  6. package/dest/indexeddb/index.js +4 -7
  7. package/dest/indexeddb/map.d.ts +18 -6
  8. package/dest/indexeddb/map.d.ts.map +1 -1
  9. package/dest/indexeddb/map.js +51 -63
  10. package/dest/indexeddb/multi_map.d.ts +14 -0
  11. package/dest/indexeddb/multi_map.d.ts.map +1 -0
  12. package/dest/indexeddb/multi_map.js +107 -0
  13. package/dest/indexeddb/set.d.ts +1 -1
  14. package/dest/indexeddb/set.d.ts.map +1 -1
  15. package/dest/indexeddb/singleton.d.ts +3 -2
  16. package/dest/indexeddb/singleton.d.ts.map +1 -1
  17. package/dest/indexeddb/singleton.js +6 -2
  18. package/dest/indexeddb/store.d.ts +15 -16
  19. package/dest/indexeddb/store.d.ts.map +1 -1
  20. package/dest/indexeddb/store.js +51 -46
  21. package/dest/interfaces/array.d.ts +5 -4
  22. package/dest/interfaces/array.d.ts.map +1 -1
  23. package/dest/interfaces/array.js +1 -3
  24. package/dest/interfaces/array_test_suite.d.ts +1 -1
  25. package/dest/interfaces/common.d.ts +11 -9
  26. package/dest/interfaces/common.d.ts.map +1 -1
  27. package/dest/interfaces/common.js +8 -3
  28. package/dest/interfaces/counter.d.ts +1 -1
  29. package/dest/interfaces/index.d.ts +4 -2
  30. package/dest/interfaces/index.d.ts.map +1 -1
  31. package/dest/interfaces/index.js +2 -0
  32. package/dest/interfaces/map.d.ts +21 -49
  33. package/dest/interfaces/map.d.ts.map +1 -1
  34. package/dest/interfaces/map.js +1 -1
  35. package/dest/interfaces/map_test_suite.d.ts +1 -1
  36. package/dest/interfaces/map_test_suite.d.ts.map +1 -1
  37. package/dest/interfaces/map_test_suite.js +181 -70
  38. package/dest/interfaces/multi_map.d.ts +41 -0
  39. package/dest/interfaces/multi_map.d.ts.map +1 -0
  40. package/dest/interfaces/multi_map.js +3 -0
  41. package/dest/interfaces/multi_map_test_suite.d.ts +3 -0
  42. package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -0
  43. package/dest/interfaces/multi_map_test_suite.js +364 -0
  44. package/dest/interfaces/set.d.ts +1 -1
  45. package/dest/interfaces/set_test_suite.d.ts +1 -1
  46. package/dest/interfaces/singleton.d.ts +1 -1
  47. package/dest/interfaces/singleton_test_suite.d.ts +1 -1
  48. package/dest/interfaces/store.d.ts +18 -43
  49. package/dest/interfaces/store.d.ts.map +1 -1
  50. package/dest/interfaces/utils.d.ts +3 -1
  51. package/dest/interfaces/utils.d.ts.map +1 -1
  52. package/dest/interfaces/utils.js +3 -1
  53. package/dest/lmdb/array.d.ts +3 -2
  54. package/dest/lmdb/array.d.ts.map +1 -1
  55. package/dest/lmdb/array.js +4 -2
  56. package/dest/lmdb/counter.d.ts +1 -1
  57. package/dest/lmdb/counter.d.ts.map +1 -1
  58. package/dest/lmdb/index.d.ts +3 -3
  59. package/dest/lmdb/index.d.ts.map +1 -1
  60. package/dest/lmdb/index.js +5 -5
  61. package/dest/lmdb/map.d.ts +11 -23
  62. package/dest/lmdb/map.d.ts.map +1 -1
  63. package/dest/lmdb/map.js +15 -81
  64. package/dest/lmdb/multi_map.d.ts +13 -0
  65. package/dest/lmdb/multi_map.d.ts.map +1 -0
  66. package/dest/lmdb/multi_map.js +44 -0
  67. package/dest/lmdb/set.d.ts +1 -1
  68. package/dest/lmdb/set.d.ts.map +1 -1
  69. package/dest/lmdb/singleton.d.ts +1 -1
  70. package/dest/lmdb/singleton.d.ts.map +1 -1
  71. package/dest/lmdb/store.d.ts +11 -27
  72. package/dest/lmdb/store.d.ts.map +1 -1
  73. package/dest/lmdb/store.js +24 -40
  74. package/dest/lmdb-v2/array.d.ts +4 -3
  75. package/dest/lmdb-v2/array.d.ts.map +1 -1
  76. package/dest/lmdb-v2/array.js +4 -2
  77. package/dest/lmdb-v2/factory.d.ts +7 -10
  78. package/dest/lmdb-v2/factory.d.ts.map +1 -1
  79. package/dest/lmdb-v2/factory.js +26 -12
  80. package/dest/lmdb-v2/index.d.ts +1 -1
  81. package/dest/lmdb-v2/map.d.ts +12 -45
  82. package/dest/lmdb-v2/map.d.ts.map +1 -1
  83. package/dest/lmdb-v2/map.js +17 -104
  84. package/dest/lmdb-v2/message.d.ts +24 -5
  85. package/dest/lmdb-v2/message.d.ts.map +1 -1
  86. package/dest/lmdb-v2/message.js +6 -4
  87. package/dest/lmdb-v2/multi_map.d.ts +52 -0
  88. package/dest/lmdb-v2/multi_map.d.ts.map +1 -0
  89. package/dest/lmdb-v2/multi_map.js +123 -0
  90. package/dest/lmdb-v2/read_transaction.d.ts +3 -1
  91. package/dest/lmdb-v2/read_transaction.d.ts.map +1 -1
  92. package/dest/lmdb-v2/read_transaction.js +34 -0
  93. package/dest/lmdb-v2/set.d.ts +15 -0
  94. package/dest/lmdb-v2/set.d.ts.map +1 -0
  95. package/dest/lmdb-v2/set.js +23 -0
  96. package/dest/lmdb-v2/singleton.d.ts +2 -2
  97. package/dest/lmdb-v2/singleton.d.ts.map +1 -1
  98. package/dest/lmdb-v2/singleton.js +1 -1
  99. package/dest/lmdb-v2/store.d.ts +13 -13
  100. package/dest/lmdb-v2/store.d.ts.map +1 -1
  101. package/dest/lmdb-v2/store.js +19 -29
  102. package/dest/lmdb-v2/tx-helpers.d.ts +6 -0
  103. package/dest/lmdb-v2/tx-helpers.d.ts.map +1 -0
  104. package/dest/lmdb-v2/tx-helpers.js +21 -0
  105. package/dest/lmdb-v2/utils.d.ts +3 -14
  106. package/dest/lmdb-v2/utils.d.ts.map +1 -1
  107. package/dest/lmdb-v2/utils.js +0 -94
  108. package/dest/lmdb-v2/write_transaction.d.ts +3 -5
  109. package/dest/lmdb-v2/write_transaction.d.ts.map +1 -1
  110. package/dest/lmdb-v2/write_transaction.js +9 -5
  111. package/dest/stores/index.d.ts +2 -1
  112. package/dest/stores/index.d.ts.map +1 -1
  113. package/dest/stores/index.js +1 -0
  114. package/dest/stores/l2_tips_store.d.ts +24 -8
  115. package/dest/stores/l2_tips_store.d.ts.map +1 -1
  116. package/dest/stores/l2_tips_store.js +63 -45
  117. package/dest/utils.d.ts +9 -6
  118. package/dest/utils.d.ts.map +1 -1
  119. package/dest/utils.js +51 -16
  120. package/package.json +30 -22
  121. package/src/indexeddb/array.ts +9 -5
  122. package/src/indexeddb/index.ts +11 -9
  123. package/src/indexeddb/map.ts +51 -56
  124. package/src/indexeddb/multi_map.ts +105 -0
  125. package/src/indexeddb/singleton.ts +5 -2
  126. package/src/indexeddb/store.ts +77 -60
  127. package/src/interfaces/array.ts +5 -3
  128. package/src/interfaces/common.ts +20 -9
  129. package/src/interfaces/index.ts +3 -1
  130. package/src/interfaces/map.ts +19 -53
  131. package/src/interfaces/map_test_suite.ts +101 -44
  132. package/src/interfaces/multi_map.ts +45 -0
  133. package/src/interfaces/multi_map_test_suite.ts +374 -0
  134. package/src/interfaces/store.ts +18 -53
  135. package/src/interfaces/utils.ts +2 -0
  136. package/src/lmdb/array.ts +2 -1
  137. package/src/lmdb/index.ts +12 -7
  138. package/src/lmdb/map.ts +23 -94
  139. package/src/lmdb/multi_map.ts +51 -0
  140. package/src/lmdb/store.ts +36 -56
  141. package/src/lmdb-v2/array.ts +8 -3
  142. package/src/lmdb-v2/factory.ts +30 -19
  143. package/src/lmdb-v2/map.ts +30 -127
  144. package/src/lmdb-v2/message.ts +23 -0
  145. package/src/lmdb-v2/multi_map.ts +150 -0
  146. package/src/lmdb-v2/read_transaction.ts +40 -0
  147. package/src/lmdb-v2/set.ts +33 -0
  148. package/src/lmdb-v2/singleton.ts +6 -2
  149. package/src/lmdb-v2/store.ts +24 -42
  150. package/src/lmdb-v2/tx-helpers.ts +29 -0
  151. package/src/lmdb-v2/utils.ts +0 -118
  152. package/src/lmdb-v2/write_transaction.ts +11 -10
  153. package/src/stores/index.ts +2 -0
  154. package/src/stores/l2_tips_store.ts +67 -49
  155. package/src/utils.ts +79 -21
  156. package/dest/config.d.ts +0 -17
  157. package/dest/config.d.ts.map +0 -1
  158. package/dest/config.js +0 -24
  159. package/dest/interfaces/store_test_suite.d.ts +0 -3
  160. package/dest/interfaces/store_test_suite.d.ts.map +0 -1
  161. package/dest/interfaces/store_test_suite.js +0 -37
  162. package/src/config.ts +0 -34
  163. package/src/interfaces/store_test_suite.ts +0 -56
@@ -1,22 +1,23 @@
1
1
  import type { IDBPDatabase, IDBPObjectStore } from 'idb';
2
+ import { hash } from 'ohash';
2
3
 
3
- import type { Key, Range } from '../interfaces/common.js';
4
- import type { AztecAsyncMultiMap } from '../interfaces/map.js';
4
+ import type { Key, Range, Value } from '../interfaces/common.js';
5
+ import type { AztecAsyncMap } from '../interfaces/map.js';
5
6
  import type { AztecIDBSchema } from './store.js';
6
7
 
7
8
  /**
8
9
  * A map backed by IndexedDB.
9
10
  */
10
- export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K, V> {
11
+ export class IndexedDBAztecMap<K extends Key, V extends Value> implements AztecAsyncMap<K, V> {
11
12
  protected name: string;
12
- #container: string;
13
+ protected container: string;
13
14
 
14
15
  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
15
16
  #rootDB: IDBPDatabase<AztecIDBSchema>;
16
17
 
17
18
  constructor(rootDB: IDBPDatabase<AztecIDBSchema>, mapName: string) {
18
19
  this.name = mapName;
19
- this.#container = `map:${mapName}`;
20
+ this.container = `map:${mapName}`;
20
21
  this.#rootDB = rootDB;
21
22
  }
22
23
 
@@ -29,21 +30,8 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
29
30
  }
30
31
 
31
32
  async getAsync(key: K): Promise<V | undefined> {
32
- const data = await this.db.get(this.#slot(key));
33
- return data?.value as V;
34
- }
35
-
36
- async *getValuesAsync(key: K): AsyncIterableIterator<V> {
37
- const index = this.db.index('keyCount');
38
- const rangeQuery = IDBKeyRange.bound(
39
- [this.#container, this.#normalizeKey(key), 0],
40
- [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
41
- false,
42
- false,
43
- );
44
- for await (const cursor of index.iterate(rangeQuery)) {
45
- yield cursor.value.value as V;
46
- }
33
+ const data = await this.db.get(this.slot(key));
34
+ return data ? this.restoreBuffers(data.value as V) : undefined;
47
35
  }
48
36
 
49
37
  async hasAsync(key: K): Promise<boolean> {
@@ -51,19 +39,29 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
51
39
  return result;
52
40
  }
53
41
 
42
+ async sizeAsync(): Promise<number> {
43
+ const index = this.db.index('key');
44
+ const rangeQuery = IDBKeyRange.bound([this.container], [this.container + '\uffff'], true, true);
45
+ return await index.count(rangeQuery);
46
+ }
47
+
54
48
  async set(key: K, val: V): Promise<void> {
55
- const count = await this.db
56
- .index('key')
57
- .count(IDBKeyRange.bound([this.#container, this.#normalizeKey(key)], [this.#container, this.#normalizeKey(key)]));
58
49
  await this.db.put({
59
50
  value: val,
60
- container: this.#container,
61
- key: this.#normalizeKey(key),
62
- keyCount: count + 1,
63
- slot: this.#slot(key, count),
51
+ hash: hash(val),
52
+ container: this.container,
53
+ key: this.normalizeKey(key),
54
+ keyCount: 1,
55
+ slot: this.slot(key),
64
56
  });
65
57
  }
66
58
 
59
+ async setMany(entries: { key: K; value: V }[]): Promise<void> {
60
+ for (const { key, value } of entries) {
61
+ await this.set(key, value);
62
+ }
63
+ }
64
+
67
65
  swap(_key: K, _fn: (val: V | undefined) => V): Promise<void> {
68
66
  throw new Error('Not implemented');
69
67
  }
@@ -77,30 +75,17 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
77
75
  }
78
76
 
79
77
  async delete(key: K): Promise<void> {
80
- await this.db.delete(this.#slot(key));
81
- }
82
-
83
- async deleteValue(key: K, val: V): Promise<void> {
84
- const index = this.db.index('keyCount');
85
- const rangeQuery = IDBKeyRange.bound(
86
- [this.#container, this.#normalizeKey(key), 0],
87
- [this.#container, this.#normalizeKey(key), Number.MAX_SAFE_INTEGER],
88
- false,
89
- false,
90
- );
91
- for await (const cursor of index.iterate(rangeQuery)) {
92
- if (JSON.stringify(cursor.value.value) === JSON.stringify(val)) {
93
- await cursor.delete();
94
- return;
95
- }
96
- }
78
+ await this.db.delete(this.slot(key));
97
79
  }
98
80
 
99
81
  async *entriesAsync(range: Range<K> = {}): AsyncIterableIterator<[K, V]> {
100
82
  const index = this.db.index('key');
83
+ const startKey = range.start ? this.normalizeKey(range.start) : [];
84
+ const endKey = range.end ? this.normalizeKey(range.end) : ['\uffff'];
85
+
101
86
  const rangeQuery = IDBKeyRange.bound(
102
- [this.#container, range.start ?? ''],
103
- [this.#container, range.end ?? '\uffff'],
87
+ [this.container, startKey],
88
+ [this.container, endKey],
104
89
  !!range.reverse,
105
90
  !range.reverse,
106
91
  );
@@ -109,7 +94,7 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
109
94
  if (range.limit && count >= range.limit) {
110
95
  return;
111
96
  }
112
- yield [cursor.value.key, cursor.value.value] as [K, V];
97
+ yield [this.#denormalizeKey(cursor.value.key), this.restoreBuffers(cursor.value.value as V)] as [K, V];
113
98
  count++;
114
99
  }
115
100
  }
@@ -122,21 +107,31 @@ export class IndexedDBAztecMap<K extends Key, V> implements AztecAsyncMultiMap<K
122
107
 
123
108
  async *keysAsync(range: Range<K> = {}): AsyncIterableIterator<K> {
124
109
  for await (const [key, _] of this.entriesAsync(range)) {
125
- yield this.#denormalizeKey(key as string);
110
+ yield key;
111
+ }
112
+ }
113
+
114
+ /**
115
+ * IndexedDB's structured clone downcasts Buffer (a Node.js subclass of Uint8Array) to plain Uint8Array.
116
+ * This breaks .toString() behavior: Buffer.toString() returns UTF-8, Uint8Array.toString() returns
117
+ * comma-separated decimal bytes. We restore Buffer identity on read so callers get consistent behavior.
118
+ */
119
+ protected restoreBuffers(val: V): V {
120
+ if (val instanceof Uint8Array && !Buffer.isBuffer(val)) {
121
+ return Buffer.from(val) as V;
126
122
  }
123
+ return val;
127
124
  }
128
125
 
129
- #denormalizeKey(key: string): K {
130
- const denormalizedKey = (key as string).split(',').map(part => (isNaN(parseInt(part)) ? part : parseInt(part)));
131
- return (denormalizedKey.length > 1 ? denormalizedKey : key) as K;
126
+ #denormalizeKey(key: (string | number | Uint8Array)[]): K {
127
+ return (key.length > 1 ? key : key[0]) as K;
132
128
  }
133
129
 
134
- #normalizeKey(key: K): string {
135
- const arrayKey = Array.isArray(key) ? key : [key];
136
- return arrayKey.join(',');
130
+ protected normalizeKey(key: K): (string | number | Uint8Array)[] {
131
+ return Array.isArray(key) ? key : [key];
137
132
  }
138
133
 
139
- #slot(key: K, index: number = 0): string {
140
- return `map:${this.name}:slot:${this.#normalizeKey(key)}:${index}`;
134
+ protected slot(key: K, index: number = 0): string {
135
+ return `map:${this.name}:slot:${this.normalizeKey(key)}:${index}`;
141
136
  }
142
137
  }
@@ -0,0 +1,105 @@
1
+ import { hash } from 'ohash';
2
+
3
+ import type { Key, Value } from '../interfaces/common.js';
4
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
5
+ import { IndexedDBAztecMap } from './map.js';
6
+
7
+ /**
8
+ * A multi map backed by IndexedDB.
9
+ */
10
+ export class IndexedDBAztecMultiMap<K extends Key, V extends Value>
11
+ extends IndexedDBAztecMap<K, V>
12
+ implements AztecAsyncMultiMap<K, V>
13
+ {
14
+ override async set(key: K, val: V): Promise<void> {
15
+ // Inserting repeated values is a no-op
16
+ const exists = !!(await this.db
17
+ .index('hash')
18
+ .get(
19
+ IDBKeyRange.bound(
20
+ [this.container, this.normalizeKey(key), hash(val)],
21
+ [this.container, this.normalizeKey(key), hash(val)],
22
+ ),
23
+ ));
24
+ if (exists) {
25
+ return;
26
+ }
27
+ // Get the maximum keyCount for the given key
28
+ // In order to support sparse multimaps, we cannot rely
29
+ // on just counting the number of entries for the key, since we would repeat slots
30
+ // if we delete an entry
31
+ // set -> container:key:0 (keyCount = 1)
32
+ // set -> container:key:1 (keyCount = 2)
33
+ // delete -> container:key:0 (keyCount = 1)
34
+ // set -> container:key:1 <--- already exists!
35
+ // Instead, we iterate in reverse order to get the last inserted entry
36
+ const index = this.db.index('keyCount');
37
+ const rangeQuery = IDBKeyRange.upperBound([this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER]);
38
+ const maxEntry = (await index.iterate(rangeQuery, 'prevunique').next()).value;
39
+ const count = maxEntry?.value?.keyCount ?? 0;
40
+ await this.db.put({
41
+ value: val,
42
+ hash: hash(val),
43
+ container: this.container,
44
+ key: this.normalizeKey(key),
45
+ keyCount: count + 1,
46
+ slot: this.slot(key, count),
47
+ });
48
+ }
49
+
50
+ async *getValuesAsync(key: K): AsyncIterableIterator<V> {
51
+ // Iterate over the whole range of keyCount for the given key
52
+ const index = this.db.index('keyCount');
53
+ const rangeQuery = IDBKeyRange.bound(
54
+ [this.container, this.normalizeKey(key), 0],
55
+ [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
56
+ false,
57
+ false,
58
+ );
59
+ for await (const cursor of index.iterate(rangeQuery)) {
60
+ yield this.restoreBuffers(cursor.value.value as V);
61
+ }
62
+ }
63
+
64
+ getValueCountAsync(key: K): Promise<number> {
65
+ // Count entries over the keyCount index range for this key
66
+ const index = this.db.index('keyCount');
67
+ const rangeQuery = IDBKeyRange.bound(
68
+ [this.container, this.normalizeKey(key), 0],
69
+ [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
70
+ false,
71
+ false,
72
+ );
73
+ return index.count(rangeQuery);
74
+ }
75
+
76
+ async deleteValue(key: K, val: V): Promise<void> {
77
+ // Since we know the value, we can hash it and directly query the "hash" index
78
+ // to avoid having to iterate over all the values
79
+ const fullKey = await this.db
80
+ .index('hash')
81
+ .getKey(
82
+ IDBKeyRange.bound(
83
+ [this.container, this.normalizeKey(key), hash(val)],
84
+ [this.container, this.normalizeKey(key), hash(val)],
85
+ ),
86
+ );
87
+ if (fullKey) {
88
+ await this.db.delete(fullKey);
89
+ }
90
+ }
91
+
92
+ override async delete(key: K): Promise<void> {
93
+ const index = this.db.index('keyCount');
94
+ const rangeQuery = IDBKeyRange.bound(
95
+ [this.container, this.normalizeKey(key), 0],
96
+ [this.container, this.normalizeKey(key), Number.MAX_SAFE_INTEGER],
97
+ false,
98
+ false,
99
+ );
100
+
101
+ for await (const cursor of index.iterate(rangeQuery)) {
102
+ await cursor.delete();
103
+ }
104
+ }
105
+ }
@@ -1,12 +1,14 @@
1
1
  import type { IDBPDatabase, IDBPObjectStore } from 'idb';
2
+ import { hash } from 'ohash';
2
3
 
4
+ import type { Value } from '../interfaces/common.js';
3
5
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
4
6
  import type { AztecIDBSchema } from './store.js';
5
7
 
6
8
  /**
7
9
  * Stores a single value in IndexedDB.
8
10
  */
9
- export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
11
+ export class IndexedDBAztecSingleton<T extends Value> implements AztecAsyncSingleton<T> {
10
12
  #_db?: IDBPObjectStore<AztecIDBSchema, ['data'], 'data', 'readwrite'>;
11
13
  #rootDB: IDBPDatabase<AztecIDBSchema>;
12
14
  #container: string;
@@ -35,9 +37,10 @@ export class IndexedDBAztecSingleton<T> implements AztecAsyncSingleton<T> {
35
37
  const result = await this.db.put({
36
38
  container: this.#container,
37
39
  slot: this.#slot,
38
- key: this.#slot,
40
+ key: [this.#slot],
39
41
  keyCount: 1,
40
42
  value: val,
43
+ hash: hash(val),
41
44
  });
42
45
  return result !== undefined;
43
46
  }
@@ -1,26 +1,41 @@
1
1
  import type { Logger } from '@aztec/foundation/log';
2
+ import { SerialQueue } from '@aztec/foundation/queue';
2
3
 
3
4
  import { type DBSchema, type IDBPDatabase, deleteDB, openDB } from 'idb';
4
5
 
5
6
  import type { AztecAsyncArray } from '../interfaces/array.js';
6
- import type { Key, StoreSize } from '../interfaces/common.js';
7
+ import type { Key, StoreSize, Value } from '../interfaces/common.js';
7
8
  import type { AztecAsyncCounter } from '../interfaces/counter.js';
8
- import type { AztecAsyncMap, AztecAsyncMultiMap } from '../interfaces/map.js';
9
+ import type { AztecAsyncMap } from '../interfaces/map.js';
10
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
9
11
  import type { AztecAsyncSet } from '../interfaces/set.js';
10
12
  import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
11
13
  import type { AztecAsyncKVStore } from '../interfaces/store.js';
12
14
  import { IndexedDBAztecArray } from './array.js';
13
15
  import { IndexedDBAztecMap } from './map.js';
16
+ import { IndexedDBAztecMultiMap } from './multi_map.js';
14
17
  import { IndexedDBAztecSet } from './set.js';
15
18
  import { IndexedDBAztecSingleton } from './singleton.js';
16
19
 
17
- export type StoredData<V> = { value: V; container: string; key: string; keyCount: number; slot: string };
20
+ export type StoredData<V extends Value> = {
21
+ value: V;
22
+ container: string;
23
+ key: (string | number | Uint8Array)[];
24
+ keyCount: number;
25
+ slot: string;
26
+ hash: string;
27
+ };
18
28
 
19
29
  export interface AztecIDBSchema extends DBSchema {
20
30
  data: {
21
31
  value: StoredData<any>;
22
32
  key: string;
23
- indexes: { container: string; key: string; keyCount: number };
33
+ indexes: {
34
+ container: string;
35
+ key: any; // Runtime: (string | number | Uint8Array)[] - idb types don't support arrays
36
+ keyCount: number;
37
+ hash: string;
38
+ };
24
39
  };
25
40
  }
26
41
 
@@ -29,18 +44,27 @@ export interface AztecIDBSchema extends DBSchema {
29
44
  */
30
45
 
31
46
  export class AztecIndexedDBStore implements AztecAsyncKVStore {
32
- #log: Logger;
33
47
  #rootDB: IDBPDatabase<AztecIDBSchema>;
34
48
  #name: string;
49
+ #txQueue: SerialQueue;
35
50
 
36
51
  #containers = new Set<
37
- IndexedDBAztecArray<any> | IndexedDBAztecMap<any, any> | IndexedDBAztecSet<any> | IndexedDBAztecSingleton<any>
52
+ | IndexedDBAztecArray<any>
53
+ | IndexedDBAztecMap<any, any>
54
+ | IndexedDBAztecMultiMap<any, any>
55
+ | IndexedDBAztecSet<any>
56
+ | IndexedDBAztecSingleton<any>
38
57
  >();
39
58
 
40
- constructor(rootDB: IDBPDatabase<AztecIDBSchema>, public readonly isEphemeral: boolean, log: Logger, name: string) {
59
+ constructor(
60
+ rootDB: IDBPDatabase<AztecIDBSchema>,
61
+ public readonly isEphemeral: boolean,
62
+ name: string,
63
+ ) {
41
64
  this.#rootDB = rootDB;
42
- this.#log = log;
43
65
  this.#name = name;
66
+ this.#txQueue = new SerialQueue();
67
+ this.#txQueue.start();
44
68
  }
45
69
  /**
46
70
  * Creates a new AztecKVStore backed by IndexedDB. The path to the database is optional. If not provided,
@@ -53,50 +77,32 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
53
77
  * @returns The store
54
78
  */
55
79
  static async open(log: Logger, name?: string, ephemeral: boolean = false): Promise<AztecIndexedDBStore> {
56
- name = name && !ephemeral ? name : self.crypto.getRandomValues(new Uint8Array(16)).join('');
80
+ name = name && !ephemeral ? name : globalThis.crypto.getRandomValues(new Uint8Array(16)).join('');
57
81
  log.debug(`Opening IndexedDB ${ephemeral ? 'temp ' : ''}database with name ${name}`);
58
82
  const rootDB = await openDB<AztecIDBSchema>(name, 1, {
59
83
  upgrade(db) {
60
84
  const objectStore = db.createObjectStore('data', { keyPath: 'slot' });
61
85
 
62
86
  objectStore.createIndex('key', ['container', 'key'], { unique: false });
63
- objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: false });
87
+ // Keep count of the maximum number of keys ever inserted in the container
88
+ // This allows unique slots for repeated keys, which is useful for multi-maps
89
+ objectStore.createIndex('keyCount', ['container', 'key', 'keyCount'], { unique: true });
90
+ // Keep an index on the pair key-hash for a given container, allowing us to efficiently
91
+ // delete unique values from multi-maps
92
+ objectStore.createIndex('hash', ['container', 'key', 'hash'], { unique: true });
64
93
  },
65
94
  });
66
95
 
67
- const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, log, name);
96
+ const kvStore = new AztecIndexedDBStore(rootDB, ephemeral, name);
68
97
  return kvStore;
69
98
  }
70
99
 
71
- /**
72
- * Forks the current DB into a new DB by backing it up to a temporary location and opening a new indexedb.
73
- * @returns A new AztecIndexedDBStore.
74
- */
75
- async fork(): Promise<AztecAsyncKVStore> {
76
- const forkedStore = await AztecIndexedDBStore.open(this.#log, undefined, true);
77
- this.#log.verbose(`Forking store to ${forkedStore.#name}`);
78
-
79
- // Copy old data to new store
80
- const oldData = this.#rootDB.transaction('data').store;
81
- const dataToWrite = [];
82
- for await (const cursor of oldData.iterate()) {
83
- dataToWrite.push(cursor.value);
84
- }
85
- const tx = forkedStore.#rootDB.transaction('data', 'readwrite').store;
86
- for (const data of dataToWrite) {
87
- await tx.add(data);
88
- }
89
-
90
- this.#log.debug(`Forked store at ${forkedStore.#name} opened successfully`);
91
- return forkedStore;
92
- }
93
-
94
100
  /**
95
101
  * Creates a new AztecMap in the store.
96
102
  * @param name - Name of the map
97
103
  * @returns A new AztecMap
98
104
  */
99
- openMap<K extends Key, V>(name: string): AztecAsyncMap<K, V> {
105
+ openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
100
106
  const map = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
101
107
  this.#containers.add(map);
102
108
  return map;
@@ -118,8 +124,8 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
118
124
  * @param name - Name of the map
119
125
  * @returns A new AztecMultiMap
120
126
  */
121
- openMultiMap<K extends Key, V>(name: string): AztecAsyncMultiMap<K, V> {
122
- const multimap = new IndexedDBAztecMap<K, V>(this.#rootDB, name);
127
+ openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
128
+ const multimap = new IndexedDBAztecMultiMap<K, V>(this.#rootDB, name);
123
129
  this.#containers.add(multimap);
124
130
  return multimap;
125
131
  }
@@ -133,7 +139,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
133
139
  * @param name - Name of the array
134
140
  * @returns A new AztecArray
135
141
  */
136
- openArray<T>(name: string): AztecAsyncArray<T> {
142
+ openArray<T extends Value>(name: string): AztecAsyncArray<T> {
137
143
  const array = new IndexedDBAztecArray<T>(this.#rootDB, name);
138
144
  this.#containers.add(array);
139
145
  return array;
@@ -144,7 +150,7 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
144
150
  * @param name - Name of the singleton
145
151
  * @returns A new AztecSingleton
146
152
  */
147
- openSingleton<T>(name: string): AztecAsyncSingleton<T> {
153
+ openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
148
154
  const singleton = new IndexedDBAztecSingleton<T>(this.#rootDB, name);
149
155
  this.#containers.add(singleton);
150
156
  return singleton;
@@ -155,22 +161,27 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
155
161
  * @param callback - Function to execute in a transaction
156
162
  * @returns A promise that resolves to the return value of the callback
157
163
  */
158
- async transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
159
- const tx = this.#rootDB.transaction('data', 'readwrite');
160
- for (const container of this.#containers) {
161
- container.db = tx.store;
162
- }
163
- // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
164
- // By then, the tx would be closed
165
- const runningPromise = callback();
166
- // Wait for the transaction to finish
167
- await tx.done;
168
- for (const container of this.#containers) {
169
- container.db = undefined;
170
- }
171
- // Return the result of the callback.
172
- // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
173
- return await runningPromise;
164
+ transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
165
+ // We can only have one transaction at a time for the same store
166
+ // So we need to wait for the current one to finish
167
+ return this.#txQueue.put(async () => {
168
+ const tx = this.#rootDB.transaction('data', 'readwrite');
169
+ for (const container of this.#containers) {
170
+ container.db = tx.store;
171
+ }
172
+ // Avoid awaiting this promise so it doesn't get scheduled in the next microtask
173
+ // By then, the tx would be closed
174
+ const runningPromise = callback();
175
+ // Wait for the transaction to finish
176
+ await tx.done;
177
+ for (const container of this.#containers) {
178
+ container.db = undefined;
179
+ }
180
+
181
+ // Return the result of the callback.
182
+ // Tx is guaranteed to already be closed, so the await doesn't hurt anything here
183
+ return await runningPromise;
184
+ });
174
185
  }
175
186
 
176
187
  /**
@@ -181,17 +192,23 @@ export class AztecIndexedDBStore implements AztecAsyncKVStore {
181
192
  }
182
193
 
183
194
  /** Deletes this store and removes the database */
184
- delete() {
195
+ async delete() {
185
196
  this.#containers.clear();
197
+ await this.#txQueue.end();
186
198
  this.#rootDB.close();
187
- return deleteDB(this.#name);
199
+ await deleteDB(this.#name);
188
200
  }
189
201
 
190
202
  estimateSize(): Promise<StoreSize> {
191
- return Promise.resolve({ mappingSize: 0, actualSize: 0, numItems: 0 });
203
+ return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
192
204
  }
193
205
 
194
- close(): Promise<void> {
195
- return Promise.resolve();
206
+ async close(): Promise<void> {
207
+ await this.#txQueue.end();
208
+ this.#rootDB.close();
209
+ }
210
+
211
+ backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
212
+ throw new Error('Method not implemented.');
196
213
  }
197
214
  }
@@ -1,7 +1,9 @@
1
+ import type { Value } from './common.js';
2
+
1
3
  /**
2
4
  * An array backed by a persistent store. Can not have any holes in it.
3
5
  */
4
- interface BaseAztecArray<T> {
6
+ interface BaseAztecArray<T extends Value> {
5
7
  /**
6
8
  * Pushes values to the end of the array
7
9
  * @param vals - The values to push to the end of the array
@@ -27,7 +29,7 @@ interface BaseAztecArray<T> {
27
29
  /**
28
30
  * An array backed by a persistent store. Can not have any holes in it.
29
31
  */
30
- export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
32
+ export interface AztecAsyncArray<T extends Value> extends BaseAztecArray<T> {
31
33
  /**
32
34
  * The size of the array
33
35
  */
@@ -58,7 +60,7 @@ export interface AztecAsyncArray<T> extends BaseAztecArray<T> {
58
60
  [Symbol.asyncIterator](): AsyncIterableIterator<T>;
59
61
  }
60
62
 
61
- export interface AztecArray<T> extends BaseAztecArray<T> {
63
+ export interface AztecArray<T extends Value> extends BaseAztecArray<T> {
62
64
  /**
63
65
  * The size of the array
64
66
  */
@@ -1,12 +1,10 @@
1
- /**
2
- * The key type for use with the kv-store
3
- */
4
- export type Key = string | number | Array<string | number>;
1
+ /** The key type for use with the kv-store */
2
+ export type Key = string | number | Uint8Array | Array<string | number>;
5
3
 
6
- /**
7
- * A range of keys to iterate over.
8
- */
9
- export type Range<K extends Key = Key> = {
4
+ export type Value = NonNullable<any>;
5
+
6
+ /** A range of keys of arbitrary type. */
7
+ export type CustomRange<K> = {
10
8
  /** The key of the first item to include */
11
9
  start?: K;
12
10
  /** The key of the last item to include */
@@ -17,4 +15,17 @@ export type Range<K extends Key = Key> = {
17
15
  limit?: number;
18
16
  };
19
17
 
20
- export type StoreSize = { mappingSize: number; actualSize: number; numItems: number };
18
+ /** Maps a custom range into a range of valid key types to iterate over. */
19
+ export function mapRange<CK, K extends Key = Key>(range: CustomRange<CK>, mapFn: (key: CK) => K): Range<K> {
20
+ return {
21
+ start: range.start ? mapFn(range.start) : undefined,
22
+ end: range.end ? mapFn(range.end) : undefined,
23
+ reverse: range.reverse,
24
+ limit: range.limit,
25
+ };
26
+ }
27
+
28
+ /** A range of keys to iterate over. */
29
+ export type Range<K extends Key = Key> = CustomRange<K>;
30
+
31
+ export type StoreSize = { mappingSize: number; physicalFileSize: number; actualSize: number; numItems: number };
@@ -4,4 +4,6 @@ export * from './counter.js';
4
4
  export * from './singleton.js';
5
5
  export * from './store.js';
6
6
  export * from './set.js';
7
- export type { Range, StoreSize } from './common.js';
7
+ export * from './multi_map.js';
8
+ export { mapRange } from './common.js';
9
+ export type { CustomRange, Range, StoreSize } from './common.js';