@aztec/kv-store 0.0.1-commit.d431d1c → 0.0.1-commit.d58ff9d0

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (163) hide show
  1. package/README.md +10 -1
  2. package/dest/bench/shared_map_bench.d.ts +19 -0
  3. package/dest/bench/shared_map_bench.d.ts.map +1 -0
  4. package/dest/bench/shared_map_bench.js +91 -0
  5. package/dest/deprecated/indexeddb/array.d.ts +22 -0
  6. package/dest/deprecated/indexeddb/array.d.ts.map +1 -0
  7. package/dest/{indexeddb → deprecated/indexeddb}/array.js +18 -6
  8. package/dest/deprecated/indexeddb/index.d.ts +7 -0
  9. package/dest/deprecated/indexeddb/index.d.ts.map +1 -0
  10. package/dest/deprecated/indexeddb/index.js +8 -0
  11. package/dest/deprecated/indexeddb/map.d.ts +38 -0
  12. package/dest/deprecated/indexeddb/map.d.ts.map +1 -0
  13. package/dest/{indexeddb → deprecated/indexeddb}/map.js +23 -13
  14. package/dest/deprecated/indexeddb/multi_map.d.ts +14 -0
  15. package/dest/deprecated/indexeddb/multi_map.d.ts.map +1 -0
  16. package/dest/{indexeddb → deprecated/indexeddb}/multi_map.js +16 -1
  17. package/dest/deprecated/indexeddb/set.d.ts +17 -0
  18. package/dest/deprecated/indexeddb/set.d.ts.map +1 -0
  19. package/dest/deprecated/indexeddb/singleton.d.ts +17 -0
  20. package/dest/deprecated/indexeddb/singleton.d.ts.map +1 -0
  21. package/dest/{indexeddb → deprecated/indexeddb}/singleton.js +3 -1
  22. package/dest/deprecated/indexeddb/store.d.ts +98 -0
  23. package/dest/deprecated/indexeddb/store.d.ts.map +1 -0
  24. package/dest/{indexeddb → deprecated/indexeddb}/store.js +8 -4
  25. package/dest/interfaces/array_test_suite.d.ts +1 -1
  26. package/dest/interfaces/array_test_suite.d.ts.map +1 -1
  27. package/dest/interfaces/array_test_suite.js +33 -34
  28. package/dest/interfaces/index.d.ts +2 -2
  29. package/dest/interfaces/index.d.ts.map +1 -1
  30. package/dest/interfaces/map_test_suite.d.ts +1 -1
  31. package/dest/interfaces/map_test_suite.d.ts.map +1 -1
  32. package/dest/interfaces/map_test_suite.js +74 -29
  33. package/dest/interfaces/multi_map_test_suite.d.ts +1 -1
  34. package/dest/interfaces/multi_map_test_suite.d.ts.map +1 -1
  35. package/dest/interfaces/multi_map_test_suite.js +84 -60
  36. package/dest/interfaces/set_test_suite.d.ts +1 -1
  37. package/dest/interfaces/set_test_suite.d.ts.map +1 -1
  38. package/dest/interfaces/set_test_suite.js +13 -14
  39. package/dest/interfaces/singleton_test_suite.d.ts +1 -1
  40. package/dest/interfaces/singleton_test_suite.d.ts.map +1 -1
  41. package/dest/interfaces/singleton_test_suite.js +6 -7
  42. package/dest/interfaces/utils.d.ts +2 -1
  43. package/dest/interfaces/utils.d.ts.map +1 -1
  44. package/dest/interfaces/utils.js +2 -1
  45. package/dest/lmdb/index.d.ts +3 -3
  46. package/dest/lmdb/index.d.ts.map +1 -1
  47. package/dest/lmdb/index.js +4 -4
  48. package/dest/lmdb/store.d.ts +3 -3
  49. package/dest/lmdb/store.d.ts.map +1 -1
  50. package/dest/lmdb/store.js +12 -8
  51. package/dest/lmdb-v2/factory.d.ts +33 -7
  52. package/dest/lmdb-v2/factory.d.ts.map +1 -1
  53. package/dest/lmdb-v2/factory.js +72 -16
  54. package/dest/lmdb-v2/read_transaction.js +21 -19
  55. package/dest/lmdb-v2/store.d.ts +3 -3
  56. package/dest/lmdb-v2/store.d.ts.map +1 -1
  57. package/dest/lmdb-v2/store.js +5 -4
  58. package/dest/sqlite-opfs/array.d.ts +21 -0
  59. package/dest/sqlite-opfs/array.d.ts.map +1 -0
  60. package/dest/sqlite-opfs/array.js +128 -0
  61. package/dest/sqlite-opfs/errors.d.ts +54 -0
  62. package/dest/sqlite-opfs/errors.d.ts.map +1 -0
  63. package/dest/sqlite-opfs/errors.js +77 -0
  64. package/dest/sqlite-opfs/index.d.ts +14 -0
  65. package/dest/sqlite-opfs/index.d.ts.map +1 -0
  66. package/dest/sqlite-opfs/index.js +16 -0
  67. package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts +32 -0
  68. package/dest/sqlite-opfs/internal/ordered-binary-browser.d.ts.map +1 -0
  69. package/dest/sqlite-opfs/internal/ordered-binary-browser.js +448 -0
  70. package/dest/sqlite-opfs/manage.d.ts +22 -0
  71. package/dest/sqlite-opfs/manage.d.ts.map +1 -0
  72. package/dest/sqlite-opfs/manage.js +38 -0
  73. package/dest/sqlite-opfs/map.d.ts +35 -0
  74. package/dest/sqlite-opfs/map.d.ts.map +1 -0
  75. package/dest/sqlite-opfs/map.js +163 -0
  76. package/dest/sqlite-opfs/messages.d.ts +66 -0
  77. package/dest/sqlite-opfs/messages.d.ts.map +1 -0
  78. package/dest/sqlite-opfs/messages.js +5 -0
  79. package/dest/sqlite-opfs/multi_map.d.ts +16 -0
  80. package/dest/sqlite-opfs/multi_map.d.ts.map +1 -0
  81. package/dest/sqlite-opfs/multi_map.js +67 -0
  82. package/dest/sqlite-opfs/pool_integrity.d.ts +20 -0
  83. package/dest/sqlite-opfs/pool_integrity.d.ts.map +1 -0
  84. package/dest/sqlite-opfs/pool_integrity.js +282 -0
  85. package/dest/sqlite-opfs/pool_lock.d.ts +8 -0
  86. package/dest/sqlite-opfs/pool_lock.d.ts.map +1 -0
  87. package/dest/sqlite-opfs/pool_lock.js +47 -0
  88. package/dest/sqlite-opfs/set.d.ts +13 -0
  89. package/dest/sqlite-opfs/set.d.ts.map +1 -0
  90. package/dest/sqlite-opfs/set.js +19 -0
  91. package/dest/sqlite-opfs/singleton.d.ts +13 -0
  92. package/dest/sqlite-opfs/singleton.d.ts.map +1 -0
  93. package/dest/sqlite-opfs/singleton.js +48 -0
  94. package/dest/sqlite-opfs/store.d.ts +84 -0
  95. package/dest/sqlite-opfs/store.d.ts.map +1 -0
  96. package/dest/sqlite-opfs/store.js +317 -0
  97. package/dest/sqlite-opfs/worker.d.ts +2 -0
  98. package/dest/sqlite-opfs/worker.d.ts.map +1 -0
  99. package/dest/sqlite-opfs/worker.js +275 -0
  100. package/dest/stores/l2_tips_store.d.ts +7 -13
  101. package/dest/stores/l2_tips_store.d.ts.map +1 -1
  102. package/dest/stores/l2_tips_store.js +24 -43
  103. package/dest/utils.d.ts +9 -6
  104. package/dest/utils.d.ts.map +1 -1
  105. package/dest/utils.js +51 -16
  106. package/package.json +30 -26
  107. package/src/bench/shared_map_bench.ts +111 -0
  108. package/src/{indexeddb → deprecated/indexeddb}/array.ts +6 -6
  109. package/src/deprecated/indexeddb/index.ts +12 -0
  110. package/src/{indexeddb → deprecated/indexeddb}/map.ts +26 -13
  111. package/src/{indexeddb → deprecated/indexeddb}/multi_map.ts +17 -3
  112. package/src/{indexeddb → deprecated/indexeddb}/set.ts +2 -2
  113. package/src/{indexeddb → deprecated/indexeddb}/singleton.ts +3 -3
  114. package/src/{indexeddb → deprecated/indexeddb}/store.ts +23 -15
  115. package/src/interfaces/array_test_suite.ts +33 -35
  116. package/src/interfaces/index.ts +1 -1
  117. package/src/interfaces/map_test_suite.ts +56 -30
  118. package/src/interfaces/multi_map_test_suite.ts +88 -58
  119. package/src/interfaces/set_test_suite.ts +13 -15
  120. package/src/interfaces/singleton_test_suite.ts +6 -8
  121. package/src/interfaces/utils.ts +1 -0
  122. package/src/lmdb/index.ts +10 -5
  123. package/src/lmdb/store.ts +12 -8
  124. package/src/lmdb-v2/factory.ts +99 -19
  125. package/src/lmdb-v2/read_transaction.ts +23 -23
  126. package/src/lmdb-v2/store.ts +7 -4
  127. package/src/sqlite-opfs/array.ts +124 -0
  128. package/src/sqlite-opfs/errors.ts +94 -0
  129. package/src/sqlite-opfs/index.ts +27 -0
  130. package/src/sqlite-opfs/internal/ordered-binary-browser.js +465 -0
  131. package/src/sqlite-opfs/manage.ts +46 -0
  132. package/src/sqlite-opfs/map.ts +163 -0
  133. package/src/sqlite-opfs/messages.ts +39 -0
  134. package/src/sqlite-opfs/multi_map.ts +74 -0
  135. package/src/sqlite-opfs/pool_integrity.ts +327 -0
  136. package/src/sqlite-opfs/pool_lock.ts +62 -0
  137. package/src/sqlite-opfs/set.ts +29 -0
  138. package/src/sqlite-opfs/singleton.ts +48 -0
  139. package/src/sqlite-opfs/store.ts +330 -0
  140. package/src/sqlite-opfs/worker.ts +259 -0
  141. package/src/stores/l2_tips_store.ts +20 -41
  142. package/src/utils.ts +79 -21
  143. package/dest/config.d.ts +0 -17
  144. package/dest/config.d.ts.map +0 -1
  145. package/dest/config.js +0 -26
  146. package/dest/indexeddb/array.d.ts +0 -22
  147. package/dest/indexeddb/array.d.ts.map +0 -1
  148. package/dest/indexeddb/index.d.ts +0 -7
  149. package/dest/indexeddb/index.d.ts.map +0 -1
  150. package/dest/indexeddb/index.js +0 -19
  151. package/dest/indexeddb/map.d.ts +0 -32
  152. package/dest/indexeddb/map.d.ts.map +0 -1
  153. package/dest/indexeddb/multi_map.d.ts +0 -13
  154. package/dest/indexeddb/multi_map.d.ts.map +0 -1
  155. package/dest/indexeddb/set.d.ts +0 -17
  156. package/dest/indexeddb/set.d.ts.map +0 -1
  157. package/dest/indexeddb/singleton.d.ts +0 -17
  158. package/dest/indexeddb/singleton.d.ts.map +0 -1
  159. package/dest/indexeddb/store.d.ts +0 -96
  160. package/dest/indexeddb/store.d.ts.map +0 -1
  161. package/src/config.ts +0 -36
  162. package/src/indexeddb/index.ts +0 -29
  163. /package/dest/{indexeddb → deprecated/indexeddb}/set.js +0 -0
@@ -0,0 +1,330 @@
1
+ import type { Logger } from '@aztec/foundation/log';
2
+ import { SerialQueue } from '@aztec/foundation/queue';
3
+
4
+ import type { AztecAsyncArray } from '../interfaces/array.js';
5
+ import type { Key, StoreSize, Value } from '../interfaces/common.js';
6
+ import type { AztecAsyncCounter } from '../interfaces/counter.js';
7
+ import type { AztecAsyncMap } from '../interfaces/map.js';
8
+ import type { AztecAsyncMultiMap } from '../interfaces/multi_map.js';
9
+ import type { AztecAsyncSet } from '../interfaces/set.js';
10
+ import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
11
+ import type { AztecAsyncKVStore } from '../interfaces/store.js';
12
+ import { SQLiteOPFSAztecArray } from './array.js';
13
+ import { SqliteCorruptionError, SqliteEncryptionError, isCorruptionMessage } from './errors.js';
14
+ import { SQLiteOPFSAztecMap } from './map.js';
15
+ import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
16
+ import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
17
+ import { quarantineDuplicatePool } from './pool_integrity.js';
18
+ import { type PoolLockLease, acquirePoolLock, normalizePoolDirectory } from './pool_lock.js';
19
+ import { SQLiteOPFSAztecSet } from './set.js';
20
+ import { SQLiteOPFSAztecSingleton } from './singleton.js';
21
+
22
+ /**
23
+ * Main-thread handle for a SQLite database persisted to OPFS via the `opfs-sahpool`
24
+ * VFS. Owns a dedicated Web Worker (the SAH Pool VFS requires Worker context) and
25
+ * routes every SQL op through it via typed postMessage RPC.
26
+ *
27
+ * Transaction ordering is guaranteed by a `SerialQueue` on the main thread combined
28
+ * with an `#inTx` flag: outside a `transactionAsync` block, each op acquires the
29
+ * queue for its own auto-commit; inside a block, the outer call holds the queue and
30
+ * nested ops bypass it to avoid deadlock.
31
+ */
32
+ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
33
+ readonly #worker: Worker;
34
+ readonly #pending = new Map<number, { resolve: (r: WorkerResponse) => void; reject: (err: Error) => void }>();
35
+ readonly #txQueue = new SerialQueue();
36
+ readonly #name: string;
37
+ readonly #log: Logger;
38
+ #nextId = 0;
39
+ #inTx = false;
40
+ #closed = false;
41
+ #workerFailed = false;
42
+
43
+ private constructor(
44
+ worker: Worker,
45
+ name: string,
46
+ log: Logger,
47
+ public readonly isEphemeral: boolean,
48
+ private readonly poolLock?: PoolLockLease,
49
+ ) {
50
+ this.#worker = worker;
51
+ this.#name = name;
52
+ this.#log = log;
53
+ this.#worker.onmessage = (ev: MessageEvent<WorkerResponse>) => {
54
+ const { id } = ev.data;
55
+ const handler = this.#pending.get(id);
56
+ if (!handler) {
57
+ this.#log.warn(`SQLite worker: no pending handler for id ${id}`);
58
+ return;
59
+ }
60
+ this.#pending.delete(id);
61
+ handler.resolve(ev.data);
62
+ };
63
+ this.#worker.onerror = ev => {
64
+ this.#workerFailed = true;
65
+ this.#log.error(`SQLite worker crashed: ${ev.message}`);
66
+ this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
67
+ };
68
+ this.#txQueue.start();
69
+ }
70
+
71
+ /**
72
+ * Opens (or creates) a SQLite database stored in the OPFS SAH Pool. When `ephemeral`
73
+ * is true the database lives only in memory and is lost when the worker terminates.
74
+ * Pass `poolDirectory` to place the SAH Pool in a non-default OPFS subdirectory —
75
+ * required when multiple stores coexist in the same tab, because the SAH Pool holds
76
+ * an exclusive lock on its directory.
77
+ *
78
+ * Persistent stores hold an origin-wide Web Lock for the pool directory until close
79
+ * or delete. If another store instance already owns it, open fails immediately with
80
+ * `SqlitePoolBusyError`.
81
+ *
82
+ * Pass `encryptionKey` (exactly 32 bytes) to enable at-rest encryption via sqlite3mc's
83
+ * ChaCha20 page cipher. The key buffer is **transferred** to the worker — its
84
+ * ArrayBuffer detaches on the caller side after `postMessage`. This is intentional:
85
+ * the API encodes a one-key-one-owner invariant. A caller that wants to use the same
86
+ * key for multiple stores must explicitly clone it per call (e.g.
87
+ * `new Uint8Array(savedKey)`), making the duplication a visible, deliberate decision
88
+ * rather than a silent structured-clone operation. The default path (one `.open()`,
89
+ * one consumption of the key) leaves zero key bytes on the main thread after the call.
90
+ */
91
+ static async open(
92
+ log: Logger,
93
+ name?: string,
94
+ ephemeral: boolean = false,
95
+ poolDirectory?: string,
96
+ encryptionKey?: Uint8Array,
97
+ ): Promise<AztecSQLiteOPFSStore> {
98
+ if (encryptionKey !== undefined && encryptionKey.length !== 32) {
99
+ throw new SqliteEncryptionError(
100
+ 'invalid_key_length',
101
+ `encryptionKey must be 32 bytes (got ${encryptionKey.length})`,
102
+ );
103
+ }
104
+ if (encryptionKey !== undefined && ephemeral) {
105
+ throw new SqliteEncryptionError(
106
+ 'encryption_not_supported_for_ephemeral',
107
+ 'encryptionKey is not supported for ephemeral (:memory:) stores',
108
+ );
109
+ }
110
+ const dbName = name && !ephemeral ? name : `tmp-${globalThis.crypto.getRandomValues(new Uint8Array(8)).join('')}`;
111
+ log.debug(
112
+ `Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}${encryptionKey ? 'encrypted ' : ''}database ${dbName}`,
113
+ );
114
+ const effectivePoolDirectory = ephemeral ? undefined : normalizePoolDirectory(poolDirectory);
115
+ const poolLock = effectivePoolDirectory ? await acquirePoolLock(effectivePoolDirectory) : undefined;
116
+ let worker: Worker | undefined;
117
+ try {
118
+ if (effectivePoolDirectory) {
119
+ const quarantine = await quarantineDuplicatePool(effectivePoolDirectory);
120
+ if (quarantine) {
121
+ log.warn(`Quarantined SQLite-OPFS pool with duplicate logical file mappings`, quarantine);
122
+ }
123
+ }
124
+ worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
125
+ const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral, poolLock);
126
+ // Transfer (not clone) the key buffer to the worker so we don't leave a
127
+ // second copy on the main thread. Caveat: this detaches the caller's
128
+ // encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
129
+ const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
130
+ await store.#sendRequest(
131
+ { type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory: effectivePoolDirectory, encryptionKey },
132
+ transfer,
133
+ );
134
+ return store;
135
+ } catch (err) {
136
+ worker?.terminate();
137
+ await poolLock?.release();
138
+ throw err;
139
+ }
140
+ }
141
+
142
+ openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
143
+ return new SQLiteOPFSAztecMap<K, V>(this, name);
144
+ }
145
+
146
+ openSet<K extends Key>(name: string): AztecAsyncSet<K> {
147
+ return new SQLiteOPFSAztecSet<K>(this, name);
148
+ }
149
+
150
+ openMultiMap<K extends Key, V extends Value>(name: string): AztecAsyncMultiMap<K, V> {
151
+ return new SQLiteOPFSAztecMultiMap<K, V>(this, name);
152
+ }
153
+
154
+ openCounter<K extends Key>(_name: string): AztecAsyncCounter<K> {
155
+ throw new Error('Method not implemented.');
156
+ }
157
+
158
+ openArray<T extends Value>(name: string): AztecAsyncArray<T> {
159
+ return new SQLiteOPFSAztecArray<T>(this, name);
160
+ }
161
+
162
+ openSingleton<T extends Value>(name: string): AztecAsyncSingleton<T> {
163
+ return new SQLiteOPFSAztecSingleton<T>(this, name);
164
+ }
165
+
166
+ transactionAsync<T>(callback: () => Promise<T>): Promise<T> {
167
+ // Nested calls join the outer transaction — SQLite does not support nested BEGIN,
168
+ // and re-acquiring the SerialQueue while the outer call holds it would deadlock.
169
+ // Errors in the nested callback propagate to the outer catch, which rolls back the
170
+ // whole thing (the standard "nested tx = savepoint-free join" semantic).
171
+ if (this.#inTx) {
172
+ return callback();
173
+ }
174
+ return this.#txQueue.put(async () => {
175
+ this.#inTx = true;
176
+ await this.#sendRequest({ type: 'begin', id: this.#allocId() });
177
+ try {
178
+ const result = await callback();
179
+ await this.#sendRequest({ type: 'commit', id: this.#allocId() });
180
+ return result;
181
+ } catch (err) {
182
+ await this.#sendRequest({ type: 'rollback', id: this.#allocId() }).catch(rollbackErr =>
183
+ this.#log.warn(`SQLite ROLLBACK failed: ${rollbackErr instanceof Error ? rollbackErr.message : rollbackErr}`),
184
+ );
185
+ throw err;
186
+ } finally {
187
+ this.#inTx = false;
188
+ }
189
+ });
190
+ }
191
+
192
+ async clear(): Promise<void> {
193
+ await this.runAsync('DELETE FROM data');
194
+ }
195
+
196
+ async delete(): Promise<void> {
197
+ if (this.#closed) {
198
+ return;
199
+ }
200
+ this.#closed = true;
201
+ try {
202
+ await this.#txQueue.end();
203
+ await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
204
+ this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
205
+ );
206
+ } finally {
207
+ this.#worker.terminate();
208
+ this.#rejectPending('SQLite store deleted');
209
+ await this.poolLock?.release();
210
+ }
211
+ }
212
+
213
+ /**
214
+ * Placeholder — returns zeros to mirror the IndexedDB backend. SQLite exposes real
215
+ * numbers cheaply via `PRAGMA page_count` / `page_size` / `freelist_count` and
216
+ * `SELECT COUNT(*) FROM data`, which would populate `physicalFileSize`, `actualSize`,
217
+ * and `numItems` meaningfully (`mappingSize` stays 0 — it's an LMDB mmap concept).
218
+ * Upgrade when any caller actually consumes these values; all current consumers
219
+ * tolerate zeros.
220
+ */
221
+ estimateSize(): Promise<StoreSize> {
222
+ return Promise.resolve({ mappingSize: 0, physicalFileSize: 0, actualSize: 0, numItems: 0 });
223
+ }
224
+
225
+ async close(): Promise<void> {
226
+ if (this.#closed) {
227
+ return;
228
+ }
229
+ this.#closed = true;
230
+ try {
231
+ await this.#txQueue.end();
232
+ await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
233
+ } finally {
234
+ this.#worker.terminate();
235
+ this.#rejectPending('SQLite store closed');
236
+ await this.poolLock?.release();
237
+ }
238
+ }
239
+
240
+ backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
241
+ throw new Error('Method not implemented.');
242
+ }
243
+
244
+ /**
245
+ * Returns a raw SQLite image (bytes suitable for writing as a `.sqlite` file and
246
+ * opening in any SQLite tool). Works only for non-ephemeral DBs because the OPFS
247
+ * SAH Pool has to be initialized. Useful for inspection/debugging.
248
+ */
249
+ async exportDb(): Promise<Uint8Array> {
250
+ const resp = await this.#sendRequest({ type: 'export', id: this.#allocId() });
251
+ if (!('bytes' in resp) || !resp.bytes) {
252
+ throw new Error('exportDb: worker returned no bytes');
253
+ }
254
+ return resp.bytes;
255
+ }
256
+
257
+ /**
258
+ * Runs a write statement (INSERT/UPDATE/DELETE/DDL). If called inside a
259
+ * `transactionAsync` block, bypasses the queue; otherwise acquires it so the
260
+ * op runs in its own auto-commit.
261
+ */
262
+ runAsync(sql: string, bind?: SqlValue[]): Promise<{ changes: number }> {
263
+ const send = () =>
264
+ this.#sendRequest({ type: 'run', id: this.#allocId(), sql, bind }).then(r => ({
265
+ changes: 'changes' in r ? (r.changes ?? 0) : 0,
266
+ }));
267
+ return this.#inTx ? send() : this.#txQueue.put(send);
268
+ }
269
+
270
+ /** Runs a SELECT statement and returns rows in array row-mode. */
271
+ allAsync(sql: string, bind?: SqlValue[]): Promise<ResultRow[]> {
272
+ const send = () =>
273
+ this.#sendRequest({ type: 'all', id: this.#allocId(), sql, bind }).then(r => ('rows' in r ? (r.rows ?? []) : []));
274
+ return this.#inTx ? send() : this.#txQueue.put(send);
275
+ }
276
+
277
+ #allocId(): number {
278
+ return ++this.#nextId;
279
+ }
280
+
281
+ /**
282
+ * Reject any in-flight requests with `reason`, so callers awaiting a response to a
283
+ * request sent to a now-terminated worker don't hang forever. Called from
284
+ * close()/delete() and from the worker.onerror handler.
285
+ */
286
+ #rejectPending(reason: string): void {
287
+ if (this.#pending.size === 0) {
288
+ return;
289
+ }
290
+ const err = new Error(reason);
291
+ for (const { reject } of this.#pending.values()) {
292
+ reject(err);
293
+ }
294
+ this.#pending.clear();
295
+ }
296
+
297
+ #sendRequest(req: WorkerRequest, transfer?: Transferable[]): Promise<WorkerResponse> {
298
+ if (this.#workerFailed) {
299
+ return Promise.reject(new Error('SQLite worker has crashed'));
300
+ }
301
+ return new Promise<WorkerResponse>((resolve, reject) => {
302
+ this.#pending.set(req.id, {
303
+ resolve: resp => {
304
+ if (resp.type === 'err') {
305
+ // Re-hydrate typed errors so consumers can pattern-match on
306
+ // `instanceof`. Encryption is tagged on the wire (some cases are
307
+ // pre-flight throws with no message to match); corruption is a
308
+ // single unambiguous message, so we classify it here rather than
309
+ // adding a redundant wire field. Everything else stays a plain Error.
310
+ if (resp.encryptionCode !== undefined) {
311
+ reject(new SqliteEncryptionError(resp.encryptionCode, resp.message));
312
+ } else if (isCorruptionMessage(resp.message)) {
313
+ reject(new SqliteCorruptionError(resp.message));
314
+ } else {
315
+ reject(new Error(resp.message));
316
+ }
317
+ } else {
318
+ resolve(resp);
319
+ }
320
+ },
321
+ reject,
322
+ });
323
+ if (transfer && transfer.length > 0) {
324
+ this.#worker.postMessage(req, transfer);
325
+ } else {
326
+ this.#worker.postMessage(req);
327
+ }
328
+ });
329
+ }
330
+ }
@@ -0,0 +1,259 @@
1
+ /// <reference lib="webworker" />
2
+ import sqlite3InitModule, { type Database, type SAHPoolUtil, type Sqlite3Static } from '@aztec/sqlite3mc-wasm';
3
+
4
+ import { SqliteEncryptionError, type SqliteEncryptionErrorCode, isDecryptFailureMessage } from './errors.js';
5
+ import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
6
+ import { DEFAULT_SAH_POOL_DIRECTORY } from './pool_lock.js';
7
+
8
+ const SCHEMA_SQL = `
9
+ CREATE TABLE IF NOT EXISTS data (
10
+ slot TEXT NOT NULL PRIMARY KEY,
11
+ container TEXT NOT NULL,
12
+ key BLOB NOT NULL,
13
+ key_count INTEGER NOT NULL,
14
+ hash TEXT NOT NULL,
15
+ value BLOB
16
+ ) WITHOUT ROWID;
17
+
18
+ CREATE INDEX IF NOT EXISTS idx_container_key ON data(container, key);
19
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_count ON data(container, key, key_count);
20
+ CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
21
+ `;
22
+
23
+ const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
24
+ const MC_SAH_POOL_VFS_NAME = `multipleciphers-${SAH_POOL_VFS_NAME}`;
25
+
26
+ let sqlite3: Sqlite3Static | undefined;
27
+ let pool: SAHPoolUtil | undefined;
28
+ let db: Database | undefined;
29
+ let dbPath: string | undefined;
30
+
31
+ async function ensurePool(directory: string): Promise<SAHPoolUtil> {
32
+ const s = (sqlite3 ??= await sqlite3InitModule());
33
+ if (!pool) {
34
+ pool = await s.installOpfsSAHPoolVfs({
35
+ name: SAH_POOL_VFS_NAME,
36
+ directory,
37
+ initialCapacity: 8,
38
+ });
39
+ // Register a sqlite3mc-wrapped VFS pointing at our SAH Pool VFS.
40
+ // Encrypted DBs must be opened through this wrapper so sqlite3mc can
41
+ // intercept file I/O; plain DBs continue using the SAH Pool VFS directly.
42
+ // The wrapper name is `multipleciphers-<underlying>`.
43
+ (s.capi as unknown as { sqlite3mc_vfs_create(name: string, makeDefault: number): number }).sqlite3mc_vfs_create(
44
+ SAH_POOL_VFS_NAME,
45
+ 0,
46
+ );
47
+ }
48
+ return pool!;
49
+ }
50
+
51
+ /**
52
+ * Applies sqlite3mc's ChaCha20 page cipher using a pre-derived 32-byte key.
53
+ * The PRAGMAs must run before any schema DDL so sqlite3mc can decrypt existing
54
+ * pages and encrypt new ones. Zeroes the caller-held key array after the PRAGMA
55
+ * completes to minimize residency of the raw key bytes outside sqlite3mc's heap.
56
+ */
57
+ function applyEncryptionKey(conn: Database, key: Uint8Array): void {
58
+ const hex = Array.from(key, b => b.toString(16).padStart(2, '0')).join('');
59
+ conn.exec(`PRAGMA cipher = 'chacha20'`);
60
+ conn.exec(`PRAGMA key = "x'${hex}'"`);
61
+ key.fill(0);
62
+ }
63
+
64
+ async function handleInit(
65
+ dbName: string,
66
+ ephemeral: boolean,
67
+ directory?: string,
68
+ encryptionKey?: Uint8Array,
69
+ ): Promise<void> {
70
+ const s = (sqlite3 ??= await sqlite3InitModule());
71
+ if (encryptionKey !== undefined && ephemeral) {
72
+ throw new SqliteEncryptionError(
73
+ 'encryption_not_supported_for_ephemeral',
74
+ 'encryptionKey is not supported for ephemeral (:memory:) stores',
75
+ );
76
+ }
77
+ if (ephemeral) {
78
+ db = new s.oo1.DB(':memory:', 'c');
79
+ } else {
80
+ const activePool = await ensurePool(directory ?? DEFAULT_SAH_POOL_DIRECTORY);
81
+ dbPath = normalizeDbPath(dbName);
82
+ if (encryptionKey !== undefined) {
83
+ const conn = new s.oo1.DB({ filename: dbPath, flags: 'c', vfs: MC_SAH_POOL_VFS_NAME });
84
+ db = conn;
85
+ applyEncryptionKey(conn, encryptionKey);
86
+ } else {
87
+ db = new activePool.OpfsSAHPoolDb(dbPath);
88
+ }
89
+ }
90
+ runSql(SCHEMA_SQL);
91
+ }
92
+
93
+ function handleClose(): void {
94
+ try {
95
+ db?.close();
96
+ } finally {
97
+ db = undefined;
98
+ dbPath = undefined;
99
+ releasePool();
100
+ }
101
+ }
102
+
103
+ /**
104
+ * Releases the SAH pool's OPFS sync access handles before the terminal RPC is acked. Worker
105
+ * termination releases them only asynchronously, so without this a caller that deletes or reopens
106
+ * the store directory right after close()/delete() resolves races Chromium's cleanup
107
+ * (NoModificationAllowedError from removeEntry, or a hang installing a new pool on the directory).
108
+ * pauseVfs releases the handles without touching file contents; the worker is terminated right
109
+ * after, so the pool is never resumed.
110
+ */
111
+ function releasePool(): void {
112
+ pool?.pauseVfs();
113
+ pool = undefined;
114
+ }
115
+
116
+ async function handleExport(): Promise<Uint8Array> {
117
+ if (!db || !dbPath) {
118
+ throw new Error('SQLite worker: no database open to export');
119
+ }
120
+ if (!pool) {
121
+ throw new Error('SQLite worker: no SAH Pool available (ephemeral DBs cannot be exported)');
122
+ }
123
+ return await pool.exportFile(dbPath);
124
+ }
125
+
126
+ function handleDeleteDb(dbName: string): void {
127
+ const path = normalizeDbPath(dbName);
128
+ if (db && dbPath === path) {
129
+ db.close();
130
+ db = undefined;
131
+ dbPath = undefined;
132
+ }
133
+ // Ephemeral :memory: DBs never back a file — skip installing a pool just to unlink
134
+ // nothing. installOpfsSAHPoolVfs acquires an exclusive lock on the OPFS SAH
135
+ // directory, and under heavy test churn that can contend with workers from
136
+ // previous tests whose OPFS handles Chromium hasn't yet released, hanging the RPC
137
+ // and then the whole test run.
138
+ if (!pool) {
139
+ return;
140
+ }
141
+ try {
142
+ pool.unlink(path);
143
+ } catch {
144
+ // File may not exist; ignore.
145
+ }
146
+ // Guarded because pauseVfs refuses (SQLITE_MISUSE) while any file is open through the VFS.
147
+ if (!db) {
148
+ releasePool();
149
+ }
150
+ }
151
+
152
+ function requireDb(): Database {
153
+ if (!db) {
154
+ throw new Error('SQLite worker: no database open');
155
+ }
156
+ return db;
157
+ }
158
+
159
+ function runSql(sql: string, bind?: SqlValue[]): { changes: number } {
160
+ const conn = requireDb();
161
+ conn.exec({ sql, bind });
162
+ return { changes: conn.changes() };
163
+ }
164
+
165
+ function selectAll(sql: string, bind?: SqlValue[]): ResultRow[] {
166
+ const conn = requireDb();
167
+ const rows: ResultRow[] = [];
168
+ conn.exec({ sql, bind, rowMode: 'array', resultRows: rows });
169
+ return rows;
170
+ }
171
+
172
+ function normalizeDbPath(dbName: string): string {
173
+ return dbName.startsWith('/') ? dbName : `/${dbName}`;
174
+ }
175
+
176
+ function respond(msg: WorkerResponse): void {
177
+ (self as DedicatedWorkerGlobalScope).postMessage(msg);
178
+ }
179
+
180
+ (self as DedicatedWorkerGlobalScope).onmessage = async (ev: MessageEvent<WorkerRequest>) => {
181
+ const req = ev.data;
182
+ try {
183
+ switch (req.type) {
184
+ case 'init':
185
+ await handleInit(req.dbName, req.ephemeral, req.poolDirectory, req.encryptionKey);
186
+ return respond({ type: 'ok', id: req.id });
187
+ case 'close':
188
+ handleClose();
189
+ return respond({ type: 'ok', id: req.id });
190
+ case 'deleteDb':
191
+ handleDeleteDb(req.dbName);
192
+ return respond({ type: 'ok', id: req.id });
193
+ case 'run': {
194
+ const { changes } = runSql(req.sql, req.bind);
195
+ return respond({ type: 'ok', id: req.id, changes });
196
+ }
197
+ case 'all': {
198
+ const rows = selectAll(req.sql, req.bind);
199
+ return respond({ type: 'ok', id: req.id, rows });
200
+ }
201
+ case 'export': {
202
+ const bytes = await handleExport();
203
+ return respond({ type: 'ok', id: req.id, bytes });
204
+ }
205
+ case 'begin':
206
+ runSql('BEGIN');
207
+ return respond({ type: 'ok', id: req.id });
208
+ case 'commit':
209
+ runSql('COMMIT');
210
+ return respond({ type: 'ok', id: req.id });
211
+ case 'rollback':
212
+ runSql('ROLLBACK');
213
+ return respond({ type: 'ok', id: req.id });
214
+ default: {
215
+ const _exhaustive: never = req;
216
+ throw new Error(`Unknown request: ${JSON.stringify(_exhaustive)}`);
217
+ }
218
+ }
219
+ } catch (err) {
220
+ const message = err instanceof Error ? err.message : String(err);
221
+ const encryptionCode = detectEncryptionCode(req, err, message);
222
+ if (req.type === 'init') {
223
+ try {
224
+ handleClose();
225
+ } catch {
226
+ // The main thread terminates this worker after a failed init, which releases any remaining OPFS handles.
227
+ }
228
+ }
229
+ respond({ type: 'err', id: req.id, message, encryptionCode });
230
+ }
231
+ };
232
+
233
+ /**
234
+ * Maps a thrown error during request handling to a typed encryption code, so the
235
+ * main thread can re-hydrate it as a {@link SqliteEncryptionError}. Returns
236
+ * `undefined` for non-encryption errors (preserves the existing untyped path).
237
+ *
238
+ * Two sources:
239
+ * - The error was already a `SqliteEncryptionError` (pre-flight throws inside
240
+ * this worker — e.g. ephemeral + encryptionKey). Forward its code as-is.
241
+ * - The error came from SQLite/sqlite3mc with a known decrypt-failure message
242
+ * during an `init` request. Both "wrong key supplied" and "no key supplied
243
+ * to an encrypted DB" surface as SQLITE_NOTADB ("file is not a database") —
244
+ * we don't constrain on `req.encryptionKey` because the no-key-on-encrypted-DB
245
+ * case is exactly when the caller most needs the typed signal.
246
+ */
247
+ function detectEncryptionCode(
248
+ req: WorkerRequest,
249
+ err: unknown,
250
+ message: string,
251
+ ): SqliteEncryptionErrorCode | undefined {
252
+ if (err instanceof SqliteEncryptionError) {
253
+ return err.code;
254
+ }
255
+ if (req.type === 'init' && isDecryptFailureMessage(message)) {
256
+ return 'decrypt_failed';
257
+ }
258
+ return undefined;
259
+ }
@@ -1,31 +1,30 @@
1
1
  import { BlockNumber, CheckpointNumber } from '@aztec/foundation/branded-types';
2
- import { type L2BlockTag, L2TipsStoreBase } from '@aztec/stdlib/block';
3
- import { PublishedCheckpoint } from '@aztec/stdlib/checkpoint';
2
+ import { type BlockHash, type CheckpointId, type L2BlockTag, L2TipsStoreBase } from '@aztec/stdlib/block';
4
3
 
5
4
  import type { AztecAsyncMap } from '../interfaces/map.js';
6
5
  import type { AztecAsyncKVStore } from '../interfaces/store.js';
7
6
 
7
+ /** Serialized form of a per-tip checkpoint id stored in the KV store. */
8
+ type StoredCheckpointId = { number: number; hash: string };
9
+
8
10
  /**
9
11
  * Persistent implementation of L2 tips store backed by a KV store.
10
12
  * Used by nodes that need to persist chain state across restarts.
11
13
  */
12
14
  export class L2TipsKVStore extends L2TipsStoreBase {
13
15
  private readonly l2TipsStore: AztecAsyncMap<L2BlockTag, BlockNumber>;
16
+ private readonly l2TipCheckpointsStore: AztecAsyncMap<L2BlockTag, StoredCheckpointId>;
14
17
  private readonly l2BlockHashesStore: AztecAsyncMap<BlockNumber, string>;
15
- private readonly l2BlockNumberToCheckpointNumberStore: AztecAsyncMap<BlockNumber, CheckpointNumber>;
16
- private readonly l2CheckpointStore: AztecAsyncMap<CheckpointNumber, Buffer>;
17
18
 
18
19
  constructor(
19
20
  private store: AztecAsyncKVStore,
20
21
  namespace: string,
22
+ initialBlockHash: BlockHash,
21
23
  ) {
22
- super();
24
+ super(initialBlockHash);
23
25
  this.l2TipsStore = store.openMap([namespace, 'l2_tips'].join('_'));
26
+ this.l2TipCheckpointsStore = store.openMap([namespace, 'l2_tip_checkpoints'].join('_'));
24
27
  this.l2BlockHashesStore = store.openMap([namespace, 'l2_block_hashes'].join('_'));
25
- this.l2BlockNumberToCheckpointNumberStore = store.openMap(
26
- [namespace, 'l2_block_number_to_checkpoint_number'].join('_'),
27
- );
28
- this.l2CheckpointStore = store.openMap([namespace, 'l2_checkpoint_store'].join('_'));
29
28
  }
30
29
 
31
30
  protected getTip(tag: L2BlockTag): Promise<BlockNumber | undefined> {
@@ -36,6 +35,18 @@ export class L2TipsKVStore extends L2TipsStoreBase {
36
35
  return this.l2TipsStore.set(tag, blockNumber);
37
36
  }
38
37
 
38
+ protected async getTipCheckpoint(tag: L2BlockTag): Promise<CheckpointId | undefined> {
39
+ const stored = await this.l2TipCheckpointsStore.getAsync(tag);
40
+ if (stored === undefined) {
41
+ return undefined;
42
+ }
43
+ return { number: CheckpointNumber(stored.number), hash: stored.hash };
44
+ }
45
+
46
+ protected setTipCheckpoint(tag: L2BlockTag, checkpoint: CheckpointId): Promise<void> {
47
+ return this.l2TipCheckpointsStore.set(tag, { number: checkpoint.number, hash: checkpoint.hash });
48
+ }
49
+
39
50
  protected getStoredBlockHash(blockNumber: BlockNumber): Promise<string | undefined> {
40
51
  return this.l2BlockHashesStore.getAsync(blockNumber);
41
52
  }
@@ -50,38 +61,6 @@ export class L2TipsKVStore extends L2TipsStoreBase {
50
61
  }
51
62
  }
52
63
 
53
- protected getCheckpointNumberForBlock(blockNumber: BlockNumber): Promise<CheckpointNumber | undefined> {
54
- return this.l2BlockNumberToCheckpointNumberStore.getAsync(blockNumber);
55
- }
56
-
57
- protected setCheckpointNumberForBlock(blockNumber: BlockNumber, checkpointNumber: CheckpointNumber): Promise<void> {
58
- return this.l2BlockNumberToCheckpointNumberStore.set(blockNumber, checkpointNumber);
59
- }
60
-
61
- protected async deleteBlockToCheckpointBefore(blockNumber: BlockNumber): Promise<void> {
62
- for await (const key of this.l2BlockNumberToCheckpointNumberStore.keysAsync({ end: blockNumber })) {
63
- await this.l2BlockNumberToCheckpointNumberStore.delete(key);
64
- }
65
- }
66
-
67
- protected async getCheckpoint(checkpointNumber: CheckpointNumber): Promise<PublishedCheckpoint | undefined> {
68
- const buffer = await this.l2CheckpointStore.getAsync(checkpointNumber);
69
- if (!buffer) {
70
- return undefined;
71
- }
72
- return PublishedCheckpoint.fromBuffer(buffer);
73
- }
74
-
75
- protected saveCheckpointData(checkpoint: PublishedCheckpoint): Promise<void> {
76
- return this.l2CheckpointStore.set(checkpoint.checkpoint.number, checkpoint.toBuffer());
77
- }
78
-
79
- protected async deleteCheckpointsBefore(checkpointNumber: CheckpointNumber): Promise<void> {
80
- for await (const key of this.l2CheckpointStore.keysAsync({ end: checkpointNumber })) {
81
- await this.l2CheckpointStore.delete(key);
82
- }
83
- }
84
-
85
64
  protected runInTransaction<T>(fn: () => Promise<T>): Promise<T> {
86
65
  return this.store.transactionAsync(fn);
87
66
  }