@aztec/kv-store 0.0.1-commit.a5db02d → 0.0.1-commit.aa0c64f
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dest/sqlite-opfs/errors.d.ts +28 -1
- package/dest/sqlite-opfs/errors.d.ts.map +1 -1
- package/dest/sqlite-opfs/errors.js +43 -0
- package/dest/sqlite-opfs/index.d.ts +3 -3
- package/dest/sqlite-opfs/index.d.ts.map +1 -1
- package/dest/sqlite-opfs/index.js +2 -2
- package/dest/sqlite-opfs/manage.d.ts +4 -2
- package/dest/sqlite-opfs/manage.d.ts.map +1 -1
- package/dest/sqlite-opfs/manage.js +11 -4
- package/dest/sqlite-opfs/pool_integrity.d.ts +20 -0
- package/dest/sqlite-opfs/pool_integrity.d.ts.map +1 -0
- package/dest/sqlite-opfs/pool_integrity.js +282 -0
- package/dest/sqlite-opfs/pool_lock.d.ts +8 -0
- package/dest/sqlite-opfs/pool_lock.d.ts.map +1 -0
- package/dest/sqlite-opfs/pool_lock.js +47 -0
- package/dest/sqlite-opfs/store.d.ts +6 -1
- package/dest/sqlite-opfs/store.d.ts.map +1 -1
- package/dest/sqlite-opfs/store.js +69 -33
- package/dest/sqlite-opfs/worker.js +17 -6
- package/package.json +7 -7
- package/src/sqlite-opfs/errors.ts +50 -0
- package/src/sqlite-opfs/index.ts +7 -2
- package/src/sqlite-opfs/manage.ts +13 -3
- package/src/sqlite-opfs/pool_integrity.ts +327 -0
- package/src/sqlite-opfs/pool_lock.ts +62 -0
- package/src/sqlite-opfs/store.ts +57 -23
- package/src/sqlite-opfs/worker.ts +17 -6
|
@@ -0,0 +1,327 @@
|
|
|
1
|
+
import { normalizePoolDirectory } from './pool_lock.js';
|
|
2
|
+
|
|
3
|
+
// The constants below mirror the opaque-file header format of the pinned opfs-sahpool VFS
|
|
4
|
+
// (`yarn-project/sqlite3mc-wasm/vendor/jswasm/sqlite3.mjs`, class `OpfsSAHPool`). The pool names its files randomly
|
|
5
|
+
// under `.opaque/` and prepends a header that maps each one back to its logical SQLite path:
|
|
6
|
+
//
|
|
7
|
+
// bytes [0, 512) logical path, NUL-terminated UTF-8 (HEADER_MAX_PATH_SIZE)
|
|
8
|
+
// bytes [512, 516) SQLite open-flags of the file, big-endian uint32 (HEADER_FLAGS_SIZE)
|
|
9
|
+
// bytes [516, 524) digest over the preceding 516 bytes, two uint32 words (HEADER_DIGEST_SIZE)
|
|
10
|
+
//
|
|
11
|
+
// Database content starts at byte 4096 (the pool's SECTOR_SIZE). We re-read the header ourselves rather than asking
|
|
12
|
+
// the VFS because the upstream pool "repairs" anything it cannot validate by disassociating the file — destroying
|
|
13
|
+
// exactly the evidence this module exists to quarantine. The browser regression test writes a real pool through the
|
|
14
|
+
// pinned VFS before duplicating an opaque file, so a vendor upgrade that changes the layout breaks detection loudly
|
|
15
|
+
// rather than silently.
|
|
16
|
+
const OPAQUE_DIRECTORY = '.opaque';
|
|
17
|
+
const HEADER_MAX_PATH_SIZE = 512;
|
|
18
|
+
const HEADER_FLAGS_SIZE = 4;
|
|
19
|
+
const HEADER_DIGEST_SIZE = 8;
|
|
20
|
+
const HEADER_CORPUS_SIZE = HEADER_MAX_PATH_SIZE + HEADER_FLAGS_SIZE;
|
|
21
|
+
const HEADER_SIZE = HEADER_CORPUS_SIZE + HEADER_DIGEST_SIZE;
|
|
22
|
+
|
|
23
|
+
// Standard SQLite open-flag bit values (sqlite3.h). Restated as literals because the header stores them numerically
|
|
24
|
+
// and this module runs on the main thread, without the sqlite3 WASM bundle that defines `capi.SQLITE_OPEN_*`.
|
|
25
|
+
const SQLITE_OPEN_DELETEONCLOSE = 0x00000008;
|
|
26
|
+
const SQLITE_OPEN_MEMORY = 0x00000080;
|
|
27
|
+
const SQLITE_OPEN_MAIN_DB = 0x00000100;
|
|
28
|
+
const SQLITE_OPEN_MAIN_JOURNAL = 0x00000800;
|
|
29
|
+
const SQLITE_OPEN_SUPER_JOURNAL = 0x00004000;
|
|
30
|
+
const SQLITE_OPEN_WAL = 0x00080000;
|
|
31
|
+
|
|
32
|
+
// A live association must name one of the file types the pool persists; transient types (temp DBs, statement
|
|
33
|
+
// journals) never survive in a valid header.
|
|
34
|
+
const PERSISTENT_FILE_TYPES =
|
|
35
|
+
SQLITE_OPEN_MAIN_DB | SQLITE_OPEN_MAIN_JOURNAL | SQLITE_OPEN_SUPER_JOURNAL | SQLITE_OPEN_WAL;
|
|
36
|
+
|
|
37
|
+
// The upstream VFS repurposes SQLITE_OPEN_MEMORY — meaningless for a file that exists on disk — as a header version
|
|
38
|
+
// marker: headers written with it set carry a real digest, while legacy headers leave it unset and store all-zero
|
|
39
|
+
// digest words.
|
|
40
|
+
const FLAG_COMPUTE_DIGEST_V2 = SQLITE_OPEN_MEMORY;
|
|
41
|
+
|
|
42
|
+
export const OPFS_QUARANTINE_ROOT_DIRECTORY = '.aztec-sqlite-quarantine';
|
|
43
|
+
|
|
44
|
+
export interface DuplicatePoolAssociation {
|
|
45
|
+
logicalPath: string;
|
|
46
|
+
opaqueFileNames: string[];
|
|
47
|
+
}
|
|
48
|
+
|
|
49
|
+
export interface PoolQuarantineMetadata {
|
|
50
|
+
formatVersion: 1;
|
|
51
|
+
originalPoolDirectory: string;
|
|
52
|
+
quarantinedAt: string;
|
|
53
|
+
duplicateAssociations: DuplicatePoolAssociation[];
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export interface PoolQuarantineResult extends PoolQuarantineMetadata {
|
|
57
|
+
quarantineDirectory: string;
|
|
58
|
+
}
|
|
59
|
+
|
|
60
|
+
/**
|
|
61
|
+
* Detects duplicate SAH logical-file associations and, if found, copies the complete pool into quarantine before
|
|
62
|
+
* removing the original. The caller must hold the pool's exclusive Web Lock for the whole operation.
|
|
63
|
+
*/
|
|
64
|
+
export async function quarantineDuplicatePool(poolDirectory: string): Promise<PoolQuarantineResult | undefined> {
|
|
65
|
+
poolDirectory = normalizePoolDirectory(poolDirectory);
|
|
66
|
+
const root = await navigator.storage.getDirectory();
|
|
67
|
+
const source = await getDirectory(root, poolDirectory);
|
|
68
|
+
if (!source) {
|
|
69
|
+
return undefined;
|
|
70
|
+
}
|
|
71
|
+
const opaque = await getChildDirectory(source, OPAQUE_DIRECTORY);
|
|
72
|
+
if (!opaque) {
|
|
73
|
+
return undefined;
|
|
74
|
+
}
|
|
75
|
+
|
|
76
|
+
const duplicateAssociations = await findDuplicateAssociations(opaque);
|
|
77
|
+
if (duplicateAssociations.length === 0) {
|
|
78
|
+
return undefined;
|
|
79
|
+
}
|
|
80
|
+
|
|
81
|
+
const quarantineRoot = await root.getDirectoryHandle(OPFS_QUARANTINE_ROOT_DIRECTORY, { create: true });
|
|
82
|
+
const quarantineName = createQuarantineName();
|
|
83
|
+
const destination = await quarantineRoot.getDirectoryHandle(quarantineName, { create: true });
|
|
84
|
+
const metadata: PoolQuarantineMetadata = {
|
|
85
|
+
formatVersion: 1,
|
|
86
|
+
originalPoolDirectory: poolDirectory,
|
|
87
|
+
quarantinedAt: new Date().toISOString(),
|
|
88
|
+
duplicateAssociations,
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
let quarantineComplete = false;
|
|
92
|
+
try {
|
|
93
|
+
await copyDirectory(source, destination);
|
|
94
|
+
await verifyDirectoryCopy(source, destination);
|
|
95
|
+
await writeJson(destination, 'quarantine.json', metadata);
|
|
96
|
+
quarantineComplete = true;
|
|
97
|
+
await removeDirectory(root, poolDirectory);
|
|
98
|
+
} catch (err) {
|
|
99
|
+
if (!quarantineComplete) {
|
|
100
|
+
await quarantineRoot.removeEntry(quarantineName, { recursive: true }).catch(() => {});
|
|
101
|
+
}
|
|
102
|
+
throw err;
|
|
103
|
+
}
|
|
104
|
+
|
|
105
|
+
return {
|
|
106
|
+
...metadata,
|
|
107
|
+
quarantineDirectory: `${OPFS_QUARANTINE_ROOT_DIRECTORY}/${quarantineName}`,
|
|
108
|
+
};
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
async function findDuplicateAssociations(opaque: FileSystemDirectoryHandle): Promise<DuplicatePoolAssociation[]> {
|
|
112
|
+
const associations = new Map<string, string[]>();
|
|
113
|
+
for await (const [opaqueName, handle] of opaque.entries()) {
|
|
114
|
+
if (handle.kind !== 'file') {
|
|
115
|
+
continue;
|
|
116
|
+
}
|
|
117
|
+
const logicalPath = await readAssociatedPath(handle as FileSystemFileHandle);
|
|
118
|
+
if (logicalPath) {
|
|
119
|
+
const names = associations.get(logicalPath) ?? [];
|
|
120
|
+
names.push(opaqueName);
|
|
121
|
+
associations.set(logicalPath, names);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
return [...associations.entries()]
|
|
125
|
+
.filter(([, names]) => names.length > 1)
|
|
126
|
+
.map(([logicalPath, opaqueFileNames]) => ({ logicalPath, opaqueFileNames: opaqueFileNames.sort() }))
|
|
127
|
+
.sort((a, b) => a.logicalPath.localeCompare(b.logicalPath));
|
|
128
|
+
}
|
|
129
|
+
|
|
130
|
+
/**
|
|
131
|
+
* Returns the logical SQLite path an opaque SAH file is associated with, or undefined if the file is not a live,
|
|
132
|
+
* valid association. Applies the same checks as the vendored pool's `getAssociatedPath`: a non-empty NUL-terminated
|
|
133
|
+
* path, open-flags naming a persistent file type without DELETEONCLOSE, and a matching header digest. Files failing
|
|
134
|
+
* any check are the pool's free-list or garbage entries — the VFS itself would disassociate them on open — so they
|
|
135
|
+
* cannot participate in a duplicate mapping.
|
|
136
|
+
*/
|
|
137
|
+
async function readAssociatedPath(handle: FileSystemFileHandle): Promise<string | undefined> {
|
|
138
|
+
const file = await handle.getFile();
|
|
139
|
+
if (file.size < HEADER_SIZE) {
|
|
140
|
+
return undefined;
|
|
141
|
+
}
|
|
142
|
+
const header = new Uint8Array(await file.slice(0, HEADER_SIZE).arrayBuffer());
|
|
143
|
+
const pathEnd = header.subarray(0, HEADER_MAX_PATH_SIZE).indexOf(0);
|
|
144
|
+
if (pathEnd <= 0) {
|
|
145
|
+
return undefined;
|
|
146
|
+
}
|
|
147
|
+
|
|
148
|
+
const dataView = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
149
|
+
const flags = dataView.getUint32(HEADER_MAX_PATH_SIZE);
|
|
150
|
+
if ((flags & SQLITE_OPEN_DELETEONCLOSE) !== 0 || (flags & PERSISTENT_FILE_TYPES) === 0) {
|
|
151
|
+
return undefined;
|
|
152
|
+
}
|
|
153
|
+
if (!hasValidDigest(header, flags)) {
|
|
154
|
+
return undefined;
|
|
155
|
+
}
|
|
156
|
+
|
|
157
|
+
try {
|
|
158
|
+
return new TextDecoder('utf-8', { fatal: true }).decode(header.subarray(0, pathEnd));
|
|
159
|
+
} catch {
|
|
160
|
+
return undefined;
|
|
161
|
+
}
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Byte-for-byte port of the vendored pool's `computeDigest`, checked against the digest words stored in the header.
|
|
166
|
+
*/
|
|
167
|
+
function hasValidDigest(header: Uint8Array, flags: number): boolean {
|
|
168
|
+
let expected0 = 0;
|
|
169
|
+
let expected1 = 0;
|
|
170
|
+
if ((flags & FLAG_COMPUTE_DIGEST_V2) !== 0) {
|
|
171
|
+
// These seeds (0xdeadbeef, 0x41c6ce57) and odd multipliers (2654435761, 104729) are the upstream author's choices
|
|
172
|
+
// (a cyrb53-hash variant) and carry no meaning here beyond having to match the vendored implementation bit for
|
|
173
|
+
// bit.
|
|
174
|
+
expected0 = 0xdeadbeef;
|
|
175
|
+
expected1 = 0x41c6ce57;
|
|
176
|
+
for (const value of header.subarray(0, HEADER_CORPUS_SIZE)) {
|
|
177
|
+
expected0 = Math.imul(expected0 ^ value, 2654435761);
|
|
178
|
+
expected1 = Math.imul(expected1 ^ value, 104729);
|
|
179
|
+
}
|
|
180
|
+
expected0 >>>= 0;
|
|
181
|
+
expected1 >>>= 0;
|
|
182
|
+
}
|
|
183
|
+
const dataView = new DataView(header.buffer, header.byteOffset, header.byteLength);
|
|
184
|
+
return (
|
|
185
|
+
dataView.getUint32(HEADER_CORPUS_SIZE, true) === expected0 &&
|
|
186
|
+
dataView.getUint32(HEADER_CORPUS_SIZE + 4, true) === expected1
|
|
187
|
+
);
|
|
188
|
+
}
|
|
189
|
+
|
|
190
|
+
async function getDirectory(
|
|
191
|
+
root: FileSystemDirectoryHandle,
|
|
192
|
+
path: string,
|
|
193
|
+
): Promise<FileSystemDirectoryHandle | undefined> {
|
|
194
|
+
let current = root;
|
|
195
|
+
try {
|
|
196
|
+
for (const segment of path.split('/')) {
|
|
197
|
+
current = await current.getDirectoryHandle(segment);
|
|
198
|
+
}
|
|
199
|
+
return current;
|
|
200
|
+
} catch (err) {
|
|
201
|
+
if (err instanceof DOMException && err.name === 'NotFoundError') {
|
|
202
|
+
return undefined;
|
|
203
|
+
}
|
|
204
|
+
throw err;
|
|
205
|
+
}
|
|
206
|
+
}
|
|
207
|
+
|
|
208
|
+
async function getChildDirectory(
|
|
209
|
+
parent: FileSystemDirectoryHandle,
|
|
210
|
+
name: string,
|
|
211
|
+
): Promise<FileSystemDirectoryHandle | undefined> {
|
|
212
|
+
try {
|
|
213
|
+
return await parent.getDirectoryHandle(name);
|
|
214
|
+
} catch (err) {
|
|
215
|
+
if (err instanceof DOMException && err.name === 'NotFoundError') {
|
|
216
|
+
return undefined;
|
|
217
|
+
}
|
|
218
|
+
throw err;
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
async function removeDirectory(root: FileSystemDirectoryHandle, path: string): Promise<void> {
|
|
223
|
+
const segments = path.split('/');
|
|
224
|
+
const name = segments.pop()!;
|
|
225
|
+
let parent = root;
|
|
226
|
+
for (const segment of segments) {
|
|
227
|
+
parent = await parent.getDirectoryHandle(segment);
|
|
228
|
+
}
|
|
229
|
+
await parent.removeEntry(name, { recursive: true });
|
|
230
|
+
}
|
|
231
|
+
|
|
232
|
+
async function copyDirectory(source: FileSystemDirectoryHandle, destination: FileSystemDirectoryHandle): Promise<void> {
|
|
233
|
+
for await (const [name, handle] of source.entries()) {
|
|
234
|
+
if (handle.kind === 'directory') {
|
|
235
|
+
const childDestination = await destination.getDirectoryHandle(name, { create: true });
|
|
236
|
+
await copyDirectory(handle as FileSystemDirectoryHandle, childDestination);
|
|
237
|
+
continue;
|
|
238
|
+
}
|
|
239
|
+
|
|
240
|
+
const sourceFile = await (handle as FileSystemFileHandle).getFile();
|
|
241
|
+
const destinationHandle = await destination.getFileHandle(name, { create: true });
|
|
242
|
+
const writable = await destinationHandle.createWritable();
|
|
243
|
+
try {
|
|
244
|
+
await writable.write(sourceFile);
|
|
245
|
+
await writable.close();
|
|
246
|
+
} catch (err) {
|
|
247
|
+
await writable.abort().catch(() => {});
|
|
248
|
+
throw err;
|
|
249
|
+
}
|
|
250
|
+
}
|
|
251
|
+
}
|
|
252
|
+
|
|
253
|
+
async function verifyDirectoryCopy(
|
|
254
|
+
source: FileSystemDirectoryHandle,
|
|
255
|
+
destination: FileSystemDirectoryHandle,
|
|
256
|
+
): Promise<void> {
|
|
257
|
+
const sourceEntries = await getSortedEntries(source);
|
|
258
|
+
const destinationEntries = await getSortedEntries(destination);
|
|
259
|
+
if (
|
|
260
|
+
sourceEntries.length !== destinationEntries.length ||
|
|
261
|
+
sourceEntries.some(([name, handle], index) => {
|
|
262
|
+
const destinationEntry = destinationEntries[index];
|
|
263
|
+
return name !== destinationEntry[0] || handle.kind !== destinationEntry[1].kind;
|
|
264
|
+
})
|
|
265
|
+
) {
|
|
266
|
+
throw new Error('Failed to verify quarantined OPFS directory structure');
|
|
267
|
+
}
|
|
268
|
+
|
|
269
|
+
for (let i = 0; i < sourceEntries.length; i++) {
|
|
270
|
+
const [, sourceHandle] = sourceEntries[i];
|
|
271
|
+
const [, destinationHandle] = destinationEntries[i];
|
|
272
|
+
if (sourceHandle.kind === 'directory') {
|
|
273
|
+
await verifyDirectoryCopy(
|
|
274
|
+
sourceHandle as FileSystemDirectoryHandle,
|
|
275
|
+
destinationHandle as FileSystemDirectoryHandle,
|
|
276
|
+
);
|
|
277
|
+
} else {
|
|
278
|
+
await verifyFilesEqual(sourceHandle as FileSystemFileHandle, destinationHandle as FileSystemFileHandle);
|
|
279
|
+
}
|
|
280
|
+
}
|
|
281
|
+
}
|
|
282
|
+
|
|
283
|
+
async function getSortedEntries(directory: FileSystemDirectoryHandle): Promise<[string, FileSystemHandle][]> {
|
|
284
|
+
const entries: [string, FileSystemHandle][] = [];
|
|
285
|
+
for await (const entry of directory.entries()) {
|
|
286
|
+
entries.push(entry);
|
|
287
|
+
}
|
|
288
|
+
return entries.sort(([a], [b]) => a.localeCompare(b));
|
|
289
|
+
}
|
|
290
|
+
|
|
291
|
+
async function verifyFilesEqual(sourceHandle: FileSystemFileHandle, destinationHandle: FileSystemFileHandle) {
|
|
292
|
+
const source = await sourceHandle.getFile();
|
|
293
|
+
const destination = await destinationHandle.getFile();
|
|
294
|
+
if (source.size !== destination.size) {
|
|
295
|
+
throw new Error(`Failed to verify quarantined OPFS file "${source.name}"`);
|
|
296
|
+
}
|
|
297
|
+
|
|
298
|
+
const chunkSize = 1024 * 1024;
|
|
299
|
+
for (let offset = 0; offset < source.size; offset += chunkSize) {
|
|
300
|
+
const [sourceChunk, destinationChunk] = await Promise.all([
|
|
301
|
+
source.slice(offset, offset + chunkSize).arrayBuffer(),
|
|
302
|
+
destination.slice(offset, offset + chunkSize).arrayBuffer(),
|
|
303
|
+
]);
|
|
304
|
+
const sourceBytes = new Uint8Array(sourceChunk);
|
|
305
|
+
const destinationBytes = new Uint8Array(destinationChunk);
|
|
306
|
+
if (sourceBytes.some((value, index) => value !== destinationBytes[index])) {
|
|
307
|
+
throw new Error(`Failed to verify quarantined OPFS file "${source.name}"`);
|
|
308
|
+
}
|
|
309
|
+
}
|
|
310
|
+
}
|
|
311
|
+
|
|
312
|
+
async function writeJson(directory: FileSystemDirectoryHandle, name: string, value: unknown): Promise<void> {
|
|
313
|
+
const handle = await directory.getFileHandle(name, { create: true });
|
|
314
|
+
const writable = await handle.createWritable();
|
|
315
|
+
try {
|
|
316
|
+
await writable.write(JSON.stringify(value, undefined, 2));
|
|
317
|
+
await writable.close();
|
|
318
|
+
} catch (err) {
|
|
319
|
+
await writable.abort().catch(() => {});
|
|
320
|
+
throw err;
|
|
321
|
+
}
|
|
322
|
+
}
|
|
323
|
+
|
|
324
|
+
function createQuarantineName(): string {
|
|
325
|
+
const random = globalThis.crypto.getRandomValues(new Uint8Array(8));
|
|
326
|
+
return `${Date.now()}-${[...random].map(byte => byte.toString(16).padStart(2, '0')).join('')}`;
|
|
327
|
+
}
|
|
@@ -0,0 +1,62 @@
|
|
|
1
|
+
import { SqlitePoolBusyError, SqliteWebLocksUnavailableError } from './errors.js';
|
|
2
|
+
|
|
3
|
+
export const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
|
|
4
|
+
|
|
5
|
+
const WEB_LOCK_PREFIX = 'aztec.sqlite-opfs.pool:';
|
|
6
|
+
|
|
7
|
+
export interface PoolLockLease {
|
|
8
|
+
release(): Promise<void>;
|
|
9
|
+
}
|
|
10
|
+
|
|
11
|
+
export function normalizePoolDirectory(poolDirectory?: string): string {
|
|
12
|
+
const path = poolDirectory
|
|
13
|
+
?.split('/')
|
|
14
|
+
.filter(segment => segment.length > 0)
|
|
15
|
+
.join('/');
|
|
16
|
+
return path || DEFAULT_SAH_POOL_DIRECTORY;
|
|
17
|
+
}
|
|
18
|
+
|
|
19
|
+
export async function acquirePoolLock(poolDirectory?: string): Promise<PoolLockLease> {
|
|
20
|
+
if (!navigator.locks) {
|
|
21
|
+
throw new SqliteWebLocksUnavailableError();
|
|
22
|
+
}
|
|
23
|
+
const normalizedDirectory = normalizePoolDirectory(poolDirectory);
|
|
24
|
+
const acquired = Promise.withResolvers<Lock | null>();
|
|
25
|
+
const hold = Promise.withResolvers<void>();
|
|
26
|
+
const request = navigator.locks.request(
|
|
27
|
+
`${WEB_LOCK_PREFIX}${normalizedDirectory}`,
|
|
28
|
+
{ mode: 'exclusive', ifAvailable: true },
|
|
29
|
+
async lock => {
|
|
30
|
+
acquired.resolve(lock);
|
|
31
|
+
if (lock) {
|
|
32
|
+
await hold.promise;
|
|
33
|
+
}
|
|
34
|
+
},
|
|
35
|
+
);
|
|
36
|
+
request.catch(acquired.reject);
|
|
37
|
+
|
|
38
|
+
if (!(await acquired.promise)) {
|
|
39
|
+
await request;
|
|
40
|
+
throw new SqlitePoolBusyError(normalizedDirectory);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
let released = false;
|
|
44
|
+
return {
|
|
45
|
+
release: async () => {
|
|
46
|
+
if (!released) {
|
|
47
|
+
released = true;
|
|
48
|
+
hold.resolve();
|
|
49
|
+
}
|
|
50
|
+
await request;
|
|
51
|
+
},
|
|
52
|
+
};
|
|
53
|
+
}
|
|
54
|
+
|
|
55
|
+
export async function withPoolLock<T>(poolDirectory: string, callback: () => Promise<T>): Promise<T> {
|
|
56
|
+
const lease = await acquirePoolLock(poolDirectory);
|
|
57
|
+
try {
|
|
58
|
+
return await callback();
|
|
59
|
+
} finally {
|
|
60
|
+
await lease.release();
|
|
61
|
+
}
|
|
62
|
+
}
|
package/src/sqlite-opfs/store.ts
CHANGED
|
@@ -10,10 +10,12 @@ import type { AztecAsyncSet } from '../interfaces/set.js';
|
|
|
10
10
|
import type { AztecAsyncSingleton } from '../interfaces/singleton.js';
|
|
11
11
|
import type { AztecAsyncKVStore } from '../interfaces/store.js';
|
|
12
12
|
import { SQLiteOPFSAztecArray } from './array.js';
|
|
13
|
-
import { SqliteEncryptionError } from './errors.js';
|
|
13
|
+
import { SqliteCorruptionError, SqliteEncryptionError, isCorruptionMessage } from './errors.js';
|
|
14
14
|
import { SQLiteOPFSAztecMap } from './map.js';
|
|
15
15
|
import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
|
|
16
16
|
import { SQLiteOPFSAztecMultiMap } from './multi_map.js';
|
|
17
|
+
import { quarantineDuplicatePool } from './pool_integrity.js';
|
|
18
|
+
import { type PoolLockLease, acquirePoolLock, normalizePoolDirectory } from './pool_lock.js';
|
|
17
19
|
import { SQLiteOPFSAztecSet } from './set.js';
|
|
18
20
|
import { SQLiteOPFSAztecSingleton } from './singleton.js';
|
|
19
21
|
|
|
@@ -36,12 +38,14 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
36
38
|
#nextId = 0;
|
|
37
39
|
#inTx = false;
|
|
38
40
|
#closed = false;
|
|
41
|
+
#workerFailed = false;
|
|
39
42
|
|
|
40
43
|
private constructor(
|
|
41
44
|
worker: Worker,
|
|
42
45
|
name: string,
|
|
43
46
|
log: Logger,
|
|
44
47
|
public readonly isEphemeral: boolean,
|
|
48
|
+
private readonly poolLock?: PoolLockLease,
|
|
45
49
|
) {
|
|
46
50
|
this.#worker = worker;
|
|
47
51
|
this.#name = name;
|
|
@@ -57,6 +61,7 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
57
61
|
handler.resolve(ev.data);
|
|
58
62
|
};
|
|
59
63
|
this.#worker.onerror = ev => {
|
|
64
|
+
this.#workerFailed = true;
|
|
60
65
|
this.#log.error(`SQLite worker crashed: ${ev.message}`);
|
|
61
66
|
this.#rejectPending(`SQLite worker crashed: ${ev.message}`);
|
|
62
67
|
};
|
|
@@ -70,6 +75,10 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
70
75
|
* required when multiple stores coexist in the same tab, because the SAH Pool holds
|
|
71
76
|
* an exclusive lock on its directory.
|
|
72
77
|
*
|
|
78
|
+
* Persistent stores hold an origin-wide Web Lock for the pool directory until close
|
|
79
|
+
* or delete. If another store instance already owns it, open fails immediately with
|
|
80
|
+
* `SqlitePoolBusyError`.
|
|
81
|
+
*
|
|
73
82
|
* Pass `encryptionKey` (exactly 32 bytes) to enable at-rest encryption via sqlite3mc's
|
|
74
83
|
* ChaCha20 page cipher. The key buffer is **transferred** to the worker — its
|
|
75
84
|
* ArrayBuffer detaches on the caller side after `postMessage`. This is intentional:
|
|
@@ -102,22 +111,32 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
102
111
|
log.debug(
|
|
103
112
|
`Opening SQLite-OPFS ${ephemeral ? 'ephemeral ' : ''}${encryptionKey ? 'encrypted ' : ''}database ${dbName}`,
|
|
104
113
|
);
|
|
105
|
-
const
|
|
106
|
-
const
|
|
107
|
-
|
|
108
|
-
// second copy on the main thread. Caveat: this detaches the caller's
|
|
109
|
-
// encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
|
|
110
|
-
const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
|
|
114
|
+
const effectivePoolDirectory = ephemeral ? undefined : normalizePoolDirectory(poolDirectory);
|
|
115
|
+
const poolLock = effectivePoolDirectory ? await acquirePoolLock(effectivePoolDirectory) : undefined;
|
|
116
|
+
let worker: Worker | undefined;
|
|
111
117
|
try {
|
|
118
|
+
if (effectivePoolDirectory) {
|
|
119
|
+
const quarantine = await quarantineDuplicatePool(effectivePoolDirectory);
|
|
120
|
+
if (quarantine) {
|
|
121
|
+
log.warn(`Quarantined SQLite-OPFS pool with duplicate logical file mappings`, quarantine);
|
|
122
|
+
}
|
|
123
|
+
}
|
|
124
|
+
worker = new Worker(new URL('./worker.js', import.meta.url), { type: 'module' });
|
|
125
|
+
const store = new AztecSQLiteOPFSStore(worker, dbName, log, ephemeral, poolLock);
|
|
126
|
+
// Transfer (not clone) the key buffer to the worker so we don't leave a
|
|
127
|
+
// second copy on the main thread. Caveat: this detaches the caller's
|
|
128
|
+
// encryptionKey.buffer — subsequent reads from the same Uint8Array are empty.
|
|
129
|
+
const transfer = encryptionKey ? [encryptionKey.buffer as ArrayBuffer] : undefined;
|
|
112
130
|
await store.#sendRequest(
|
|
113
|
-
{ type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory, encryptionKey },
|
|
131
|
+
{ type: 'init', id: store.#allocId(), dbName, ephemeral, poolDirectory: effectivePoolDirectory, encryptionKey },
|
|
114
132
|
transfer,
|
|
115
133
|
);
|
|
134
|
+
return store;
|
|
116
135
|
} catch (err) {
|
|
117
|
-
worker
|
|
136
|
+
worker?.terminate();
|
|
137
|
+
await poolLock?.release();
|
|
118
138
|
throw err;
|
|
119
139
|
}
|
|
120
|
-
return store;
|
|
121
140
|
}
|
|
122
141
|
|
|
123
142
|
openMap<K extends Key, V extends Value>(name: string): AztecAsyncMap<K, V> {
|
|
@@ -179,12 +198,16 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
179
198
|
return;
|
|
180
199
|
}
|
|
181
200
|
this.#closed = true;
|
|
182
|
-
|
|
183
|
-
|
|
184
|
-
this.#
|
|
185
|
-
|
|
186
|
-
|
|
187
|
-
|
|
201
|
+
try {
|
|
202
|
+
await this.#txQueue.end();
|
|
203
|
+
await this.#sendRequest({ type: 'deleteDb', id: this.#allocId(), dbName: this.#name }).catch(err =>
|
|
204
|
+
this.#log.warn(`SQLite deleteDb failed: ${err instanceof Error ? err.message : err}`),
|
|
205
|
+
);
|
|
206
|
+
} finally {
|
|
207
|
+
this.#worker.terminate();
|
|
208
|
+
this.#rejectPending('SQLite store deleted');
|
|
209
|
+
await this.poolLock?.release();
|
|
210
|
+
}
|
|
188
211
|
}
|
|
189
212
|
|
|
190
213
|
/**
|
|
@@ -204,10 +227,14 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
204
227
|
return;
|
|
205
228
|
}
|
|
206
229
|
this.#closed = true;
|
|
207
|
-
|
|
208
|
-
|
|
209
|
-
|
|
210
|
-
|
|
230
|
+
try {
|
|
231
|
+
await this.#txQueue.end();
|
|
232
|
+
await this.#sendRequest({ type: 'close', id: this.#allocId() }).catch(() => {});
|
|
233
|
+
} finally {
|
|
234
|
+
this.#worker.terminate();
|
|
235
|
+
this.#rejectPending('SQLite store closed');
|
|
236
|
+
await this.poolLock?.release();
|
|
237
|
+
}
|
|
211
238
|
}
|
|
212
239
|
|
|
213
240
|
backupTo(_dstPath: string, _compact?: boolean): Promise<void> {
|
|
@@ -268,15 +295,22 @@ export class AztecSQLiteOPFSStore implements AztecAsyncKVStore {
|
|
|
268
295
|
}
|
|
269
296
|
|
|
270
297
|
#sendRequest(req: WorkerRequest, transfer?: Transferable[]): Promise<WorkerResponse> {
|
|
298
|
+
if (this.#workerFailed) {
|
|
299
|
+
return Promise.reject(new Error('SQLite worker has crashed'));
|
|
300
|
+
}
|
|
271
301
|
return new Promise<WorkerResponse>((resolve, reject) => {
|
|
272
302
|
this.#pending.set(req.id, {
|
|
273
303
|
resolve: resp => {
|
|
274
304
|
if (resp.type === 'err') {
|
|
275
|
-
// Re-hydrate
|
|
276
|
-
//
|
|
277
|
-
//
|
|
305
|
+
// Re-hydrate typed errors so consumers can pattern-match on
|
|
306
|
+
// `instanceof`. Encryption is tagged on the wire (some cases are
|
|
307
|
+
// pre-flight throws with no message to match); corruption is a
|
|
308
|
+
// single unambiguous message, so we classify it here rather than
|
|
309
|
+
// adding a redundant wire field. Everything else stays a plain Error.
|
|
278
310
|
if (resp.encryptionCode !== undefined) {
|
|
279
311
|
reject(new SqliteEncryptionError(resp.encryptionCode, resp.message));
|
|
312
|
+
} else if (isCorruptionMessage(resp.message)) {
|
|
313
|
+
reject(new SqliteCorruptionError(resp.message));
|
|
280
314
|
} else {
|
|
281
315
|
reject(new Error(resp.message));
|
|
282
316
|
}
|
|
@@ -3,6 +3,7 @@ import sqlite3InitModule, { type Database, type SAHPoolUtil, type Sqlite3Static
|
|
|
3
3
|
|
|
4
4
|
import { SqliteEncryptionError, type SqliteEncryptionErrorCode, isDecryptFailureMessage } from './errors.js';
|
|
5
5
|
import type { ResultRow, SqlValue, WorkerRequest, WorkerResponse } from './messages.js';
|
|
6
|
+
import { DEFAULT_SAH_POOL_DIRECTORY } from './pool_lock.js';
|
|
6
7
|
|
|
7
8
|
const SCHEMA_SQL = `
|
|
8
9
|
CREATE TABLE IF NOT EXISTS data (
|
|
@@ -19,7 +20,6 @@ const SCHEMA_SQL = `
|
|
|
19
20
|
CREATE UNIQUE INDEX IF NOT EXISTS idx_container_key_hash ON data(container, key, hash);
|
|
20
21
|
`;
|
|
21
22
|
|
|
22
|
-
const DEFAULT_SAH_POOL_DIRECTORY = '.aztec-kv';
|
|
23
23
|
const SAH_POOL_VFS_NAME = 'aztec-kv-opfs';
|
|
24
24
|
const MC_SAH_POOL_VFS_NAME = `multipleciphers-${SAH_POOL_VFS_NAME}`;
|
|
25
25
|
|
|
@@ -91,10 +91,13 @@ async function handleInit(
|
|
|
91
91
|
}
|
|
92
92
|
|
|
93
93
|
function handleClose(): void {
|
|
94
|
-
|
|
95
|
-
|
|
96
|
-
|
|
97
|
-
|
|
94
|
+
try {
|
|
95
|
+
db?.close();
|
|
96
|
+
} finally {
|
|
97
|
+
db = undefined;
|
|
98
|
+
dbPath = undefined;
|
|
99
|
+
releasePool();
|
|
100
|
+
}
|
|
98
101
|
}
|
|
99
102
|
|
|
100
103
|
/**
|
|
@@ -215,7 +218,15 @@ function respond(msg: WorkerResponse): void {
|
|
|
215
218
|
}
|
|
216
219
|
} catch (err) {
|
|
217
220
|
const message = err instanceof Error ? err.message : String(err);
|
|
218
|
-
|
|
221
|
+
const encryptionCode = detectEncryptionCode(req, err, message);
|
|
222
|
+
if (req.type === 'init') {
|
|
223
|
+
try {
|
|
224
|
+
handleClose();
|
|
225
|
+
} catch {
|
|
226
|
+
// The main thread terminates this worker after a failed init, which releases any remaining OPFS handles.
|
|
227
|
+
}
|
|
228
|
+
}
|
|
229
|
+
respond({ type: 'err', id: req.id, message, encryptionCode });
|
|
219
230
|
}
|
|
220
231
|
};
|
|
221
232
|
|