@rivetkit/sqlite-vfs 2.0.4-rc.1

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
package/src/vfs.ts ADDED
@@ -0,0 +1,1131 @@
1
+ /**
2
+ * SQLite raw database with KV storage backend
3
+ *
4
+ * This module provides a SQLite API that uses a KV-backed VFS
5
+ * for storage. Each SqliteVfs instance is independent and can be
6
+ * used concurrently with other instances.
7
+ *
8
+ * Keep this VFS on direct VFS.Base callbacks for minimal wrapper overhead.
9
+ * Use @rivetkit/sqlite/src/FacadeVFS.js as the reference implementation for
10
+ * callback ABI and pointer/data conversion behavior.
11
+ * This implementation is optimized for single-writer semantics because each
12
+ * actor owns one SQLite database.
13
+ * SQLite invokes this VFS with byte-range file operations. This VFS maps those
14
+ * ranges onto fixed-size KV chunks keyed by file tag and chunk index.
15
+ * We intentionally rely on SQLite's pager cache for hot page reuse and do not
16
+ * add a second cache in this VFS. This avoids duplicate cache invalidation
17
+ * logic and keeps memory usage predictable for each actor.
18
+ */
19
+
20
+ import * as VFS from "@rivetkit/sqlite/src/VFS.js";
21
+ import {
22
+ Factory,
23
+ SQLITE_OPEN_CREATE,
24
+ SQLITE_OPEN_READWRITE,
25
+ SQLITE_ROW,
26
+ } from "@rivetkit/sqlite";
27
+ import { readFileSync } from "node:fs";
28
+ import { createRequire } from "node:module";
29
+ import {
30
+ CHUNK_SIZE,
31
+ FILE_TAG_JOURNAL,
32
+ FILE_TAG_MAIN,
33
+ FILE_TAG_SHM,
34
+ FILE_TAG_WAL,
35
+ getChunkKey,
36
+ getMetaKey,
37
+ type SqliteFileTag,
38
+ } from "./kv";
39
+ import {
40
+ FILE_META_VERSIONED,
41
+ CURRENT_VERSION,
42
+ } from "../schemas/file-meta/versioned";
43
+ import type { FileMeta } from "../schemas/file-meta/mod";
44
+ import type { KvVfsOptions } from "./types";
45
+
46
+ type SqliteEsmFactory = (config?: { wasmBinary?: ArrayBuffer | Uint8Array }) => Promise<unknown>;
47
+ type SQLite3Api = ReturnType<typeof Factory>;
48
+ type SqliteBindings = Parameters<SQLite3Api["bind_collection"]>[1];
49
+ type SqliteVfsRegistration = Parameters<SQLite3Api["vfs_register"]>[0];
50
+
51
+ interface SQLiteModule {
52
+ UTF8ToString: (ptr: number) => string;
53
+ HEAPU8: Uint8Array;
54
+ }
55
+
56
+ const TEXT_ENCODER = new TextEncoder();
57
+ const TEXT_DECODER = new TextDecoder();
58
+ const SQLITE_MAX_PATHNAME_BYTES = 64;
59
+
60
+ // Chunk keys encode the chunk index in 32 bits, so a file can span at most
61
+ // 2^32 chunks. At 4 KiB/chunk this yields a hard limit of 16 TiB.
62
+ const UINT32_SIZE = 0x100000000;
63
+ const MAX_CHUNK_INDEX = 0xffffffff;
64
+ const MAX_FILE_SIZE_BYTES = (MAX_CHUNK_INDEX + 1) * CHUNK_SIZE;
65
+ const MAX_FILE_SIZE_HI32 = Math.floor(MAX_FILE_SIZE_BYTES / UINT32_SIZE);
66
+ const MAX_FILE_SIZE_LO32 = MAX_FILE_SIZE_BYTES % UINT32_SIZE;
67
+
68
+ // libvfs captures this async/sync mask at registration time. Any VFS callback
69
+ // that returns a Promise must be listed here so SQLite uses async relays.
70
+ const SQLITE_ASYNC_METHODS = new Set([
71
+ "xOpen",
72
+ "xClose",
73
+ "xRead",
74
+ "xWrite",
75
+ "xTruncate",
76
+ "xSync",
77
+ "xFileSize",
78
+ "xDelete",
79
+ "xAccess",
80
+ ]);
81
+
82
+ interface LoadedSqliteRuntime {
83
+ sqlite3: SQLite3Api;
84
+ module: SQLiteModule;
85
+ }
86
+
87
+ function isSqliteEsmFactory(value: unknown): value is SqliteEsmFactory {
88
+ return typeof value === "function";
89
+ }
90
+
91
+ function isSQLiteModule(value: unknown): value is SQLiteModule {
92
+ if (!value || typeof value !== "object") {
93
+ return false;
94
+ }
95
+ const candidate = value as {
96
+ UTF8ToString?: unknown;
97
+ HEAPU8?: unknown;
98
+ };
99
+ return (
100
+ typeof candidate.UTF8ToString === "function" &&
101
+ candidate.HEAPU8 instanceof Uint8Array
102
+ );
103
+ }
104
+
105
+
106
+ /**
107
+ * Lazily load and instantiate the async SQLite module for this VFS instance.
108
+ * We do this on first open so actors that do not use SQLite do not pay module
109
+ * parse and wasm initialization cost at startup, and we pass wasmBinary
110
+ * explicitly so this works consistently in both ESM and CJS bundles.
111
+ */
112
+ async function loadSqliteRuntime(): Promise<LoadedSqliteRuntime> {
113
+ // Keep the module specifier assembled at runtime so TypeScript declaration
114
+ // generation does not try to typecheck this deep dist import path.
115
+ // Uses Array.join() instead of string concatenation to prevent esbuild/tsup
116
+ // from constant-folding the expression at build time, which would allow
117
+ // Turbopack to trace into the WASM package.
118
+ const specifier = ["@rivetkit/sqlite", "dist", "wa-sqlite-async.mjs"].join("/");
119
+ const sqliteModule = await import(specifier);
120
+ if (!isSqliteEsmFactory(sqliteModule.default)) {
121
+ throw new Error("Invalid SQLite ESM factory export");
122
+ }
123
+ const sqliteEsmFactory = sqliteModule.default;
124
+ const require = createRequire(import.meta.url);
125
+ const sqliteDistPath = "@rivetkit/sqlite/dist/";
126
+ const wasmPath = require.resolve(sqliteDistPath + "wa-sqlite-async.wasm");
127
+ const wasmBinary = readFileSync(wasmPath);
128
+ const module = await sqliteEsmFactory({ wasmBinary });
129
+ if (!isSQLiteModule(module)) {
130
+ throw new Error("Invalid SQLite runtime module");
131
+ }
132
+ return {
133
+ sqlite3: Factory(module),
134
+ module,
135
+ };
136
+ }
137
+
138
+ /**
139
+ * Represents an open file
140
+ */
141
+ interface OpenFile {
142
+ /** File path */
143
+ path: string;
144
+ /** File kind tag used by compact key layout */
145
+ fileTag: SqliteFileTag;
146
+ /** Precomputed metadata key */
147
+ metaKey: Uint8Array;
148
+ /** File size in bytes */
149
+ size: number;
150
+ /** True when in-memory size has not been persisted yet */
151
+ metaDirty: boolean;
152
+ /** Open flags */
153
+ flags: number;
154
+ /** KV options for this file */
155
+ options: KvVfsOptions;
156
+ }
157
+
158
+ interface ResolvedFile {
159
+ options: KvVfsOptions;
160
+ fileTag: SqliteFileTag;
161
+ }
162
+
163
+ /**
164
+ * Encodes file metadata to a Uint8Array using BARE schema
165
+ */
166
+ function encodeFileMeta(size: number): Uint8Array {
167
+ const meta: FileMeta = { size: BigInt(size) };
168
+ return FILE_META_VERSIONED.serializeWithEmbeddedVersion(
169
+ meta,
170
+ CURRENT_VERSION,
171
+ );
172
+ }
173
+
174
+ /**
175
+ * Decodes file metadata from a Uint8Array using BARE schema
176
+ */
177
+ function decodeFileMeta(data: Uint8Array): number {
178
+ const meta = FILE_META_VERSIONED.deserializeWithEmbeddedVersion(data);
179
+ return Number(meta.size);
180
+ }
181
+
182
+ function isValidFileSize(size: number): boolean {
183
+ return Number.isSafeInteger(size) && size >= 0 && size <= MAX_FILE_SIZE_BYTES;
184
+ }
185
+
186
+ /**
187
+ * Simple async mutex for serializing database operations
188
+ * @rivetkit/sqlite calls are not safe to run concurrently on one module instance
189
+ */
190
+ class AsyncMutex {
191
+ #locked = false;
192
+ #waiting: (() => void)[] = [];
193
+
194
+ async acquire(): Promise<void> {
195
+ while (this.#locked) {
196
+ await new Promise<void>((resolve) => this.#waiting.push(resolve));
197
+ }
198
+ this.#locked = true;
199
+ }
200
+
201
+ release(): void {
202
+ this.#locked = false;
203
+ const next = this.#waiting.shift();
204
+ if (next) {
205
+ next();
206
+ }
207
+ }
208
+
209
+ async run<T>(fn: () => Promise<T>): Promise<T> {
210
+ await this.acquire();
211
+ try {
212
+ return await fn();
213
+ } finally {
214
+ this.release();
215
+ }
216
+ }
217
+ }
218
+
219
+ /**
220
+ * Database wrapper that provides a simplified SQLite API
221
+ */
222
+ export class Database {
223
+ readonly #sqlite3: SQLite3Api;
224
+ readonly #handle: number;
225
+ readonly #fileName: string;
226
+ readonly #onClose: () => void;
227
+ readonly #sqliteMutex: AsyncMutex;
228
+
229
+ constructor(
230
+ sqlite3: SQLite3Api,
231
+ handle: number,
232
+ fileName: string,
233
+ onClose: () => void,
234
+ sqliteMutex: AsyncMutex,
235
+ ) {
236
+ this.#sqlite3 = sqlite3;
237
+ this.#handle = handle;
238
+ this.#fileName = fileName;
239
+ this.#onClose = onClose;
240
+ this.#sqliteMutex = sqliteMutex;
241
+ }
242
+
243
+ /**
244
+ * Execute SQL with optional row callback
245
+ * @param sql - SQL statement to execute
246
+ * @param callback - Called for each result row with (row, columns)
247
+ */
248
+ async exec(sql: string, callback?: (row: unknown[], columns: string[]) => void): Promise<void> {
249
+ await this.#sqliteMutex.run(async () => {
250
+ await this.#sqlite3.exec(this.#handle, sql, callback);
251
+ });
252
+ }
253
+
254
+ /**
255
+ * Execute a parameterized SQL statement (no result rows)
256
+ * @param sql - SQL statement with ? placeholders
257
+ * @param params - Parameter values to bind
258
+ */
259
+ async run(sql: string, params?: SqliteBindings): Promise<void> {
260
+ await this.#sqliteMutex.run(async () => {
261
+ for await (const stmt of this.#sqlite3.statements(this.#handle, sql)) {
262
+ if (params) {
263
+ this.#sqlite3.bind_collection(stmt, params);
264
+ }
265
+ while ((await this.#sqlite3.step(stmt)) === SQLITE_ROW) {
266
+ // Consume rows for statements that return results.
267
+ }
268
+ }
269
+ });
270
+ }
271
+
272
+ /**
273
+ * Execute a parameterized SQL query and return results
274
+ * @param sql - SQL query with ? placeholders
275
+ * @param params - Parameter values to bind
276
+ * @returns Object with rows (array of arrays) and columns (column names)
277
+ */
278
+ async query(sql: string, params?: SqliteBindings): Promise<{ rows: unknown[][]; columns: string[] }> {
279
+ return this.#sqliteMutex.run(async () => {
280
+ const rows: unknown[][] = [];
281
+ let columns: string[] = [];
282
+ for await (const stmt of this.#sqlite3.statements(this.#handle, sql)) {
283
+ if (params) {
284
+ this.#sqlite3.bind_collection(stmt, params);
285
+ }
286
+
287
+ while ((await this.#sqlite3.step(stmt)) === SQLITE_ROW) {
288
+ if (columns.length === 0) {
289
+ columns = this.#sqlite3.column_names(stmt);
290
+ }
291
+ rows.push(this.#sqlite3.row(stmt));
292
+ }
293
+ }
294
+
295
+ return { rows, columns };
296
+ });
297
+ }
298
+
299
+ /**
300
+ * Close the database
301
+ */
302
+ async close(): Promise<void> {
303
+ await this.#sqliteMutex.run(async () => {
304
+ await this.#sqlite3.close(this.#handle);
305
+ });
306
+ this.#onClose();
307
+ }
308
+
309
+ /**
310
+ * Get the raw @rivetkit/sqlite API (for advanced usage)
311
+ */
312
+ get sqlite3(): SQLite3Api {
313
+ return this.#sqlite3;
314
+ }
315
+
316
+ /**
317
+ * Get the raw database handle (for advanced usage)
318
+ */
319
+ get handle(): number {
320
+ return this.#handle;
321
+ }
322
+ }
323
+
324
+ /**
325
+ * SQLite VFS backed by KV storage.
326
+ *
327
+ * Each instance is independent and has its own @rivetkit/sqlite WASM module.
328
+ * This allows multiple instances to operate concurrently without interference.
329
+ */
330
+ export class SqliteVfs {
331
+ #sqlite3: SQLite3Api | null = null;
332
+ #sqliteSystem: SqliteSystem | null = null;
333
+ #initPromise: Promise<void> | null = null;
334
+ #openMutex = new AsyncMutex();
335
+ #sqliteMutex = new AsyncMutex();
336
+ #instanceId: string;
337
+ #destroyed = false;
338
+
339
+ constructor() {
340
+ // Generate unique instance ID for VFS name
341
+ this.#instanceId = crypto.randomUUID().replace(/-/g, '').slice(0, 8);
342
+ }
343
+
344
+ /**
345
+ * Initialize @rivetkit/sqlite and VFS (called once per instance)
346
+ */
347
+ async #ensureInitialized(): Promise<void> {
348
+ if (this.#destroyed) {
349
+ throw new Error("SqliteVfs is closed");
350
+ }
351
+
352
+ // Fast path: already initialized
353
+ if (this.#sqlite3 && this.#sqliteSystem) {
354
+ return;
355
+ }
356
+
357
+ // Synchronously create the promise if not started
358
+ if (!this.#initPromise) {
359
+ this.#initPromise = (async () => {
360
+ const { sqlite3, module } = await loadSqliteRuntime();
361
+ if (this.#destroyed) {
362
+ return;
363
+ }
364
+ this.#sqlite3 = sqlite3;
365
+ this.#sqliteSystem = new SqliteSystem(
366
+ sqlite3,
367
+ module,
368
+ `kv-vfs-${this.#instanceId}`,
369
+ );
370
+ this.#sqliteSystem.register();
371
+ })();
372
+ }
373
+
374
+ // Wait for initialization
375
+ try {
376
+ await this.#initPromise;
377
+ } catch (error) {
378
+ this.#initPromise = null;
379
+ throw error;
380
+ }
381
+ }
382
+
383
+ /**
384
+ * Open a SQLite database using KV storage backend
385
+ *
386
+ * @param fileName - The database file name (typically the actor ID)
387
+ * @param options - KV storage operations for this database
388
+ * @returns A Database instance
389
+ */
390
+ async open(
391
+ fileName: string,
392
+ options: KvVfsOptions,
393
+ ): Promise<Database> {
394
+ if (this.#destroyed) {
395
+ throw new Error("SqliteVfs is closed");
396
+ }
397
+
398
+ // Serialize all open operations within this instance
399
+ await this.#openMutex.acquire();
400
+ try {
401
+ // Initialize @rivetkit/sqlite and SqliteSystem on first call
402
+ await this.#ensureInitialized();
403
+
404
+ if (!this.#sqlite3 || !this.#sqliteSystem) {
405
+ throw new Error("Failed to initialize SQLite");
406
+ }
407
+ const sqlite3 = this.#sqlite3;
408
+ const sqliteSystem = this.#sqliteSystem;
409
+
410
+ // Register this filename with its KV options
411
+ sqliteSystem.registerFile(fileName, options);
412
+
413
+ // Open database
414
+ const db = await this.#sqliteMutex.run(async () =>
415
+ sqlite3.open_v2(
416
+ fileName,
417
+ SQLITE_OPEN_READWRITE | SQLITE_OPEN_CREATE,
418
+ sqliteSystem.name,
419
+ ),
420
+ );
421
+ // TODO: Benchmark PRAGMA tuning for KV-backed SQLite after open.
422
+ // Start with journal_mode=PERSIST and journal_size_limit to reduce
423
+ // journal churn on high-latency KV without introducing WAL.
424
+
425
+ // Create cleanup callback
426
+ const onClose = () => {
427
+ sqliteSystem.unregisterFile(fileName);
428
+ };
429
+
430
+ return new Database(
431
+ sqlite3,
432
+ db,
433
+ fileName,
434
+ onClose,
435
+ this.#sqliteMutex,
436
+ );
437
+ } finally {
438
+ this.#openMutex.release();
439
+ }
440
+ }
441
+
442
+ /**
443
+ * Tears down this VFS instance and releases internal references.
444
+ */
445
+ async destroy(): Promise<void> {
446
+ if (this.#destroyed) {
447
+ return;
448
+ }
449
+ this.#destroyed = true;
450
+
451
+ const initPromise = this.#initPromise;
452
+ if (initPromise) {
453
+ try {
454
+ await initPromise;
455
+ } catch {
456
+ // Initialization failure already surfaced to caller.
457
+ }
458
+ }
459
+
460
+ if (this.#sqliteSystem) {
461
+ await this.#sqliteSystem.close();
462
+ }
463
+
464
+ this.#sqliteSystem = null;
465
+ this.#sqlite3 = null;
466
+ this.#initPromise = null;
467
+ }
468
+
469
+ /**
470
+ * Alias for destroy to align with DB-style lifecycle naming.
471
+ */
472
+ async close(): Promise<void> {
473
+ await this.destroy();
474
+ }
475
+ }
476
+
477
+ /**
478
+ * Internal VFS implementation
479
+ */
480
+ class SqliteSystem implements SqliteVfsRegistration {
481
+ readonly name: string;
482
+ readonly mxPathName = SQLITE_MAX_PATHNAME_BYTES;
483
+ readonly mxPathname = SQLITE_MAX_PATHNAME_BYTES;
484
+ #mainFileName: string | null = null;
485
+ #mainFileOptions: KvVfsOptions | null = null;
486
+ readonly #openFiles: Map<number, OpenFile> = new Map();
487
+ readonly #sqlite3: SQLite3Api;
488
+ readonly #module: SQLiteModule;
489
+ #heapDataView: DataView;
490
+ #heapDataViewBuffer: ArrayBufferLike;
491
+
492
+ constructor(sqlite3: SQLite3Api, module: SQLiteModule, name: string) {
493
+ this.name = name;
494
+ this.#sqlite3 = sqlite3;
495
+ this.#module = module;
496
+ this.#heapDataViewBuffer = module.HEAPU8.buffer;
497
+ this.#heapDataView = new DataView(this.#heapDataViewBuffer);
498
+ }
499
+
500
+ async close(): Promise<void> {
501
+ this.#openFiles.clear();
502
+ this.#mainFileName = null;
503
+ this.#mainFileOptions = null;
504
+ }
505
+
506
+ isReady(): boolean {
507
+ return true;
508
+ }
509
+
510
+ hasAsyncMethod(methodName: string): boolean {
511
+ return SQLITE_ASYNC_METHODS.has(methodName);
512
+ }
513
+
514
+ /**
515
+ * Registers the VFS with SQLite
516
+ */
517
+ register(): void {
518
+ this.#sqlite3.vfs_register(this, false);
519
+ }
520
+
521
+ /**
522
+ * Registers a file with its KV options (before opening)
523
+ */
524
+ registerFile(fileName: string, options: KvVfsOptions): void {
525
+ if (!this.#mainFileName) {
526
+ this.#mainFileName = fileName;
527
+ this.#mainFileOptions = options;
528
+ return;
529
+ }
530
+
531
+ if (this.#mainFileName !== fileName) {
532
+ throw new Error(
533
+ `SqliteSystem is actor-scoped and expects one main file. Got ${fileName}, expected ${this.#mainFileName}.`,
534
+ );
535
+ }
536
+
537
+ this.#mainFileOptions = options;
538
+ }
539
+
540
+ /**
541
+ * Unregisters a file's KV options (after closing)
542
+ */
543
+ unregisterFile(fileName: string): void {
544
+ if (this.#mainFileName === fileName) {
545
+ this.#mainFileName = null;
546
+ this.#mainFileOptions = null;
547
+ }
548
+ }
549
+
550
+ /**
551
+ * Resolve file path to the actor's main DB file or known SQLite sidecars.
552
+ */
553
+ #resolveFile(path: string): ResolvedFile | null {
554
+ if (!this.#mainFileName || !this.#mainFileOptions) {
555
+ return null;
556
+ }
557
+
558
+ if (path === this.#mainFileName) {
559
+ return { options: this.#mainFileOptions, fileTag: FILE_TAG_MAIN };
560
+ }
561
+ if (path === `${this.#mainFileName}-journal`) {
562
+ return { options: this.#mainFileOptions, fileTag: FILE_TAG_JOURNAL };
563
+ }
564
+ if (path === `${this.#mainFileName}-wal`) {
565
+ return { options: this.#mainFileOptions, fileTag: FILE_TAG_WAL };
566
+ }
567
+ if (path === `${this.#mainFileName}-shm`) {
568
+ return { options: this.#mainFileOptions, fileTag: FILE_TAG_SHM };
569
+ }
570
+
571
+ return null;
572
+ }
573
+
574
+ #resolveFileOrThrow(path: string): ResolvedFile {
575
+ const resolved = this.#resolveFile(path);
576
+ if (resolved) {
577
+ return resolved;
578
+ }
579
+
580
+ if (!this.#mainFileName) {
581
+ throw new Error(`No KV options registered for file: ${path}`);
582
+ }
583
+
584
+ throw new Error(
585
+ `Unsupported SQLite file path ${path}. Expected one of ${this.#mainFileName}, ${this.#mainFileName}-journal, ${this.#mainFileName}-wal, ${this.#mainFileName}-shm.`,
586
+ );
587
+ }
588
+
589
+ #chunkKey(file: OpenFile, chunkIndex: number): Uint8Array {
590
+ return getChunkKey(file.fileTag, chunkIndex);
591
+ }
592
+
593
+ async xOpen(
594
+ _pVfs: number,
595
+ zName: number,
596
+ fileId: number,
597
+ flags: number,
598
+ pOutFlags: number,
599
+ ): Promise<number> {
600
+ const path = this.#decodeFilename(zName, flags);
601
+ if (!path) {
602
+ return VFS.SQLITE_CANTOPEN;
603
+ }
604
+
605
+ // Get the registered KV options for this file
606
+ // For journal/wal files, use the main database's options
607
+ const { options, fileTag } = this.#resolveFileOrThrow(path);
608
+ const metaKey = getMetaKey(fileTag);
609
+
610
+ // Get existing file size if the file exists
611
+ const sizeData = await options.get(metaKey);
612
+
613
+ let size: number;
614
+
615
+ if (sizeData) {
616
+ // File exists, use existing size
617
+ size = decodeFileMeta(sizeData);
618
+ if (!isValidFileSize(size)) {
619
+ return VFS.SQLITE_IOERR;
620
+ }
621
+ } else if (flags & VFS.SQLITE_OPEN_CREATE) {
622
+ // File doesn't exist, create it
623
+ size = 0;
624
+ await options.put(metaKey, encodeFileMeta(size));
625
+ } else {
626
+ // File doesn't exist and we're not creating it
627
+ return VFS.SQLITE_CANTOPEN;
628
+ }
629
+
630
+ // Store open file info with options
631
+ this.#openFiles.set(fileId, {
632
+ path,
633
+ fileTag,
634
+ metaKey,
635
+ size,
636
+ metaDirty: false,
637
+ flags,
638
+ options,
639
+ });
640
+
641
+ // Set output flags to the actual flags used.
642
+ this.#writeInt32(pOutFlags, flags);
643
+
644
+ return VFS.SQLITE_OK;
645
+ }
646
+
647
+ async xClose(fileId: number): Promise<number> {
648
+ const file = this.#openFiles.get(fileId);
649
+ if (!file) {
650
+ return VFS.SQLITE_OK;
651
+ }
652
+
653
+ // Delete-on-close files should skip metadata flush because the file will
654
+ // be removed immediately.
655
+ if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
656
+ await this.#delete(file.path);
657
+ this.#openFiles.delete(fileId);
658
+ return VFS.SQLITE_OK;
659
+ }
660
+
661
+ if (file.metaDirty) {
662
+ await file.options.put(file.metaKey, encodeFileMeta(file.size));
663
+ file.metaDirty = false;
664
+ }
665
+
666
+ this.#openFiles.delete(fileId);
667
+ return VFS.SQLITE_OK;
668
+ }
669
+
670
+ async xRead(
671
+ fileId: number,
672
+ pData: number,
673
+ iAmt: number,
674
+ iOffsetLo: number,
675
+ iOffsetHi: number,
676
+ ): Promise<number> {
677
+ if (iAmt === 0) {
678
+ return VFS.SQLITE_OK;
679
+ }
680
+
681
+ const file = this.#openFiles.get(fileId);
682
+ if (!file) {
683
+ return VFS.SQLITE_IOERR_READ;
684
+ }
685
+
686
+ const data = this.#module.HEAPU8.subarray(pData, pData + iAmt);
687
+ const options = file.options;
688
+ const requestedLength = iAmt;
689
+ const iOffset = delegalize(iOffsetLo, iOffsetHi);
690
+ if (iOffset < 0) {
691
+ return VFS.SQLITE_IOERR_READ;
692
+ }
693
+ const fileSize = file.size;
694
+
695
+ // If offset is beyond file size, return short read with zeroed buffer
696
+ if (iOffset >= fileSize) {
697
+ data.fill(0);
698
+ return VFS.SQLITE_IOERR_SHORT_READ;
699
+ }
700
+
701
+ // Calculate which chunks we need to read
702
+ const startChunk = Math.floor(iOffset / CHUNK_SIZE);
703
+ const endChunk = Math.floor((iOffset + requestedLength - 1) / CHUNK_SIZE);
704
+
705
+ // Fetch all needed chunks
706
+ const chunkKeys: Uint8Array[] = [];
707
+ for (let i = startChunk; i <= endChunk; i++) {
708
+ chunkKeys.push(this.#chunkKey(file, i));
709
+ }
710
+
711
+ const chunks = await options.getBatch(chunkKeys);
712
+
713
+ // Copy data from chunks to output buffer
714
+ for (let i = startChunk; i <= endChunk; i++) {
715
+ const chunkData = chunks[i - startChunk];
716
+ const chunkOffset = i * CHUNK_SIZE;
717
+
718
+ // Calculate the range within this chunk
719
+ const readStart = Math.max(0, iOffset - chunkOffset);
720
+ const readEnd = Math.min(
721
+ CHUNK_SIZE,
722
+ iOffset + requestedLength - chunkOffset,
723
+ );
724
+
725
+ if (chunkData) {
726
+ // Copy available data
727
+ const sourceStart = readStart;
728
+ const sourceEnd = Math.min(readEnd, chunkData.length);
729
+ const destStart = chunkOffset + readStart - iOffset;
730
+
731
+ if (sourceEnd > sourceStart) {
732
+ data.set(chunkData.subarray(sourceStart, sourceEnd), destStart);
733
+ }
734
+
735
+ // Zero-fill if chunk is smaller than expected
736
+ if (sourceEnd < readEnd) {
737
+ const zeroStart = destStart + (sourceEnd - sourceStart);
738
+ const zeroEnd = destStart + (readEnd - readStart);
739
+ data.fill(0, zeroStart, zeroEnd);
740
+ }
741
+ } else {
742
+ // Chunk doesn't exist, zero-fill
743
+ const destStart = chunkOffset + readStart - iOffset;
744
+ const destEnd = destStart + (readEnd - readStart);
745
+ data.fill(0, destStart, destEnd);
746
+ }
747
+ }
748
+
749
+ // If we read less than requested (past EOF), return short read
750
+ const actualBytes = Math.min(requestedLength, fileSize - iOffset);
751
+ if (actualBytes < requestedLength) {
752
+ data.fill(0, actualBytes);
753
+ return VFS.SQLITE_IOERR_SHORT_READ;
754
+ }
755
+
756
+ return VFS.SQLITE_OK;
757
+ }
758
+
759
+ async xWrite(
760
+ fileId: number,
761
+ pData: number,
762
+ iAmt: number,
763
+ iOffsetLo: number,
764
+ iOffsetHi: number,
765
+ ): Promise<number> {
766
+ if (iAmt === 0) {
767
+ return VFS.SQLITE_OK;
768
+ }
769
+
770
+ const file = this.#openFiles.get(fileId);
771
+ if (!file) {
772
+ return VFS.SQLITE_IOERR_WRITE;
773
+ }
774
+
775
+ const data = this.#module.HEAPU8.subarray(pData, pData + iAmt);
776
+ const iOffset = delegalize(iOffsetLo, iOffsetHi);
777
+ if (iOffset < 0) {
778
+ return VFS.SQLITE_IOERR_WRITE;
779
+ }
780
+ const options = file.options;
781
+ const writeLength = iAmt;
782
+ const writeEndOffset = iOffset + writeLength;
783
+ if (writeEndOffset > MAX_FILE_SIZE_BYTES) {
784
+ return VFS.SQLITE_IOERR_WRITE;
785
+ }
786
+
787
+ // Calculate which chunks we need to modify
788
+ const startChunk = Math.floor(iOffset / CHUNK_SIZE);
789
+ const endChunk = Math.floor((iOffset + writeLength - 1) / CHUNK_SIZE);
790
+
791
+ interface WritePlan {
792
+ chunkKey: Uint8Array;
793
+ chunkOffset: number;
794
+ writeStart: number;
795
+ writeEnd: number;
796
+ existingChunkIndex: number;
797
+ }
798
+
799
+ // Only fetch chunks where we must preserve existing prefix/suffix bytes.
800
+ const plans: WritePlan[] = [];
801
+ const chunkKeysToFetch: Uint8Array[] = [];
802
+ for (let i = startChunk; i <= endChunk; i++) {
803
+ const chunkOffset = i * CHUNK_SIZE;
804
+ const writeStart = Math.max(0, iOffset - chunkOffset);
805
+ const writeEnd = Math.min(
806
+ CHUNK_SIZE,
807
+ iOffset + writeLength - chunkOffset,
808
+ );
809
+ const existingBytesInChunk = Math.max(
810
+ 0,
811
+ Math.min(CHUNK_SIZE, file.size - chunkOffset),
812
+ );
813
+ const needsExisting = writeStart > 0 || existingBytesInChunk > writeEnd;
814
+ const chunkKey = this.#chunkKey(file, i);
815
+ let existingChunkIndex = -1;
816
+ if (needsExisting) {
817
+ existingChunkIndex = chunkKeysToFetch.length;
818
+ chunkKeysToFetch.push(chunkKey);
819
+ }
820
+ plans.push({
821
+ chunkKey,
822
+ chunkOffset,
823
+ writeStart,
824
+ writeEnd,
825
+ existingChunkIndex,
826
+ });
827
+ }
828
+
829
+ const existingChunks = chunkKeysToFetch.length > 0
830
+ ? await options.getBatch(chunkKeysToFetch)
831
+ : [];
832
+
833
+ // Prepare new chunk data
834
+ const entriesToWrite: [Uint8Array, Uint8Array][] = [];
835
+
836
+ for (const plan of plans) {
837
+ const existingChunk =
838
+ plan.existingChunkIndex >= 0
839
+ ? existingChunks[plan.existingChunkIndex]
840
+ : null;
841
+ // Create new chunk data
842
+ let newChunk: Uint8Array;
843
+ if (existingChunk) {
844
+ newChunk = new Uint8Array(Math.max(existingChunk.length, plan.writeEnd));
845
+ newChunk.set(existingChunk);
846
+ } else {
847
+ newChunk = new Uint8Array(plan.writeEnd);
848
+ }
849
+
850
+ // Copy data from input buffer to chunk
851
+ const sourceStart = plan.chunkOffset + plan.writeStart - iOffset;
852
+ const sourceEnd = sourceStart + (plan.writeEnd - plan.writeStart);
853
+ newChunk.set(data.subarray(sourceStart, sourceEnd), plan.writeStart);
854
+
855
+ entriesToWrite.push([plan.chunkKey, newChunk]);
856
+ }
857
+
858
+ // Update file size if we wrote past the end
859
+ const previousSize = file.size;
860
+ const newSize = Math.max(file.size, writeEndOffset);
861
+ if (newSize !== previousSize) {
862
+ file.size = newSize;
863
+ file.metaDirty = true;
864
+ }
865
+ if (file.metaDirty) {
866
+ entriesToWrite.push([file.metaKey, encodeFileMeta(file.size)]);
867
+ }
868
+
869
+ // Write all chunks and metadata
870
+ await options.putBatch(entriesToWrite);
871
+ if (file.metaDirty) {
872
+ file.metaDirty = false;
873
+ }
874
+
875
+ return VFS.SQLITE_OK;
876
+ }
877
+
878
+ async xTruncate(
879
+ fileId: number,
880
+ sizeLo: number,
881
+ sizeHi: number,
882
+ ): Promise<number> {
883
+ const file = this.#openFiles.get(fileId);
884
+ if (!file) {
885
+ return VFS.SQLITE_IOERR_TRUNCATE;
886
+ }
887
+
888
+ const size = delegalize(sizeLo, sizeHi);
889
+ if (size < 0 || size > MAX_FILE_SIZE_BYTES) {
890
+ return VFS.SQLITE_IOERR_TRUNCATE;
891
+ }
892
+ const options = file.options;
893
+
894
+ // If truncating to larger size, just update metadata
895
+ if (size >= file.size) {
896
+ if (size > file.size) {
897
+ file.size = size;
898
+ file.metaDirty = true;
899
+ await options.put(file.metaKey, encodeFileMeta(file.size));
900
+ file.metaDirty = false;
901
+ }
902
+ return VFS.SQLITE_OK;
903
+ }
904
+
905
+ // Calculate which chunks to delete
906
+ // Note: When size=0, lastChunkToKeep = floor(-1/4096) = -1, which means
907
+ // all chunks (starting from index 0) will be deleted in the loop below.
908
+ const lastChunkToKeep = Math.floor((size - 1) / CHUNK_SIZE);
909
+ const lastExistingChunk = Math.floor((file.size - 1) / CHUNK_SIZE);
910
+
911
+ // Delete chunks beyond the new size
912
+ const keysToDelete: Uint8Array[] = [];
913
+ for (let i = lastChunkToKeep + 1; i <= lastExistingChunk; i++) {
914
+ keysToDelete.push(this.#chunkKey(file, i));
915
+ }
916
+
917
+ if (keysToDelete.length > 0) {
918
+ await options.deleteBatch(keysToDelete);
919
+ }
920
+
921
+ // Truncate the last kept chunk if needed
922
+ if (size > 0 && size % CHUNK_SIZE !== 0) {
923
+ const lastChunkKey = this.#chunkKey(file, lastChunkToKeep);
924
+ const lastChunkData = await options.get(lastChunkKey);
925
+
926
+ if (lastChunkData && lastChunkData.length > size % CHUNK_SIZE) {
927
+ const truncatedChunk = lastChunkData.subarray(0, size % CHUNK_SIZE);
928
+ await options.put(lastChunkKey, truncatedChunk);
929
+ }
930
+ }
931
+
932
+ // Update file size
933
+ file.size = size;
934
+ file.metaDirty = true;
935
+ await options.put(file.metaKey, encodeFileMeta(file.size));
936
+ file.metaDirty = false;
937
+
938
+ return VFS.SQLITE_OK;
939
+ }
940
+
941
+ async xSync(fileId: number, _flags: number): Promise<number> {
942
+ const file = this.#openFiles.get(fileId);
943
+ if (!file || !file.metaDirty) {
944
+ return VFS.SQLITE_OK;
945
+ }
946
+
947
+ await file.options.put(file.metaKey, encodeFileMeta(file.size));
948
+ file.metaDirty = false;
949
+ return VFS.SQLITE_OK;
950
+ }
951
+
952
+ async xFileSize(fileId: number, pSize: number): Promise<number> {
953
+ const file = this.#openFiles.get(fileId);
954
+ if (!file) {
955
+ return VFS.SQLITE_IOERR_FSTAT;
956
+ }
957
+
958
+ // Set size as 64-bit integer.
959
+ this.#writeBigInt64(pSize, BigInt(file.size));
960
+ return VFS.SQLITE_OK;
961
+ }
962
+
963
+ async xDelete(_pVfs: number, zName: number, _syncDir: number): Promise<number> {
964
+ await this.#delete(this.#module.UTF8ToString(zName));
965
+ return VFS.SQLITE_OK;
966
+ }
967
+
968
+ /**
969
+ * Internal delete implementation
970
+ */
971
+ async #delete(path: string): Promise<void> {
972
+ const { options, fileTag } = this.#resolveFileOrThrow(path);
973
+ const metaKey = getMetaKey(fileTag);
974
+
975
+ // Get file size to find out how many chunks to delete
976
+ const sizeData = await options.get(metaKey);
977
+
978
+ if (!sizeData) {
979
+ // File doesn't exist, that's OK
980
+ return;
981
+ }
982
+
983
+ const size = decodeFileMeta(sizeData);
984
+
985
+ // Delete all chunks
986
+ const keysToDelete: Uint8Array[] = [metaKey];
987
+ const numChunks = Math.ceil(size / CHUNK_SIZE);
988
+ for (let i = 0; i < numChunks; i++) {
989
+ keysToDelete.push(getChunkKey(fileTag, i));
990
+ }
991
+
992
+ await options.deleteBatch(keysToDelete);
993
+ }
994
+
995
+ async xAccess(
996
+ _pVfs: number,
997
+ zName: number,
998
+ _flags: number,
999
+ pResOut: number,
1000
+ ): Promise<number> {
1001
+ // TODO: Measure how often xAccess runs during open and whether these
1002
+ // existence checks add meaningful KV round-trip overhead. If they do,
1003
+ // consider serving file existence from in-memory state.
1004
+ const path = this.#module.UTF8ToString(zName);
1005
+ const resolved = this.#resolveFile(path);
1006
+ if (!resolved) {
1007
+ // File not registered, doesn't exist
1008
+ this.#writeInt32(pResOut, 0);
1009
+ return VFS.SQLITE_OK;
1010
+ }
1011
+
1012
+ const compactMetaKey = getMetaKey(resolved.fileTag);
1013
+ const metaData = await resolved.options.get(compactMetaKey);
1014
+
1015
+ // Set result: 1 if file exists, 0 otherwise
1016
+ this.#writeInt32(pResOut, metaData ? 1 : 0);
1017
+ return VFS.SQLITE_OK;
1018
+ }
1019
+
1020
+ xCheckReservedLock(_fileId: number, pResOut: number): number {
1021
+ // This VFS is actor-scoped with one writer, so there is no external
1022
+ // reserved lock state to report.
1023
+ this.#writeInt32(pResOut, 0);
1024
+ return VFS.SQLITE_OK;
1025
+ }
1026
+
1027
+ xLock(_fileId: number, _flags: number): number {
1028
+ return VFS.SQLITE_OK;
1029
+ }
1030
+
1031
+ xUnlock(_fileId: number, _flags: number): number {
1032
+ return VFS.SQLITE_OK;
1033
+ }
1034
+
1035
+ xFileControl(_fileId: number, _flags: number, _pArg: number): number {
1036
+ return VFS.SQLITE_NOTFOUND;
1037
+ }
1038
+
1039
+ xDeviceCharacteristics(_fileId: number): number {
1040
+ return 0;
1041
+ }
1042
+
1043
+ xFullPathname(_pVfs: number, zName: number, nOut: number, zOut: number): number {
1044
+ const path = this.#module.UTF8ToString(zName);
1045
+ const bytes = TEXT_ENCODER.encode(path);
1046
+ const out = this.#module.HEAPU8.subarray(zOut, zOut + nOut);
1047
+ if (bytes.length >= out.length) {
1048
+ return VFS.SQLITE_IOERR;
1049
+ }
1050
+ out.set(bytes, 0);
1051
+ out[bytes.length] = 0;
1052
+ return VFS.SQLITE_OK;
1053
+ }
1054
+
1055
+ #decodeFilename(zName: number, flags: number): string | null {
1056
+ if (!zName) {
1057
+ return null;
1058
+ }
1059
+
1060
+ if (flags & VFS.SQLITE_OPEN_URI) {
1061
+ // Decode SQLite URI filename layout: path\0key\0value\0...\0
1062
+ let pName = zName;
1063
+ let state: 1 | 2 | 3 | null = 1;
1064
+ const charCodes: number[] = [];
1065
+ while (state) {
1066
+ const charCode = this.#module.HEAPU8[pName++];
1067
+ if (charCode) {
1068
+ charCodes.push(charCode);
1069
+ continue;
1070
+ }
1071
+
1072
+ if (!this.#module.HEAPU8[pName]) {
1073
+ state = null;
1074
+ }
1075
+ switch (state) {
1076
+ case 1:
1077
+ charCodes.push("?".charCodeAt(0));
1078
+ state = 2;
1079
+ break;
1080
+ case 2:
1081
+ charCodes.push("=".charCodeAt(0));
1082
+ state = 3;
1083
+ break;
1084
+ case 3:
1085
+ charCodes.push("&".charCodeAt(0));
1086
+ state = 2;
1087
+ break;
1088
+ }
1089
+ }
1090
+ return TEXT_DECODER.decode(new Uint8Array(charCodes));
1091
+ }
1092
+
1093
+ return this.#module.UTF8ToString(zName);
1094
+ }
1095
+
1096
+ #heapView(): DataView {
1097
+ const heapBuffer = this.#module.HEAPU8.buffer;
1098
+ if (heapBuffer !== this.#heapDataViewBuffer) {
1099
+ this.#heapDataViewBuffer = heapBuffer;
1100
+ this.#heapDataView = new DataView(heapBuffer);
1101
+ }
1102
+ return this.#heapDataView;
1103
+ }
1104
+
1105
+ #writeInt32(pointer: number, value: number): void {
1106
+ const heapByteOffset = this.#module.HEAPU8.byteOffset + pointer;
1107
+ this.#heapView().setInt32(heapByteOffset, value, true);
1108
+ }
1109
+
1110
+ #writeBigInt64(pointer: number, value: bigint): void {
1111
+ const heapByteOffset = this.#module.HEAPU8.byteOffset + pointer;
1112
+ this.#heapView().setBigInt64(heapByteOffset, value, true);
1113
+ }
1114
+ }
1115
+
1116
+ /**
1117
+ * Rebuild an i64 from Emscripten's legalized (lo32, hi32) pair.
1118
+ * SQLite passes file offsets and sizes this way. We decode into unsigned words
1119
+ * and reject values above the VFS max file size.
1120
+ */
1121
+ function delegalize(lo32: number, hi32: number): number {
1122
+ const hi = hi32 >>> 0;
1123
+ const lo = lo32 >>> 0;
1124
+ if (hi > MAX_FILE_SIZE_HI32) {
1125
+ return -1;
1126
+ }
1127
+ if (hi === MAX_FILE_SIZE_HI32 && lo > MAX_FILE_SIZE_LO32) {
1128
+ return -1;
1129
+ }
1130
+ return (hi * UINT32_SIZE) + lo;
1131
+ }