@livestore/sqlite-wasm 0.4.0-dev.1 → 0.4.0-dev.11
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/.tsbuildinfo +1 -1
- package/dist/browser/mod.d.ts +1 -0
- package/dist/browser/mod.d.ts.map +1 -1
- package/dist/browser/mod.js.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts +17 -0
- package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.js +72 -1
- package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
- package/dist/cf/BlockManager.d.ts +61 -0
- package/dist/cf/BlockManager.d.ts.map +1 -0
- package/dist/cf/BlockManager.js +157 -0
- package/dist/cf/BlockManager.js.map +1 -0
- package/dist/cf/CloudflareSqlVFS.d.ts +51 -0
- package/dist/cf/CloudflareSqlVFS.d.ts.map +1 -0
- package/dist/cf/CloudflareSqlVFS.js +351 -0
- package/dist/cf/CloudflareSqlVFS.js.map +1 -0
- package/dist/cf/CloudflareWorkerVFS.d.ts +72 -0
- package/dist/cf/CloudflareWorkerVFS.d.ts.map +1 -0
- package/dist/cf/CloudflareWorkerVFS.js +552 -0
- package/dist/cf/CloudflareWorkerVFS.js.map +1 -0
- package/dist/cf/mod.d.ts +43 -0
- package/dist/cf/mod.d.ts.map +1 -0
- package/dist/cf/mod.js +74 -0
- package/dist/cf/mod.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +314 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +266 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +462 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts +2 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts.map +1 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +334 -0
- package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts +2 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts.map +1 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +354 -0
- package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -0
- package/dist/load-wasm/mod.node.d.ts.map +1 -1
- package/dist/load-wasm/mod.node.js +1 -2
- package/dist/load-wasm/mod.node.js.map +1 -1
- package/dist/load-wasm/mod.workerd.d.ts +2 -0
- package/dist/load-wasm/mod.workerd.d.ts.map +1 -0
- package/dist/load-wasm/mod.workerd.js +28 -0
- package/dist/load-wasm/mod.workerd.js.map +1 -0
- package/dist/make-sqlite-db.d.ts +1 -0
- package/dist/make-sqlite-db.d.ts.map +1 -1
- package/dist/make-sqlite-db.js +28 -4
- package/dist/make-sqlite-db.js.map +1 -1
- package/dist/node/NodeFS.d.ts +1 -2
- package/dist/node/NodeFS.d.ts.map +1 -1
- package/dist/node/NodeFS.js +1 -6
- package/dist/node/NodeFS.js.map +1 -1
- package/dist/node/mod.js +3 -8
- package/dist/node/mod.js.map +1 -1
- package/package.json +21 -8
- package/src/browser/mod.ts +1 -0
- package/src/browser/opfs/AccessHandlePoolVFS.ts +79 -1
- package/src/cf/BlockManager.ts +225 -0
- package/src/cf/CloudflareSqlVFS.ts +450 -0
- package/src/cf/CloudflareWorkerVFS.ts +664 -0
- package/src/cf/README.md +60 -0
- package/src/cf/mod.ts +143 -0
- package/src/cf/test/README.md +224 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +389 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +322 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +585 -0
- package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +403 -0
- package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +433 -0
- package/src/load-wasm/mod.node.ts +1 -2
- package/src/load-wasm/mod.workerd.ts +28 -0
- package/src/make-sqlite-db.ts +38 -4
- package/src/node/NodeFS.ts +1 -9
- package/src/node/mod.ts +3 -10
package/dist/node/mod.js
CHANGED
|
@@ -38,19 +38,15 @@ export const sqliteDbFactory = ({ sqlite3, }) => Effect.andThen(FileSystem.FileS
|
|
|
38
38
|
},
|
|
39
39
|
});
|
|
40
40
|
}));
|
|
41
|
-
const nodeFsVfsMap = new Map();
|
|
42
41
|
const makeNodeFsDb = ({ sqlite3, fileName, directory, fs, }) => Effect.gen(function* () {
|
|
43
42
|
// NOTE to keep the filePath short, we use the directory name in the vfs name
|
|
44
43
|
// If this is becoming a problem, we can use a hashed version of the directory name
|
|
45
44
|
const vfsName = `node-fs-${directory}`;
|
|
46
|
-
if (
|
|
45
|
+
if (sqlite3.vfs_registered.has(vfsName) === false) {
|
|
47
46
|
// TODO refactor with Effect FileSystem instead of using `node:fs` directly inside of NodeFS
|
|
48
|
-
const nodeFsVfs = new NodeFS(vfsName, sqlite3.module, directory
|
|
49
|
-
nodeFsVfsMap.delete(vfsName);
|
|
50
|
-
});
|
|
47
|
+
const nodeFsVfs = new NodeFS(vfsName, sqlite3.module, directory);
|
|
51
48
|
// @ts-expect-error TODO fix types
|
|
52
49
|
sqlite3.vfs_register(nodeFsVfs, false);
|
|
53
|
-
nodeFsVfsMap.set(vfsName, nodeFsVfs);
|
|
54
50
|
}
|
|
55
51
|
yield* fs.makeDirectory(directory, { recursive: true });
|
|
56
52
|
const FILE_NAME_MAX_LENGTH = 56;
|
|
@@ -59,7 +55,6 @@ const makeNodeFsDb = ({ sqlite3, fileName, directory, fs, }) => Effect.gen(funct
|
|
|
59
55
|
}
|
|
60
56
|
// NOTE SQLite will return a "disk I/O error" if the file path is too long.
|
|
61
57
|
const dbPointer = sqlite3.open_v2Sync(fileName, undefined, vfsName);
|
|
62
|
-
|
|
63
|
-
return { dbPointer, vfs };
|
|
58
|
+
return { dbPointer, vfs: {} };
|
|
64
59
|
}).pipe(UnexpectedError.mapToUnexpectedError);
|
|
65
60
|
//# sourceMappingURL=mod.js.map
|
package/dist/node/mod.js.map
CHANGED
|
@@ -1 +1 @@
|
|
|
1
|
-
{"version":3,"file":"mod.js","sourceRoot":"","sources":["../../src/node/mod.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAA0D,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC3G,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAA;AAI5D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AA0CpC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,OAAO,GAGR,EAAiE,EAAE,CAClE,MAAM,CAAC,OAAO,CACZ,UAAU,CAAC,UAAU,EACrB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QAClD,OAAO,YAAY,CAA+B;YAChD,OAAO;YACP,QAAQ,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,GAAG;gBACH,SAAS;gBACT,eAAe,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE;gBACzC,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;aAC7C;SACF,CAAQ,CAAA;IACX,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC;QAC7C,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,EAAE;KACH,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE3D,OAAO,YAAY,CAAyB;QAC1C,OAAO;QACP,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI;YACV,GAAG;YACH,SAAS;YACT,eAAe,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;YACzE,QAAQ,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACtC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;SAC7C;KACF,CAAC,CAAA;AACJ,CAAC,CAAC,CACL,CAAA;AAEH,MAAM,YAAY,GAAG,
|
|
1
|
+
{"version":3,"file":"mod.js","sourceRoot":"","sources":["../../src/node/mod.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAA0D,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC3G,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAA;AAI5D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AA0CpC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,OAAO,GAGR,EAAiE,EAAE,CAClE,MAAM,CAAC,OAAO,CACZ,UAAU,CAAC,UAAU,EACrB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QAClD,OAAO,YAAY,CAA+B;YAChD,OAAO;YACP,QAAQ,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,GAAG;gBACH,SAAS;gBACT,eAAe,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE;gBACzC,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;aAC7C;SACF,CAAQ,CAAA;IACX,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC;QAC7C,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,EAAE;KACH,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE3D,OAAO,YAAY,CAAyB;QAC1C,OAAO;QACP,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI;YACV,GAAG;YACH,SAAS;YACT,eAAe,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;YACzE,QAAQ,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACtC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;SAC7C;KACF,CAAC,CAAA;AACJ,CAAC,CAAC,CACL,CAAA;AAEH,MAAM,YAAY,GAAG,CAAC,EACpB,OAAO,EACP,QAAQ,EACR,SAAS,EACT,EAAE,GAMH,EAAE,EAAE,CACH,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,6EAA6E;IAC7E,mFAAmF;IACnF,MAAM,OAAO,GAAG,WAAW,SAAS,EAAE,CAAA;IACtC,IAAI,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;QAClD,4FAA4F;QAC5F,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,OAAO,EAAG,OAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACzE,kCAAkC;QAClC,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvD,MAAM,oBAAoB,GAAG,EAAE,CAAA;IAC/B,IAAI,QAAQ,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,mCAAmC,oBAAoB,cAAc,CAAC,CAAA;IAC7G,CAAC;IAED,2EAA2E;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;IAEnE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAsF,EAAE,CAAA;AACnH,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA"}
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "@livestore/sqlite-wasm",
|
|
3
|
-
"version": "0.4.0-dev.
|
|
3
|
+
"version": "0.4.0-dev.11",
|
|
4
4
|
"type": "module",
|
|
5
5
|
"exports": {
|
|
6
6
|
".": {
|
|
@@ -9,6 +9,7 @@
|
|
|
9
9
|
},
|
|
10
10
|
"./load-wasm": {
|
|
11
11
|
"types": "./dist/load-wasm/mod.browser.d.ts",
|
|
12
|
+
"workerd": "./dist/load-wasm/mod.workerd.js",
|
|
12
13
|
"browser": "./dist/load-wasm/mod.browser.js",
|
|
13
14
|
"worker": "./dist/load-wasm/mod.browser.js",
|
|
14
15
|
"node": "./dist/load-wasm/mod.node.js",
|
|
@@ -18,20 +19,28 @@
|
|
|
18
19
|
"types": "./dist/node/mod.d.ts",
|
|
19
20
|
"default": "./dist/node/mod.js"
|
|
20
21
|
},
|
|
22
|
+
"./cf": {
|
|
23
|
+
"types": "./dist/cf/mod.d.ts",
|
|
24
|
+
"default": "./dist/cf/mod.js"
|
|
25
|
+
},
|
|
21
26
|
"./browser": {
|
|
22
27
|
"types": "./dist/browser/mod.d.ts",
|
|
23
28
|
"default": "./dist/browser/mod.js"
|
|
24
29
|
}
|
|
25
30
|
},
|
|
26
31
|
"dependencies": {
|
|
27
|
-
"@
|
|
28
|
-
"@livestore/common": "0.4.0-dev.
|
|
29
|
-
"@livestore/
|
|
32
|
+
"@cloudflare/workers-types": "4.20250923.0",
|
|
33
|
+
"@livestore/common": "0.4.0-dev.11",
|
|
34
|
+
"@livestore/common-cf": "0.4.0-dev.11",
|
|
35
|
+
"@livestore/wa-sqlite": "0.4.0-dev.11",
|
|
36
|
+
"@livestore/utils": "0.4.0-dev.11"
|
|
30
37
|
},
|
|
31
38
|
"devDependencies": {
|
|
32
|
-
"@types/chrome": "
|
|
33
|
-
"@types/node": "24.2
|
|
34
|
-
"@types/wicg-file-system-access": "^2023.10.6"
|
|
39
|
+
"@types/chrome": "0.1.4",
|
|
40
|
+
"@types/node": "24.5.2",
|
|
41
|
+
"@types/wicg-file-system-access": "^2023.10.6",
|
|
42
|
+
"vitest": "3.2.4",
|
|
43
|
+
"wrangler": "4.42.2"
|
|
35
44
|
},
|
|
36
45
|
"files": [
|
|
37
46
|
"package.json",
|
|
@@ -42,5 +51,9 @@
|
|
|
42
51
|
"publishConfig": {
|
|
43
52
|
"access": "public"
|
|
44
53
|
},
|
|
45
|
-
"scripts": {
|
|
54
|
+
"scripts": {
|
|
55
|
+
"test": "vitest",
|
|
56
|
+
"test:ui": "vitest --ui",
|
|
57
|
+
"test:watch": "vitest --watch"
|
|
58
|
+
}
|
|
46
59
|
}
|
package/src/browser/mod.ts
CHANGED
|
@@ -1,5 +1,6 @@
|
|
|
1
1
|
import type { MakeSqliteDb, PersistenceInfo, SqliteDb } from '@livestore/common'
|
|
2
2
|
import { Effect, Hash } from '@livestore/utils/effect'
|
|
3
|
+
import type { SQLiteAPI } from '@livestore/wa-sqlite'
|
|
3
4
|
import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
|
|
4
5
|
|
|
5
6
|
import { makeInMemoryDb } from '../in-memory-vfs.ts'
|
|
@@ -20,7 +20,33 @@ const HEADER_OFFSET_DATA = SECTOR_SIZE
|
|
|
20
20
|
const PERSISTENT_FILE_TYPES =
|
|
21
21
|
VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
|
|
22
22
|
|
|
23
|
-
|
|
23
|
+
// OPFS file pool capacity must be predicted rather than dynamically increased because
|
|
24
|
+
// capacity expansion (addCapacity) is async while SQLite operations are synchronous.
|
|
25
|
+
// We cannot await in the middle of sqlite3.step() calls without making the API async.
|
|
26
|
+
//
|
|
27
|
+
// We over-allocate because:
|
|
28
|
+
// 1. SQLite’s temporary file usage is not part of its API contract.
|
|
29
|
+
// Future SQLite versions may create additional temporary files without notice.
|
|
30
|
+
// See: https://www.sqlite.org/tempfiles.html
|
|
31
|
+
// 2. In the future, we may change how we operate the SQLite DBs,
|
|
32
|
+
// which may increase the number of files needed.
|
|
33
|
+
// e.g. enabling the WAL mode, using multi-DB transactions, etc.
|
|
34
|
+
//
|
|
35
|
+
// TRADEOFF: Higher capacity means the VFS opens and keeps more file handles, consuming
|
|
36
|
+
// browser resources. Lower capacity risks "SQLITE_CANTOPEN" errors during operations.
|
|
37
|
+
//
|
|
38
|
+
// CAPACITY CALCULATION:
|
|
39
|
+
// - 2 main databases (state + eventlog) × 4 files each (main, journal, WAL, shm) = 8 files
|
|
40
|
+
// - Up to 5 SQLite temporary files (super-journal, temp DB, materializations,
|
|
41
|
+
// transient indices, VACUUM temp DB) = 5 files
|
|
42
|
+
// - Transient state database archival operations = 1 file
|
|
43
|
+
// - Safety buffer for future SQLite versions and unpredictable usage = 6 files
|
|
44
|
+
// Total: 20 files
|
|
45
|
+
//
|
|
46
|
+
// References:
|
|
47
|
+
// - https://sqlite.org/forum/info/a3da1e34d8
|
|
48
|
+
// - https://www.sqlite.org/tempfiles.html
|
|
49
|
+
const DEFAULT_CAPACITY = 20
|
|
24
50
|
|
|
25
51
|
/**
|
|
26
52
|
* This VFS uses the updated Access Handle API with all synchronous methods
|
|
@@ -66,6 +92,48 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
66
92
|
return this.#mapAccessHandleToName.get(accessHandle)!
|
|
67
93
|
}
|
|
68
94
|
|
|
95
|
+
/**
|
|
96
|
+
* Reads the SQLite payload (without the OPFS header) for the given file.
|
|
97
|
+
*
|
|
98
|
+
* @privateRemarks
|
|
99
|
+
*
|
|
100
|
+
* Since the file's access handle is a FileSystemSyncAccessHandle — which
|
|
101
|
+
* acquires an exclusive lock — we don't need to handle short reads as
|
|
102
|
+
* the file cannot be modified by other threads.
|
|
103
|
+
*/
|
|
104
|
+
readFilePayload(zName: string): ArrayBuffer {
|
|
105
|
+
const path = this.#getPath(zName)
|
|
106
|
+
const accessHandle = this.#mapPathToAccessHandle.get(path)
|
|
107
|
+
|
|
108
|
+
if (accessHandle === undefined) {
|
|
109
|
+
throw new OpfsError({
|
|
110
|
+
path,
|
|
111
|
+
cause: new Error('Cannot read payload for untracked OPFS path'),
|
|
112
|
+
})
|
|
113
|
+
}
|
|
114
|
+
|
|
115
|
+
const fileSize = accessHandle.getSize()
|
|
116
|
+
if (fileSize <= HEADER_OFFSET_DATA) {
|
|
117
|
+
throw new OpfsError({
|
|
118
|
+
path,
|
|
119
|
+
cause: new Error(
|
|
120
|
+
`OPFS file too small to contain header and payload: size ${fileSize} < HEADER_OFFSET_DATA ${HEADER_OFFSET_DATA}`,
|
|
121
|
+
),
|
|
122
|
+
})
|
|
123
|
+
}
|
|
124
|
+
|
|
125
|
+
const payloadSize = fileSize - HEADER_OFFSET_DATA
|
|
126
|
+
const payload = new Uint8Array(payloadSize)
|
|
127
|
+
const bytesRead = accessHandle.read(payload, { at: HEADER_OFFSET_DATA })
|
|
128
|
+
if (bytesRead !== payloadSize) {
|
|
129
|
+
throw new OpfsError({
|
|
130
|
+
path,
|
|
131
|
+
cause: new Error(`Failed to read full payload from OPFS file: read ${bytesRead}/${payloadSize}`),
|
|
132
|
+
})
|
|
133
|
+
}
|
|
134
|
+
return payload.buffer
|
|
135
|
+
}
|
|
136
|
+
|
|
69
137
|
resetAccessHandle(zName: string) {
|
|
70
138
|
const path = this.#getPath(zName)
|
|
71
139
|
const accessHandle = this.#mapPathToAccessHandle.get(path)!
|
|
@@ -215,6 +283,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
|
|
|
215
283
|
return this.#mapAccessHandleToName.size
|
|
216
284
|
}
|
|
217
285
|
|
|
286
|
+
/**
|
|
287
|
+
* Get all currently tracked SQLite file paths.
|
|
288
|
+
* This can be used by higher-level components for file management operations.
|
|
289
|
+
*
|
|
290
|
+
* @returns Array of currently active SQLite file paths
|
|
291
|
+
*/
|
|
292
|
+
getTrackedFilePaths(): string[] {
|
|
293
|
+
return Array.from(this.#mapPathToAccessHandle.keys())
|
|
294
|
+
}
|
|
295
|
+
|
|
218
296
|
/**
|
|
219
297
|
* Increase the capacity of the file system by n.
|
|
220
298
|
*/
|
|
@@ -0,0 +1,225 @@
|
|
|
1
|
+
import type { CfTypes } from '@livestore/common-cf'
|
|
2
|
+
|
|
3
|
+
export interface BlockRange {
|
|
4
|
+
startBlock: number
|
|
5
|
+
endBlock: number
|
|
6
|
+
startOffset: number
|
|
7
|
+
endOffset: number
|
|
8
|
+
}
|
|
9
|
+
|
|
10
|
+
export interface BlockData {
|
|
11
|
+
blockId: number
|
|
12
|
+
data: Uint8Array
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
/**
|
|
16
|
+
* BlockManager handles the conversion between file operations and block-based storage
|
|
17
|
+
* for the CloudflareSqlVFS. It manages fixed-size blocks stored in SQL tables.
|
|
18
|
+
*/
|
|
19
|
+
export class BlockManager {
|
|
20
|
+
private readonly blockSize: number
|
|
21
|
+
|
|
22
|
+
constructor(blockSize: number = 64 * 1024) {
|
|
23
|
+
this.blockSize = blockSize
|
|
24
|
+
}
|
|
25
|
+
|
|
26
|
+
/**
|
|
27
|
+
* Calculate which blocks are needed for a given file operation
|
|
28
|
+
*/
|
|
29
|
+
calculateBlockRange(offset: number, length: number): BlockRange {
|
|
30
|
+
const startBlock = Math.floor(offset / this.blockSize)
|
|
31
|
+
const endBlock = Math.floor((offset + length - 1) / this.blockSize)
|
|
32
|
+
const startOffset = offset % this.blockSize
|
|
33
|
+
const endOffset = ((offset + length - 1) % this.blockSize) + 1
|
|
34
|
+
|
|
35
|
+
return {
|
|
36
|
+
startBlock,
|
|
37
|
+
endBlock,
|
|
38
|
+
startOffset,
|
|
39
|
+
endOffset,
|
|
40
|
+
}
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
/**
|
|
44
|
+
* Read blocks from SQL storage and return as a Map
|
|
45
|
+
*/
|
|
46
|
+
readBlocks(sql: CfTypes.SqlStorage, filePath: string, blockIds: number[]): Map<number, Uint8Array> {
|
|
47
|
+
const blocks = new Map<number, Uint8Array>()
|
|
48
|
+
|
|
49
|
+
if (blockIds.length === 0) {
|
|
50
|
+
return blocks
|
|
51
|
+
}
|
|
52
|
+
|
|
53
|
+
// Build IN clause for efficient querying
|
|
54
|
+
const placeholders = blockIds.map(() => '?').join(',')
|
|
55
|
+
const query = `
|
|
56
|
+
SELECT block_id, block_data
|
|
57
|
+
FROM vfs_blocks
|
|
58
|
+
WHERE file_path = ? AND block_id IN (${placeholders})
|
|
59
|
+
ORDER BY block_id
|
|
60
|
+
`
|
|
61
|
+
|
|
62
|
+
const cursor = sql.exec<{ block_id: number; block_data: ArrayBuffer }>(query, filePath, ...blockIds)
|
|
63
|
+
|
|
64
|
+
for (const row of cursor) {
|
|
65
|
+
blocks.set(row.block_id, new Uint8Array(row.block_data))
|
|
66
|
+
}
|
|
67
|
+
|
|
68
|
+
return blocks
|
|
69
|
+
}
|
|
70
|
+
|
|
71
|
+
/**
|
|
72
|
+
* Write blocks to SQL storage using exec for now (prepared statements later)
|
|
73
|
+
*/
|
|
74
|
+
writeBlocks(sql: CfTypes.SqlStorage, filePath: string, blocks: Map<number, Uint8Array>): void {
|
|
75
|
+
if (blocks.size === 0) {
|
|
76
|
+
return
|
|
77
|
+
}
|
|
78
|
+
|
|
79
|
+
for (const [blockId, data] of blocks) {
|
|
80
|
+
sql.exec(
|
|
81
|
+
'INSERT OR REPLACE INTO vfs_blocks (file_path, block_id, block_data) VALUES (?, ?, ?)',
|
|
82
|
+
filePath,
|
|
83
|
+
blockId,
|
|
84
|
+
data,
|
|
85
|
+
)
|
|
86
|
+
}
|
|
87
|
+
}
|
|
88
|
+
|
|
89
|
+
/**
|
|
90
|
+
* Delete blocks at or after the specified block ID (used for truncation)
|
|
91
|
+
*/
|
|
92
|
+
deleteBlocksAfter(sql: CfTypes.SqlStorage, filePath: string, startBlockId: number): void {
|
|
93
|
+
sql.exec('DELETE FROM vfs_blocks WHERE file_path = ? AND block_id >= ?', filePath, startBlockId)
|
|
94
|
+
}
|
|
95
|
+
|
|
96
|
+
/**
|
|
97
|
+
* Split write data into blocks, handling partial blocks at boundaries
|
|
98
|
+
*/
|
|
99
|
+
splitIntoBlocks(
|
|
100
|
+
data: Uint8Array,
|
|
101
|
+
offset: number,
|
|
102
|
+
): Map<number, { blockId: number; blockOffset: number; data: Uint8Array }> {
|
|
103
|
+
const blocks = new Map<number, { blockId: number; blockOffset: number; data: Uint8Array }>()
|
|
104
|
+
|
|
105
|
+
let remainingData = data
|
|
106
|
+
let currentOffset = offset
|
|
107
|
+
|
|
108
|
+
while (remainingData.length > 0) {
|
|
109
|
+
const blockId = Math.floor(currentOffset / this.blockSize)
|
|
110
|
+
const blockOffset = currentOffset % this.blockSize
|
|
111
|
+
const bytesToWrite = Math.min(remainingData.length, this.blockSize - blockOffset)
|
|
112
|
+
|
|
113
|
+
const blockData = remainingData.slice(0, bytesToWrite)
|
|
114
|
+
blocks.set(blockId, {
|
|
115
|
+
blockId,
|
|
116
|
+
blockOffset,
|
|
117
|
+
data: blockData,
|
|
118
|
+
})
|
|
119
|
+
|
|
120
|
+
remainingData = remainingData.slice(bytesToWrite)
|
|
121
|
+
currentOffset += bytesToWrite
|
|
122
|
+
}
|
|
123
|
+
|
|
124
|
+
return blocks
|
|
125
|
+
}
|
|
126
|
+
|
|
127
|
+
/**
|
|
128
|
+
* Assemble read data from blocks into a continuous buffer
|
|
129
|
+
*/
|
|
130
|
+
assembleBlocks(blocks: Map<number, Uint8Array>, range: BlockRange, requestedLength: number): Uint8Array {
|
|
131
|
+
const result = new Uint8Array(requestedLength)
|
|
132
|
+
let resultOffset = 0
|
|
133
|
+
|
|
134
|
+
for (let blockId = range.startBlock; blockId <= range.endBlock; blockId++) {
|
|
135
|
+
const blockData = blocks.get(blockId)
|
|
136
|
+
if (!blockData) {
|
|
137
|
+
// Block not found - fill with zeros (sparse file behavior)
|
|
138
|
+
const zeroLength = Math.min(this.blockSize, requestedLength - resultOffset)
|
|
139
|
+
// result is already zero-filled by default
|
|
140
|
+
resultOffset += zeroLength
|
|
141
|
+
continue
|
|
142
|
+
}
|
|
143
|
+
|
|
144
|
+
// Calculate the slice of this block we need
|
|
145
|
+
const blockStartOffset = blockId === range.startBlock ? range.startOffset : 0
|
|
146
|
+
const blockEndOffset = blockId === range.endBlock ? range.endOffset : blockData.length
|
|
147
|
+
const sliceLength = blockEndOffset - blockStartOffset
|
|
148
|
+
|
|
149
|
+
if (sliceLength > 0) {
|
|
150
|
+
const slice = blockData.slice(blockStartOffset, blockEndOffset)
|
|
151
|
+
result.set(slice, resultOffset)
|
|
152
|
+
resultOffset += sliceLength
|
|
153
|
+
}
|
|
154
|
+
}
|
|
155
|
+
|
|
156
|
+
return result
|
|
157
|
+
}
|
|
158
|
+
|
|
159
|
+
/**
|
|
160
|
+
* Handle partial block writes by reading existing block, modifying, and returning complete block
|
|
161
|
+
*/
|
|
162
|
+
mergePartialBlock(
|
|
163
|
+
sql: CfTypes.SqlStorage,
|
|
164
|
+
filePath: string,
|
|
165
|
+
blockId: number,
|
|
166
|
+
blockOffset: number,
|
|
167
|
+
newData: Uint8Array,
|
|
168
|
+
): Uint8Array {
|
|
169
|
+
// Read existing block data if it exists
|
|
170
|
+
const existingBlocks = this.readBlocks(sql, filePath, [blockId])
|
|
171
|
+
const existingBlock = existingBlocks.get(blockId) || new Uint8Array(this.blockSize)
|
|
172
|
+
|
|
173
|
+
// Create a new block with the merged data
|
|
174
|
+
const mergedBlock = new Uint8Array(this.blockSize)
|
|
175
|
+
mergedBlock.set(existingBlock)
|
|
176
|
+
mergedBlock.set(newData, blockOffset)
|
|
177
|
+
|
|
178
|
+
return mergedBlock
|
|
179
|
+
}
|
|
180
|
+
|
|
181
|
+
/**
|
|
182
|
+
* Get statistics about block usage for a file
|
|
183
|
+
*/
|
|
184
|
+
getBlockStats(
|
|
185
|
+
sql: CfTypes.SqlStorage,
|
|
186
|
+
filePath: string,
|
|
187
|
+
): { totalBlocks: number; storedBlocks: number; totalBytes: number } {
|
|
188
|
+
const blockStatsCursor = sql.exec<{ stored_blocks: number; total_bytes: number }>(
|
|
189
|
+
`SELECT
|
|
190
|
+
COUNT(*) as stored_blocks,
|
|
191
|
+
COALESCE(SUM(LENGTH(block_data)), 0) as total_bytes
|
|
192
|
+
FROM vfs_blocks
|
|
193
|
+
WHERE file_path = ?`,
|
|
194
|
+
filePath,
|
|
195
|
+
)
|
|
196
|
+
|
|
197
|
+
const result = blockStatsCursor.one()
|
|
198
|
+
|
|
199
|
+
// Get file size to calculate theoretical total blocks
|
|
200
|
+
const fileSizeCursor = sql.exec<{ file_size: number }>(
|
|
201
|
+
'SELECT file_size FROM vfs_files WHERE file_path = ?',
|
|
202
|
+
filePath,
|
|
203
|
+
)
|
|
204
|
+
|
|
205
|
+
let fileSize = 0
|
|
206
|
+
try {
|
|
207
|
+
const fileSizeResult = fileSizeCursor.one()
|
|
208
|
+
fileSize = fileSizeResult.file_size
|
|
209
|
+
} catch {
|
|
210
|
+
// File doesn't exist
|
|
211
|
+
}
|
|
212
|
+
|
|
213
|
+
const totalBlocks = Math.ceil(fileSize / this.blockSize)
|
|
214
|
+
|
|
215
|
+
return {
|
|
216
|
+
totalBlocks,
|
|
217
|
+
storedBlocks: result.stored_blocks,
|
|
218
|
+
totalBytes: result.total_bytes,
|
|
219
|
+
}
|
|
220
|
+
}
|
|
221
|
+
|
|
222
|
+
getBlockSize(): number {
|
|
223
|
+
return this.blockSize
|
|
224
|
+
}
|
|
225
|
+
}
|