@livestore/sqlite-wasm 0.4.0-dev.1 → 0.4.0-dev.11

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (80) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/browser/mod.d.ts +1 -0
  3. package/dist/browser/mod.d.ts.map +1 -1
  4. package/dist/browser/mod.js.map +1 -1
  5. package/dist/browser/opfs/AccessHandlePoolVFS.d.ts +17 -0
  6. package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
  7. package/dist/browser/opfs/AccessHandlePoolVFS.js +72 -1
  8. package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
  9. package/dist/cf/BlockManager.d.ts +61 -0
  10. package/dist/cf/BlockManager.d.ts.map +1 -0
  11. package/dist/cf/BlockManager.js +157 -0
  12. package/dist/cf/BlockManager.js.map +1 -0
  13. package/dist/cf/CloudflareSqlVFS.d.ts +51 -0
  14. package/dist/cf/CloudflareSqlVFS.d.ts.map +1 -0
  15. package/dist/cf/CloudflareSqlVFS.js +351 -0
  16. package/dist/cf/CloudflareSqlVFS.js.map +1 -0
  17. package/dist/cf/CloudflareWorkerVFS.d.ts +72 -0
  18. package/dist/cf/CloudflareWorkerVFS.d.ts.map +1 -0
  19. package/dist/cf/CloudflareWorkerVFS.js +552 -0
  20. package/dist/cf/CloudflareWorkerVFS.js.map +1 -0
  21. package/dist/cf/mod.d.ts +43 -0
  22. package/dist/cf/mod.d.ts.map +1 -0
  23. package/dist/cf/mod.js +74 -0
  24. package/dist/cf/mod.js.map +1 -0
  25. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts +2 -0
  26. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts.map +1 -0
  27. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +314 -0
  28. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -0
  29. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts +2 -0
  30. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts.map +1 -0
  31. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +266 -0
  32. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -0
  33. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts +2 -0
  34. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts.map +1 -0
  35. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +462 -0
  36. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -0
  37. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts +2 -0
  38. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts.map +1 -0
  39. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +334 -0
  40. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -0
  41. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts +2 -0
  42. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts.map +1 -0
  43. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +354 -0
  44. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -0
  45. package/dist/load-wasm/mod.node.d.ts.map +1 -1
  46. package/dist/load-wasm/mod.node.js +1 -2
  47. package/dist/load-wasm/mod.node.js.map +1 -1
  48. package/dist/load-wasm/mod.workerd.d.ts +2 -0
  49. package/dist/load-wasm/mod.workerd.d.ts.map +1 -0
  50. package/dist/load-wasm/mod.workerd.js +28 -0
  51. package/dist/load-wasm/mod.workerd.js.map +1 -0
  52. package/dist/make-sqlite-db.d.ts +1 -0
  53. package/dist/make-sqlite-db.d.ts.map +1 -1
  54. package/dist/make-sqlite-db.js +28 -4
  55. package/dist/make-sqlite-db.js.map +1 -1
  56. package/dist/node/NodeFS.d.ts +1 -2
  57. package/dist/node/NodeFS.d.ts.map +1 -1
  58. package/dist/node/NodeFS.js +1 -6
  59. package/dist/node/NodeFS.js.map +1 -1
  60. package/dist/node/mod.js +3 -8
  61. package/dist/node/mod.js.map +1 -1
  62. package/package.json +21 -8
  63. package/src/browser/mod.ts +1 -0
  64. package/src/browser/opfs/AccessHandlePoolVFS.ts +79 -1
  65. package/src/cf/BlockManager.ts +225 -0
  66. package/src/cf/CloudflareSqlVFS.ts +450 -0
  67. package/src/cf/CloudflareWorkerVFS.ts +664 -0
  68. package/src/cf/README.md +60 -0
  69. package/src/cf/mod.ts +143 -0
  70. package/src/cf/test/README.md +224 -0
  71. package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +389 -0
  72. package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +322 -0
  73. package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +585 -0
  74. package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +403 -0
  75. package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +433 -0
  76. package/src/load-wasm/mod.node.ts +1 -2
  77. package/src/load-wasm/mod.workerd.ts +28 -0
  78. package/src/make-sqlite-db.ts +38 -4
  79. package/src/node/NodeFS.ts +1 -9
  80. package/src/node/mod.ts +3 -10
package/dist/node/mod.js CHANGED
@@ -38,19 +38,15 @@ export const sqliteDbFactory = ({ sqlite3, }) => Effect.andThen(FileSystem.FileS
38
38
  },
39
39
  });
40
40
  }));
41
- const nodeFsVfsMap = new Map();
42
41
  const makeNodeFsDb = ({ sqlite3, fileName, directory, fs, }) => Effect.gen(function* () {
43
42
  // NOTE to keep the filePath short, we use the directory name in the vfs name
44
43
  // If this is becoming a problem, we can use a hashed version of the directory name
45
44
  const vfsName = `node-fs-${directory}`;
46
- if (nodeFsVfsMap.has(vfsName) === false) {
45
+ if (sqlite3.vfs_registered.has(vfsName) === false) {
47
46
  // TODO refactor with Effect FileSystem instead of using `node:fs` directly inside of NodeFS
48
- const nodeFsVfs = new NodeFS(vfsName, sqlite3.module, directory, () => {
49
- nodeFsVfsMap.delete(vfsName);
50
- });
47
+ const nodeFsVfs = new NodeFS(vfsName, sqlite3.module, directory);
51
48
  // @ts-expect-error TODO fix types
52
49
  sqlite3.vfs_register(nodeFsVfs, false);
53
- nodeFsVfsMap.set(vfsName, nodeFsVfs);
54
50
  }
55
51
  yield* fs.makeDirectory(directory, { recursive: true });
56
52
  const FILE_NAME_MAX_LENGTH = 56;
@@ -59,7 +55,6 @@ const makeNodeFsDb = ({ sqlite3, fileName, directory, fs, }) => Effect.gen(funct
59
55
  }
60
56
  // NOTE SQLite will return a "disk I/O error" if the file path is too long.
61
57
  const dbPointer = sqlite3.open_v2Sync(fileName, undefined, vfsName);
62
- const vfs = nodeFsVfsMap.get(vfsName);
63
- return { dbPointer, vfs };
58
+ return { dbPointer, vfs: {} };
64
59
  }).pipe(UnexpectedError.mapToUnexpectedError);
65
60
  //# sourceMappingURL=mod.js.map
@@ -1 +1 @@
1
- {"version":3,"file":"mod.js","sourceRoot":"","sources":["../../src/node/mod.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAA0D,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC3G,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAA;AAI5D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AA0CpC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,OAAO,GAGR,EAAiE,EAAE,CAClE,MAAM,CAAC,OAAO,CACZ,UAAU,CAAC,UAAU,EACrB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QAClD,OAAO,YAAY,CAA+B;YAChD,OAAO;YACP,QAAQ,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,GAAG;gBACH,SAAS;gBACT,eAAe,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE;gBACzC,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;aAC7C;SACF,CAAQ,CAAA;IACX,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC;QAC7C,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,EAAE;KACH,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE3D,OAAO,YAAY,CAAyB;QAC1C,OAAO;QACP,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI;YACV,GAAG;YACH,SAAS;YACT,eAAe,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;YACzE,QAAQ,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACtC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;SAC7C;KACF,CAAC,CAAA;AACJ,CAAC,CAAC,CACL,CAAA;AAEH,MAAM,YAAY,GAAG,IAAI,GAAG,EAAkB,CAAA;AAE9C,MAAM,YAAY,GAAG,CAAC,EACpB,OAAO,EACP,QAAQ,EACR,SAAS,EACT,EAAE,GAMH,EAAE,EAAE,CACH,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,6EAA6E;IAC7E,mFAAmF;IACnF,MAAM,OAAO,GAAG,WAAW,SAAS,EAAE,CAAA;IACtC,IAAI,YAAY,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;QACxC,4FAA4F;QAC5F,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,OAAO,EAAG,OAAe,CAAC,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE;YAC7E,YAAY,CAAC,MAAM,CAAC,OAAO,CAAC,CAAA;QAC9B,CAAC,CAAC,CAAA;QACF,kCAAkC;QAClC,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;QACtC,YAAY,CAAC,GAAG,CAAC,OAAO,EAAE,SAAS,CAAC,CAAA;IACtC,CAAC;IAED,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvD,MAAM,oBAAoB,GAAG,EAAE,CAAA;IAC/B,IAAI,QAAQ,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,mCAAmC,oBAAoB,cAAc,CAAC,CAAA;IAC7G,CAAC;IAED,2EAA2E;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;IAEnE,MAAM,GAAG,GAAG,YAAY,CAAC,GAAG,CAAC,OAAO,CAAE,CAAA;IAEtC,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,CAAA;AAC3B,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA"}
1
+ {"version":3,"file":"mod.js","sourceRoot":"","sources":["../../src/node/mod.ts"],"names":[],"mappings":"AAAA,OAAO,IAAI,MAAM,WAAW,CAAA;AAE5B,OAAO,EAA0D,eAAe,EAAE,MAAM,mBAAmB,CAAA;AAC3G,OAAO,EAAE,MAAM,EAAE,UAAU,EAAE,MAAM,yBAAyB,CAAA;AAI5D,OAAO,EAAE,cAAc,EAAE,MAAM,qBAAqB,CAAA;AACpD,OAAO,EAAE,YAAY,EAAE,MAAM,sBAAsB,CAAA;AACnD,OAAO,EAAE,MAAM,EAAE,MAAM,aAAa,CAAA;AA0CpC,MAAM,CAAC,MAAM,eAAe,GAAG,CAAC,EAC9B,OAAO,GAGR,EAAiE,EAAE,CAClE,MAAM,CAAC,OAAO,CACZ,UAAU,CAAC,UAAU,EACrB,CAAC,EAAE,EAAE,EAAE,CAAC,CAAC,KAAK,EAAE,EAAE,CAChB,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,IAAI,KAAK,CAAC,IAAI,KAAK,WAAW,EAAE,CAAC;QAC/B,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,cAAc,CAAC,OAAO,CAAC,CAAA;QAClD,OAAO,YAAY,CAA+B;YAChD,OAAO;YACP,QAAQ,EAAE;gBACR,IAAI,EAAE,WAAW;gBACjB,GAAG;gBACH,SAAS;gBACT,eAAe,EAAE,EAAE,QAAQ,EAAE,UAAU,EAAE;gBACzC,QAAQ,EAAE,GAAG,EAAE,GAAE,CAAC;gBAClB,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;aAC7C;SACF,CAAQ,CAAA;IACX,CAAC;IAED,MAAM,EAAE,SAAS,EAAE,GAAG,EAAE,GAAG,KAAK,CAAC,CAAC,YAAY,CAAC;QAC7C,OAAO;QACP,QAAQ,EAAE,KAAK,CAAC,QAAQ;QACxB,SAAS,EAAE,KAAK,CAAC,SAAS;QAC1B,EAAE;KACH,CAAC,CAAA;IAEF,MAAM,QAAQ,GAAG,IAAI,CAAC,IAAI,CAAC,KAAK,CAAC,SAAS,EAAE,KAAK,CAAC,QAAQ,CAAC,CAAA;IAE3D,OAAO,YAAY,CAAyB;QAC1C,OAAO;QACP,QAAQ,EAAE;YACR,IAAI,EAAE,IAAI;YACV,GAAG;YACH,SAAS;YACT,eAAe,EAAE,EAAE,QAAQ,EAAE,KAAK,CAAC,QAAQ,EAAE,SAAS,EAAE,KAAK,CAAC,SAAS,EAAE;YACzE,QAAQ,EAAE,GAAG,EAAE,CAAC,GAAG,CAAC,QAAQ,CAAC,QAAQ,CAAC;YACtC,WAAW,EAAE,KAAK,CAAC,WAAW,IAAI,CAAC,GAAG,EAAE,GAAE,CAAC,CAAC;SAC7C;KACF,CAAC,CAAA;AACJ,CAAC,CAAC,CACL,CAAA;AAEH,MAAM,YAAY,GAAG,CAAC,EACpB,OAAO,EACP,QAAQ,EACR,SAAS,EACT,EAAE,GAMH,EAAE,EAAE,CACH,MAAM,CAAC,GAAG,CAAC,QAAQ,CAAC;IAClB,6EAA6E;IAC7E,mFAAmF;IACnF,MAAM,OAAO,GAAG,WAAW,SAAS,EAAE,CAAA;IACtC,IAAI,OAAO,CAAC,cAAc,CAAC,GAAG,CAAC,OAAO,CAAC,KAAK,KAAK,EAAE,CAAC;QAClD,4FAA4F;QAC5F,MAAM,SAAS,GAAG,IAAI,MAAM,CAAC,OAAO,EAAG,OAAe,CAAC,MAAM,EAAE,SAAS,CAAC,CAAA;QACzE,kCAAkC;QAClC,OAAO,CAAC,YAAY,CAAC,SAAS,EAAE,KAAK,CAAC,CAAA;IACxC,CAAC;IAED,KAAK,CAAC,CAAC,EAAE,CAAC,aAAa,CAAC,SAAS,EAAE,EAAE,SAAS,EAAE,IAAI,EAAE,CAAC,CAAA;IAEvD,MAAM,oBAAoB,GAAG,EAAE,CAAA;IAC/B,IAAI,QAAQ,CAAC,MAAM,GAAG,oBAAoB,EAAE,CAAC;QAC3C,MAAM,IAAI,KAAK,CAAC,aAAa,QAAQ,mCAAmC,oBAAoB,cAAc,CAAC,CAAA;IAC7G,CAAC;IAED,2EAA2E;IAC3E,MAAM,SAAS,GAAG,OAAO,CAAC,WAAW,CAAC,QAAQ,EAAE,SAAS,EAAE,OAAO,CAAC,CAAA;IAEnE,OAAO,EAAE,SAAS,EAAE,GAAG,EAAE,EAAsF,EAAE,CAAA;AACnH,CAAC,CAAC,CAAC,IAAI,CAAC,eAAe,CAAC,oBAAoB,CAAC,CAAA"}
package/package.json CHANGED
@@ -1,6 +1,6 @@
1
1
  {
2
2
  "name": "@livestore/sqlite-wasm",
3
- "version": "0.4.0-dev.1",
3
+ "version": "0.4.0-dev.11",
4
4
  "type": "module",
5
5
  "exports": {
6
6
  ".": {
@@ -9,6 +9,7 @@
9
9
  },
10
10
  "./load-wasm": {
11
11
  "types": "./dist/load-wasm/mod.browser.d.ts",
12
+ "workerd": "./dist/load-wasm/mod.workerd.js",
12
13
  "browser": "./dist/load-wasm/mod.browser.js",
13
14
  "worker": "./dist/load-wasm/mod.browser.js",
14
15
  "node": "./dist/load-wasm/mod.node.js",
@@ -18,20 +19,28 @@
18
19
  "types": "./dist/node/mod.d.ts",
19
20
  "default": "./dist/node/mod.js"
20
21
  },
22
+ "./cf": {
23
+ "types": "./dist/cf/mod.d.ts",
24
+ "default": "./dist/cf/mod.js"
25
+ },
21
26
  "./browser": {
22
27
  "types": "./dist/browser/mod.d.ts",
23
28
  "default": "./dist/browser/mod.js"
24
29
  }
25
30
  },
26
31
  "dependencies": {
27
- "@livestore/wa-sqlite": "1.0.8-dev.4",
28
- "@livestore/common": "0.4.0-dev.1",
29
- "@livestore/utils": "0.4.0-dev.1"
32
+ "@cloudflare/workers-types": "4.20250923.0",
33
+ "@livestore/common": "0.4.0-dev.11",
34
+ "@livestore/common-cf": "0.4.0-dev.11",
35
+ "@livestore/wa-sqlite": "0.4.0-dev.11",
36
+ "@livestore/utils": "0.4.0-dev.11"
30
37
  },
31
38
  "devDependencies": {
32
- "@types/chrome": "^0.1.1",
33
- "@types/node": "24.2.0",
34
- "@types/wicg-file-system-access": "^2023.10.6"
39
+ "@types/chrome": "0.1.4",
40
+ "@types/node": "24.5.2",
41
+ "@types/wicg-file-system-access": "^2023.10.6",
42
+ "vitest": "3.2.4",
43
+ "wrangler": "4.42.2"
35
44
  },
36
45
  "files": [
37
46
  "package.json",
@@ -42,5 +51,9 @@
42
51
  "publishConfig": {
43
52
  "access": "public"
44
53
  },
45
- "scripts": {}
54
+ "scripts": {
55
+ "test": "vitest",
56
+ "test:ui": "vitest --ui",
57
+ "test:watch": "vitest --watch"
58
+ }
46
59
  }
@@ -1,5 +1,6 @@
1
1
  import type { MakeSqliteDb, PersistenceInfo, SqliteDb } from '@livestore/common'
2
2
  import { Effect, Hash } from '@livestore/utils/effect'
3
+ import type { SQLiteAPI } from '@livestore/wa-sqlite'
3
4
  import type { MemoryVFS } from '@livestore/wa-sqlite/src/examples/MemoryVFS.js'
4
5
 
5
6
  import { makeInMemoryDb } from '../in-memory-vfs.ts'
@@ -20,7 +20,33 @@ const HEADER_OFFSET_DATA = SECTOR_SIZE
20
20
  const PERSISTENT_FILE_TYPES =
21
21
  VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
22
22
 
23
- const DEFAULT_CAPACITY = 6
23
+ // OPFS file pool capacity must be predicted rather than dynamically increased because
24
+ // capacity expansion (addCapacity) is async while SQLite operations are synchronous.
25
+ // We cannot await in the middle of sqlite3.step() calls without making the API async.
26
+ //
27
+ // We over-allocate because:
28
+ // 1. SQLite’s temporary file usage is not part of its API contract.
29
+ // Future SQLite versions may create additional temporary files without notice.
30
+ // See: https://www.sqlite.org/tempfiles.html
31
+ // 2. In the future, we may change how we operate the SQLite DBs,
32
+ // which may increase the number of files needed.
33
+ // e.g. enabling the WAL mode, using multi-DB transactions, etc.
34
+ //
35
+ // TRADEOFF: Higher capacity means the VFS opens and keeps more file handles, consuming
36
+ // browser resources. Lower capacity risks "SQLITE_CANTOPEN" errors during operations.
37
+ //
38
+ // CAPACITY CALCULATION:
39
+ // - 2 main databases (state + eventlog) × 4 files each (main, journal, WAL, shm) = 8 files
40
+ // - Up to 5 SQLite temporary files (super-journal, temp DB, materializations,
41
+ // transient indices, VACUUM temp DB) = 5 files
42
+ // - Transient state database archival operations = 1 file
43
+ // - Safety buffer for future SQLite versions and unpredictable usage = 6 files
44
+ // Total: 20 files
45
+ //
46
+ // References:
47
+ // - https://sqlite.org/forum/info/a3da1e34d8
48
+ // - https://www.sqlite.org/tempfiles.html
49
+ const DEFAULT_CAPACITY = 20
24
50
 
25
51
  /**
26
52
  * This VFS uses the updated Access Handle API with all synchronous methods
@@ -66,6 +92,48 @@ export class AccessHandlePoolVFS extends FacadeVFS {
66
92
  return this.#mapAccessHandleToName.get(accessHandle)!
67
93
  }
68
94
 
95
+ /**
96
+ * Reads the SQLite payload (without the OPFS header) for the given file.
97
+ *
98
+ * @privateRemarks
99
+ *
100
+ * Since the file's access handle is a FileSystemSyncAccessHandle — which
101
+ * acquires an exclusive lock — we don't need to handle short reads as
102
+ * the file cannot be modified by other threads.
103
+ */
104
+ readFilePayload(zName: string): ArrayBuffer {
105
+ const path = this.#getPath(zName)
106
+ const accessHandle = this.#mapPathToAccessHandle.get(path)
107
+
108
+ if (accessHandle === undefined) {
109
+ throw new OpfsError({
110
+ path,
111
+ cause: new Error('Cannot read payload for untracked OPFS path'),
112
+ })
113
+ }
114
+
115
+ const fileSize = accessHandle.getSize()
116
+ if (fileSize <= HEADER_OFFSET_DATA) {
117
+ throw new OpfsError({
118
+ path,
119
+ cause: new Error(
120
+ `OPFS file too small to contain header and payload: size ${fileSize} < HEADER_OFFSET_DATA ${HEADER_OFFSET_DATA}`,
121
+ ),
122
+ })
123
+ }
124
+
125
+ const payloadSize = fileSize - HEADER_OFFSET_DATA
126
+ const payload = new Uint8Array(payloadSize)
127
+ const bytesRead = accessHandle.read(payload, { at: HEADER_OFFSET_DATA })
128
+ if (bytesRead !== payloadSize) {
129
+ throw new OpfsError({
130
+ path,
131
+ cause: new Error(`Failed to read full payload from OPFS file: read ${bytesRead}/${payloadSize}`),
132
+ })
133
+ }
134
+ return payload.buffer
135
+ }
136
+
69
137
  resetAccessHandle(zName: string) {
70
138
  const path = this.#getPath(zName)
71
139
  const accessHandle = this.#mapPathToAccessHandle.get(path)!
@@ -215,6 +283,16 @@ export class AccessHandlePoolVFS extends FacadeVFS {
215
283
  return this.#mapAccessHandleToName.size
216
284
  }
217
285
 
286
+ /**
287
+ * Get all currently tracked SQLite file paths.
288
+ * This can be used by higher-level components for file management operations.
289
+ *
290
+ * @returns Array of currently active SQLite file paths
291
+ */
292
+ getTrackedFilePaths(): string[] {
293
+ return Array.from(this.#mapPathToAccessHandle.keys())
294
+ }
295
+
218
296
  /**
219
297
  * Increase the capacity of the file system by n.
220
298
  */
@@ -0,0 +1,225 @@
1
+ import type { CfTypes } from '@livestore/common-cf'
2
+
3
+ export interface BlockRange {
4
+ startBlock: number
5
+ endBlock: number
6
+ startOffset: number
7
+ endOffset: number
8
+ }
9
+
10
+ export interface BlockData {
11
+ blockId: number
12
+ data: Uint8Array
13
+ }
14
+
15
+ /**
16
+ * BlockManager handles the conversion between file operations and block-based storage
17
+ * for the CloudflareSqlVFS. It manages fixed-size blocks stored in SQL tables.
18
+ */
19
+ export class BlockManager {
20
+ private readonly blockSize: number
21
+
22
+ constructor(blockSize: number = 64 * 1024) {
23
+ this.blockSize = blockSize
24
+ }
25
+
26
+ /**
27
+ * Calculate which blocks are needed for a given file operation
28
+ */
29
+ calculateBlockRange(offset: number, length: number): BlockRange {
30
+ const startBlock = Math.floor(offset / this.blockSize)
31
+ const endBlock = Math.floor((offset + length - 1) / this.blockSize)
32
+ const startOffset = offset % this.blockSize
33
+ const endOffset = ((offset + length - 1) % this.blockSize) + 1
34
+
35
+ return {
36
+ startBlock,
37
+ endBlock,
38
+ startOffset,
39
+ endOffset,
40
+ }
41
+ }
42
+
43
+ /**
44
+ * Read blocks from SQL storage and return as a Map
45
+ */
46
+ readBlocks(sql: CfTypes.SqlStorage, filePath: string, blockIds: number[]): Map<number, Uint8Array> {
47
+ const blocks = new Map<number, Uint8Array>()
48
+
49
+ if (blockIds.length === 0) {
50
+ return blocks
51
+ }
52
+
53
+ // Build IN clause for efficient querying
54
+ const placeholders = blockIds.map(() => '?').join(',')
55
+ const query = `
56
+ SELECT block_id, block_data
57
+ FROM vfs_blocks
58
+ WHERE file_path = ? AND block_id IN (${placeholders})
59
+ ORDER BY block_id
60
+ `
61
+
62
+ const cursor = sql.exec<{ block_id: number; block_data: ArrayBuffer }>(query, filePath, ...blockIds)
63
+
64
+ for (const row of cursor) {
65
+ blocks.set(row.block_id, new Uint8Array(row.block_data))
66
+ }
67
+
68
+ return blocks
69
+ }
70
+
71
+ /**
72
+ * Write blocks to SQL storage using exec for now (prepared statements later)
73
+ */
74
+ writeBlocks(sql: CfTypes.SqlStorage, filePath: string, blocks: Map<number, Uint8Array>): void {
75
+ if (blocks.size === 0) {
76
+ return
77
+ }
78
+
79
+ for (const [blockId, data] of blocks) {
80
+ sql.exec(
81
+ 'INSERT OR REPLACE INTO vfs_blocks (file_path, block_id, block_data) VALUES (?, ?, ?)',
82
+ filePath,
83
+ blockId,
84
+ data,
85
+ )
86
+ }
87
+ }
88
+
89
+ /**
90
+ * Delete blocks at or after the specified block ID (used for truncation)
91
+ */
92
+ deleteBlocksAfter(sql: CfTypes.SqlStorage, filePath: string, startBlockId: number): void {
93
+ sql.exec('DELETE FROM vfs_blocks WHERE file_path = ? AND block_id >= ?', filePath, startBlockId)
94
+ }
95
+
96
+ /**
97
+ * Split write data into blocks, handling partial blocks at boundaries
98
+ */
99
+ splitIntoBlocks(
100
+ data: Uint8Array,
101
+ offset: number,
102
+ ): Map<number, { blockId: number; blockOffset: number; data: Uint8Array }> {
103
+ const blocks = new Map<number, { blockId: number; blockOffset: number; data: Uint8Array }>()
104
+
105
+ let remainingData = data
106
+ let currentOffset = offset
107
+
108
+ while (remainingData.length > 0) {
109
+ const blockId = Math.floor(currentOffset / this.blockSize)
110
+ const blockOffset = currentOffset % this.blockSize
111
+ const bytesToWrite = Math.min(remainingData.length, this.blockSize - blockOffset)
112
+
113
+ const blockData = remainingData.slice(0, bytesToWrite)
114
+ blocks.set(blockId, {
115
+ blockId,
116
+ blockOffset,
117
+ data: blockData,
118
+ })
119
+
120
+ remainingData = remainingData.slice(bytesToWrite)
121
+ currentOffset += bytesToWrite
122
+ }
123
+
124
+ return blocks
125
+ }
126
+
127
+ /**
128
+ * Assemble read data from blocks into a continuous buffer
129
+ */
130
+ assembleBlocks(blocks: Map<number, Uint8Array>, range: BlockRange, requestedLength: number): Uint8Array {
131
+ const result = new Uint8Array(requestedLength)
132
+ let resultOffset = 0
133
+
134
+ for (let blockId = range.startBlock; blockId <= range.endBlock; blockId++) {
135
+ const blockData = blocks.get(blockId)
136
+ if (!blockData) {
137
+ // Block not found - fill with zeros (sparse file behavior)
138
+ const zeroLength = Math.min(this.blockSize, requestedLength - resultOffset)
139
+ // result is already zero-filled by default
140
+ resultOffset += zeroLength
141
+ continue
142
+ }
143
+
144
+ // Calculate the slice of this block we need
145
+ const blockStartOffset = blockId === range.startBlock ? range.startOffset : 0
146
+ const blockEndOffset = blockId === range.endBlock ? range.endOffset : blockData.length
147
+ const sliceLength = blockEndOffset - blockStartOffset
148
+
149
+ if (sliceLength > 0) {
150
+ const slice = blockData.slice(blockStartOffset, blockEndOffset)
151
+ result.set(slice, resultOffset)
152
+ resultOffset += sliceLength
153
+ }
154
+ }
155
+
156
+ return result
157
+ }
158
+
159
+ /**
160
+ * Handle partial block writes by reading existing block, modifying, and returning complete block
161
+ */
162
+ mergePartialBlock(
163
+ sql: CfTypes.SqlStorage,
164
+ filePath: string,
165
+ blockId: number,
166
+ blockOffset: number,
167
+ newData: Uint8Array,
168
+ ): Uint8Array {
169
+ // Read existing block data if it exists
170
+ const existingBlocks = this.readBlocks(sql, filePath, [blockId])
171
+ const existingBlock = existingBlocks.get(blockId) || new Uint8Array(this.blockSize)
172
+
173
+ // Create a new block with the merged data
174
+ const mergedBlock = new Uint8Array(this.blockSize)
175
+ mergedBlock.set(existingBlock)
176
+ mergedBlock.set(newData, blockOffset)
177
+
178
+ return mergedBlock
179
+ }
180
+
181
+ /**
182
+ * Get statistics about block usage for a file
183
+ */
184
+ getBlockStats(
185
+ sql: CfTypes.SqlStorage,
186
+ filePath: string,
187
+ ): { totalBlocks: number; storedBlocks: number; totalBytes: number } {
188
+ const blockStatsCursor = sql.exec<{ stored_blocks: number; total_bytes: number }>(
189
+ `SELECT
190
+ COUNT(*) as stored_blocks,
191
+ COALESCE(SUM(LENGTH(block_data)), 0) as total_bytes
192
+ FROM vfs_blocks
193
+ WHERE file_path = ?`,
194
+ filePath,
195
+ )
196
+
197
+ const result = blockStatsCursor.one()
198
+
199
+ // Get file size to calculate theoretical total blocks
200
+ const fileSizeCursor = sql.exec<{ file_size: number }>(
201
+ 'SELECT file_size FROM vfs_files WHERE file_path = ?',
202
+ filePath,
203
+ )
204
+
205
+ let fileSize = 0
206
+ try {
207
+ const fileSizeResult = fileSizeCursor.one()
208
+ fileSize = fileSizeResult.file_size
209
+ } catch {
210
+ // File doesn't exist
211
+ }
212
+
213
+ const totalBlocks = Math.ceil(fileSize / this.blockSize)
214
+
215
+ return {
216
+ totalBlocks,
217
+ storedBlocks: result.stored_blocks,
218
+ totalBytes: result.total_bytes,
219
+ }
220
+ }
221
+
222
+ getBlockSize(): number {
223
+ return this.blockSize
224
+ }
225
+ }