@livestore/sqlite-wasm 0.4.0-dev.2 → 0.4.0-dev.20

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (97) hide show
  1. package/dist/.tsbuildinfo +1 -1
  2. package/dist/FacadeVFS.d.ts.map +1 -1
  3. package/dist/FacadeVFS.js +4 -0
  4. package/dist/FacadeVFS.js.map +1 -1
  5. package/dist/browser/mod.d.ts +15 -7
  6. package/dist/browser/mod.d.ts.map +1 -1
  7. package/dist/browser/mod.js +49 -43
  8. package/dist/browser/mod.js.map +1 -1
  9. package/dist/browser/opfs/AccessHandlePoolVFS.d.ts +40 -17
  10. package/dist/browser/opfs/AccessHandlePoolVFS.d.ts.map +1 -1
  11. package/dist/browser/opfs/AccessHandlePoolVFS.js +211 -143
  12. package/dist/browser/opfs/AccessHandlePoolVFS.js.map +1 -1
  13. package/dist/browser/opfs/index.d.ts +3 -2
  14. package/dist/browser/opfs/index.d.ts.map +1 -1
  15. package/dist/browser/opfs/index.js +1 -1
  16. package/dist/browser/opfs/index.js.map +1 -1
  17. package/dist/browser/opfs/opfs-sah-pool.d.ts +1 -1
  18. package/dist/browser/opfs/opfs-sah-pool.d.ts.map +1 -1
  19. package/dist/browser/opfs/opfs-sah-pool.js +1 -1
  20. package/dist/browser/opfs/opfs-sah-pool.js.map +1 -1
  21. package/dist/cf/BlockManager.d.ts +61 -0
  22. package/dist/cf/BlockManager.d.ts.map +1 -0
  23. package/dist/cf/BlockManager.js +157 -0
  24. package/dist/cf/BlockManager.js.map +1 -0
  25. package/dist/cf/CloudflareSqlVFS.d.ts +51 -0
  26. package/dist/cf/CloudflareSqlVFS.d.ts.map +1 -0
  27. package/dist/cf/CloudflareSqlVFS.js +351 -0
  28. package/dist/cf/CloudflareSqlVFS.js.map +1 -0
  29. package/dist/cf/CloudflareWorkerVFS.d.ts +72 -0
  30. package/dist/cf/CloudflareWorkerVFS.d.ts.map +1 -0
  31. package/dist/cf/CloudflareWorkerVFS.js +552 -0
  32. package/dist/cf/CloudflareWorkerVFS.js.map +1 -0
  33. package/dist/cf/mod.d.ts +43 -0
  34. package/dist/cf/mod.d.ts.map +1 -0
  35. package/dist/cf/mod.js +74 -0
  36. package/dist/cf/mod.js.map +1 -0
  37. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts +2 -0
  38. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.d.ts.map +1 -0
  39. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js +314 -0
  40. package/dist/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.js.map +1 -0
  41. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts +2 -0
  42. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.d.ts.map +1 -0
  43. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js +266 -0
  44. package/dist/cf/test/async-storage/cloudflare-worker-vfs-core.test.js.map +1 -0
  45. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts +2 -0
  46. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.d.ts.map +1 -0
  47. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js +462 -0
  48. package/dist/cf/test/async-storage/cloudflare-worker-vfs-integration.test.js.map +1 -0
  49. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts +2 -0
  50. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.d.ts.map +1 -0
  51. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js +334 -0
  52. package/dist/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.js.map +1 -0
  53. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts +2 -0
  54. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.d.ts.map +1 -0
  55. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js +354 -0
  56. package/dist/cf/test/sql/cloudflare-sql-vfs-core.test.js.map +1 -0
  57. package/dist/load-wasm/mod.node.d.ts.map +1 -1
  58. package/dist/load-wasm/mod.node.js +1 -2
  59. package/dist/load-wasm/mod.node.js.map +1 -1
  60. package/dist/load-wasm/mod.workerd.d.ts +2 -0
  61. package/dist/load-wasm/mod.workerd.d.ts.map +1 -0
  62. package/dist/load-wasm/mod.workerd.js +28 -0
  63. package/dist/load-wasm/mod.workerd.js.map +1 -0
  64. package/dist/make-sqlite-db.d.ts +1 -0
  65. package/dist/make-sqlite-db.d.ts.map +1 -1
  66. package/dist/make-sqlite-db.js +29 -8
  67. package/dist/make-sqlite-db.js.map +1 -1
  68. package/dist/node/NodeFS.d.ts +1 -2
  69. package/dist/node/NodeFS.d.ts.map +1 -1
  70. package/dist/node/NodeFS.js +1 -6
  71. package/dist/node/NodeFS.js.map +1 -1
  72. package/dist/node/mod.d.ts.map +1 -1
  73. package/dist/node/mod.js +5 -10
  74. package/dist/node/mod.js.map +1 -1
  75. package/package.json +21 -8
  76. package/src/FacadeVFS.ts +5 -0
  77. package/src/browser/mod.ts +39 -13
  78. package/src/browser/opfs/AccessHandlePoolVFS.ts +387 -225
  79. package/src/browser/opfs/index.ts +4 -3
  80. package/src/browser/opfs/opfs-sah-pool.ts +1 -1
  81. package/src/cf/BlockManager.ts +225 -0
  82. package/src/cf/CloudflareSqlVFS.ts +450 -0
  83. package/src/cf/CloudflareWorkerVFS.ts +664 -0
  84. package/src/cf/README.md +60 -0
  85. package/src/cf/mod.ts +143 -0
  86. package/src/cf/test/README.md +224 -0
  87. package/src/cf/test/async-storage/cloudflare-worker-vfs-advanced.test.ts +389 -0
  88. package/src/cf/test/async-storage/cloudflare-worker-vfs-core.test.ts +322 -0
  89. package/src/cf/test/async-storage/cloudflare-worker-vfs-integration.test.ts +585 -0
  90. package/src/cf/test/async-storage/cloudflare-worker-vfs-reliability.test.ts +403 -0
  91. package/src/cf/test/sql/cloudflare-sql-vfs-core.test.ts +433 -0
  92. package/src/load-wasm/mod.node.ts +1 -2
  93. package/src/load-wasm/mod.workerd.ts +28 -0
  94. package/src/make-sqlite-db.ts +39 -8
  95. package/src/node/NodeFS.ts +1 -9
  96. package/src/node/mod.ts +5 -12
  97. package/src/ambient.d.ts +0 -18
@@ -1,5 +1,9 @@
1
- import { Effect, Schedule, Schema } from '@livestore/utils/effect'
2
1
  // Based on https://github.com/rhashimoto/wa-sqlite/blob/master/src/examples/AccessHandlePoolVFS.js
2
+ /// <reference lib="webworker" />
3
+
4
+ import { shouldNeverHappen } from '@livestore/utils'
5
+ import { Effect, Runtime, Schedule, type Scope, Stream } from '@livestore/utils/effect'
6
+ import { Opfs, type WebError } from '@livestore/utils/effect/browser'
3
7
  import * as VFS from '@livestore/wa-sqlite/src/VFS.js'
4
8
  import { FacadeVFS } from '../../FacadeVFS.ts'
5
9
 
@@ -20,7 +24,33 @@ const HEADER_OFFSET_DATA = SECTOR_SIZE
20
24
  const PERSISTENT_FILE_TYPES =
21
25
  VFS.SQLITE_OPEN_MAIN_DB | VFS.SQLITE_OPEN_MAIN_JOURNAL | VFS.SQLITE_OPEN_SUPER_JOURNAL | VFS.SQLITE_OPEN_WAL
22
26
 
23
- const DEFAULT_CAPACITY = 6
27
+ // OPFS file pool capacity must be predicted rather than dynamically increased because
28
+ // capacity expansion (addCapacity) is async while SQLite operations are synchronous.
29
+ // We cannot await in the middle of sqlite3.step() calls without making the API async.
30
+ //
31
+ // We over-allocate because:
32
+ // 1. SQLite’s temporary file usage is not part of its API contract.
33
+ // Future SQLite versions may create additional temporary files without notice.
34
+ // See: https://www.sqlite.org/tempfiles.html
35
+ // 2. In the future, we may change how we operate the SQLite DBs,
36
+ // which may increase the number of files needed.
37
+ // e.g. enabling the WAL mode, using multi-DB transactions, etc.
38
+ //
39
+ // TRADEOFF: Higher capacity means the VFS opens and keeps more file handles, consuming
40
+ // browser resources. Lower capacity risks "SQLITE_CANTOPEN" errors during operations.
41
+ //
42
+ // CAPACITY CALCULATION:
43
+ // - 2 main databases (state + eventlog) × 4 files each (main, journal, WAL, shm) = 8 files
44
+ // - Up to 5 SQLite temporary files (super-journal, temp DB, materializations,
45
+ // transient indices, VACUUM temp DB) = 5 files
46
+ // - Transient state database archival operations = 1 file
47
+ // - Safety buffer for future SQLite versions and unpredictable usage = 6 files
48
+ // Total: 20 files
49
+ //
50
+ // References:
51
+ // - https://sqlite.org/forum/info/a3da1e34d8
52
+ // - https://www.sqlite.org/tempfiles.html
53
+ const DEFAULT_CAPACITY = 20
24
54
 
25
55
  /**
26
56
  * This VFS uses the updated Access Handle API with all synchronous methods
@@ -36,6 +66,9 @@ export class AccessHandlePoolVFS extends FacadeVFS {
36
66
  #directoryPath
37
67
  #directoryHandle: FileSystemDirectoryHandle | undefined
38
68
 
69
+ // Runtime for executing Effect operations
70
+ readonly #runtime: Runtime.Runtime<Opfs.Opfs | Scope.Scope>
71
+
39
72
  // The OPFS files all have randomly-generated names that do not match
40
73
  // the SQLite files whose data they contain. This map links those names
41
74
  // with their respective OPFS access handles.
@@ -49,33 +82,89 @@ export class AccessHandlePoolVFS extends FacadeVFS {
49
82
 
50
83
  #mapIdToFile = new Map<number, { path: string; flags: number; accessHandle: FileSystemSyncAccessHandle }>()
51
84
 
52
- static async create(name: string, directoryPath: string, module: any) {
53
- const vfs = new AccessHandlePoolVFS(name, directoryPath, module)
54
- await vfs.isReady()
85
+ static create = Effect.fn(function* (name: string, directoryPath: string, module: any) {
86
+ const runtime = yield* Effect.runtime<Opfs.Opfs | Scope.Scope>()
87
+ const vfs = new AccessHandlePoolVFS({ name, directoryPath, module, runtime })
88
+ yield* Effect.promise(() => vfs.isReady())
55
89
  return vfs
56
- }
57
-
58
- constructor(name: string, directoryPath: string, module: any) {
90
+ })
91
+
92
+ constructor({
93
+ name,
94
+ directoryPath,
95
+ module,
96
+ runtime,
97
+ }: { name: string; directoryPath: string; module: any; runtime: Runtime.Runtime<Opfs.Opfs | Scope.Scope> }) {
59
98
  super(name, module)
60
99
  this.#directoryPath = directoryPath
100
+ this.#runtime = runtime
61
101
  }
62
102
 
63
- getOpfsFileName(zName: string) {
64
- const path = this.#getPath(zName)
65
- const accessHandle = this.#mapPathToAccessHandle.get(path)!
66
- return this.#mapAccessHandleToName.get(accessHandle)!
67
- }
103
+ /**
104
+ * Get the OPFS file name that contains the data for the given SQLite file.
105
+ *
106
+ * @remarks
107
+ *
108
+ * This would be for one of the files in the pool managed by this VFS.
109
+ * It's not the same as the SQLite file name. It's a randomly-generated
110
+ * string that is not meaningful to the application.
111
+ */
112
+ getOpfsFileName = Effect.fn((zName: string) =>
113
+ Effect.gen(this, function* () {
114
+ const path = this.#getPath(zName)
115
+ const accessHandle = this.#mapPathToAccessHandle.get(path)!
116
+ return this.#mapAccessHandleToName.get(accessHandle)!
117
+ }),
118
+ )
68
119
 
69
- resetAccessHandle(zName: string) {
70
- const path = this.#getPath(zName)
71
- const accessHandle = this.#mapPathToAccessHandle.get(path)!
72
- accessHandle.truncate(HEADER_OFFSET_DATA)
73
- // accessHandle.write(new Uint8Array(), { at: HEADER_OFFSET_DATA })
74
- // accessHandle.flush()
75
- }
120
+ /**
121
+ * Reads the SQLite payload (without the OPFS header) for the given file.
122
+ *
123
+ * @privateRemarks
124
+ *
125
+ * Since the file's access handle is a FileSystemSyncAccessHandle — which
126
+ * acquires an exclusive lock — we don't need to handle short reads as
127
+ * the file cannot be modified by other threads.
128
+ */
129
+ readFilePayload = Effect.fn((zName: string) =>
130
+ Effect.gen(this, function* () {
131
+ const path = this.#getPath(zName)
132
+ const accessHandle = this.#mapPathToAccessHandle.get(path)
133
+
134
+ if (accessHandle === undefined) {
135
+ return shouldNeverHappen('Cannot read payload for untracked OPFS path')
136
+ }
137
+
138
+ const fileSize = yield* Opfs.Opfs.syncGetSize(accessHandle)
139
+ if (fileSize <= HEADER_OFFSET_DATA) {
140
+ return shouldNeverHappen(
141
+ `OPFS file too small to contain header and payload: size ${fileSize} < HEADER_OFFSET_DATA ${HEADER_OFFSET_DATA}`,
142
+ )
143
+ }
144
+
145
+ const payloadSize = fileSize - HEADER_OFFSET_DATA
146
+ const payload = new Uint8Array(payloadSize)
147
+ const bytesRead = yield* Opfs.Opfs.syncRead(accessHandle, payload, { at: HEADER_OFFSET_DATA })
148
+ if (bytesRead !== payloadSize) {
149
+ return shouldNeverHappen(`Failed to read full payload from OPFS file: read ${bytesRead}/${payloadSize}`)
150
+ }
151
+
152
+ return payload.buffer
153
+ }),
154
+ )
155
+
156
+ resetAccessHandle = Effect.fn((zName: string) =>
157
+ Effect.gen(this, function* () {
158
+ const path = this.#getPath(zName)
159
+ const accessHandle = this.#mapPathToAccessHandle.get(path)!
160
+ yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
161
+ // accessHandle.write(new Uint8Array(), { at: HEADER_OFFSET_DATA })
162
+ // accessHandle.flush()
163
+ }),
164
+ )
76
165
 
77
166
  jOpen(zName: string, fileId: number, flags: number, pOutFlags: DataView): number {
78
- try {
167
+ return Effect.gen(this, function* () {
79
168
  // First try to open a path that already exists in the file system.
80
169
  const path = zName ? this.#getPath(zName) : Math.random().toString(36)
81
170
  let accessHandle = this.#mapPathToAccessHandle.get(path)
@@ -84,16 +173,15 @@ export class AccessHandlePoolVFS extends FacadeVFS {
84
173
  if (this.getSize() < this.getCapacity()) {
85
174
  // Choose an unassociated OPFS file from the pool.
86
175
  ;[accessHandle] = this.#availableAccessHandles.keys()
87
- this.#setAssociatedPath(accessHandle!, path, flags)
176
+ yield* this.#setAssociatedPath(accessHandle!, path, flags)
88
177
  } else {
89
178
  // Out of unassociated files. This can be fixed by calling
90
179
  // addCapacity() from the application.
91
- throw new Error('cannot create file')
180
+ return yield* Effect.dieMessage('cannot create file')
92
181
  }
93
182
  }
94
- if (!accessHandle) {
95
- throw new Error('file not found')
96
- }
183
+ if (!accessHandle) return yield* Effect.dieMessage('file not found')
184
+
97
185
  // Subsequent methods are only passed the fileId, so make sure we have
98
186
  // a way to get the file resources.
99
187
  const file = { path, flags, accessHandle }
@@ -101,61 +189,102 @@ export class AccessHandlePoolVFS extends FacadeVFS {
101
189
 
102
190
  pOutFlags.setInt32(0, flags, true)
103
191
  return VFS.SQLITE_OK
104
- } catch (e: any) {
105
- console.error(e.message)
106
- return VFS.SQLITE_CANTOPEN
107
- }
192
+ }).pipe(
193
+ Effect.tapCauseLogPretty,
194
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_CANTOPEN)),
195
+ Runtime.runSync(this.#runtime),
196
+ )
108
197
  }
109
198
 
110
199
  jClose(fileId: number): number {
111
- const file = this.#mapIdToFile.get(fileId)
112
- if (file) {
113
- file.accessHandle.flush()
114
- this.#mapIdToFile.delete(fileId)
115
- if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
116
- this.#deletePath(file.path)
200
+ return Effect.gen(this, function* () {
201
+ const file = this.#mapIdToFile.get(fileId)
202
+ if (file) {
203
+ yield* Opfs.Opfs.syncFlush(file.accessHandle)
204
+ this.#mapIdToFile.delete(fileId)
205
+ if (file.flags & VFS.SQLITE_OPEN_DELETEONCLOSE) {
206
+ yield* this.#deletePath(file.path)
207
+ }
117
208
  }
118
- }
119
- return VFS.SQLITE_OK
209
+ return VFS.SQLITE_OK
210
+ }).pipe(
211
+ Effect.tapCauseLogPretty,
212
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_CLOSE)),
213
+ Runtime.runSync(this.#runtime),
214
+ )
120
215
  }
121
216
 
122
217
  jRead(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
123
- const file = this.#mapIdToFile.get(fileId)!
124
- const nBytes = file.accessHandle.read(pData.subarray(), {
125
- at: HEADER_OFFSET_DATA + iOffset,
126
- })
127
- if (nBytes < pData.byteLength) {
128
- pData.fill(0, nBytes, pData.byteLength)
129
- return VFS.SQLITE_IOERR_SHORT_READ
130
- }
131
- return VFS.SQLITE_OK
218
+ return Effect.gen(this, function* () {
219
+ const file = this.#mapIdToFile.get(fileId)!
220
+ const nBytes = yield* Opfs.Opfs.syncRead(file.accessHandle, pData.subarray(), {
221
+ at: HEADER_OFFSET_DATA + iOffset,
222
+ })
223
+ if (nBytes < pData.byteLength) {
224
+ pData.fill(0, nBytes, pData.byteLength)
225
+ return VFS.SQLITE_IOERR_SHORT_READ
226
+ }
227
+ return VFS.SQLITE_OK
228
+ }).pipe(
229
+ Effect.tapCauseLogPretty,
230
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_READ)),
231
+ Runtime.runSync(this.#runtime),
232
+ )
132
233
  }
133
234
 
134
235
  jWrite(fileId: number, pData: Uint8Array<ArrayBuffer>, iOffset: number): number {
135
- const file = this.#mapIdToFile.get(fileId)!
136
- const nBytes = file.accessHandle.write(pData.subarray(), {
137
- at: HEADER_OFFSET_DATA + iOffset,
138
- })
139
- return nBytes === pData.byteLength ? VFS.SQLITE_OK : VFS.SQLITE_IOERR
236
+ return Effect.gen(this, function* () {
237
+ const file = this.#mapIdToFile.get(fileId)!
238
+ const nBytes = yield* Opfs.Opfs.syncWrite(file.accessHandle, pData.subarray(), {
239
+ at: HEADER_OFFSET_DATA + iOffset,
240
+ })
241
+ if (nBytes !== pData.byteLength) {
242
+ return yield* Effect.dieMessage(`Wrote ${nBytes} bytes, expected ${pData.byteLength}`)
243
+ }
244
+ return VFS.SQLITE_OK
245
+ }).pipe(
246
+ Effect.tapCauseLogPretty,
247
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_WRITE)),
248
+ Runtime.runSync(this.#runtime),
249
+ )
140
250
  }
141
251
 
142
252
  jTruncate(fileId: number, iSize: number): number {
143
- const file = this.#mapIdToFile.get(fileId)!
144
- file.accessHandle.truncate(HEADER_OFFSET_DATA + iSize)
145
- return VFS.SQLITE_OK
253
+ return Effect.gen(this, function* () {
254
+ const file = this.#mapIdToFile.get(fileId)!
255
+ yield* Opfs.Opfs.syncTruncate(file.accessHandle, HEADER_OFFSET_DATA + iSize)
256
+ return VFS.SQLITE_OK
257
+ }).pipe(
258
+ Effect.tapCauseLogPretty,
259
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_TRUNCATE)),
260
+ Runtime.runSync(this.#runtime),
261
+ )
146
262
  }
147
263
 
148
264
  jSync(fileId: number, _flags: number): number {
149
- const file = this.#mapIdToFile.get(fileId)!
150
- file.accessHandle.flush()
151
- return VFS.SQLITE_OK
265
+ return Effect.gen(this, function* () {
266
+ const file = this.#mapIdToFile.get(fileId)!
267
+ yield* Opfs.Opfs.syncFlush(file.accessHandle)
268
+ return VFS.SQLITE_OK
269
+ }).pipe(
270
+ Effect.tapCauseLogPretty,
271
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSYNC)),
272
+ Runtime.runSync(this.#runtime),
273
+ )
152
274
  }
153
275
 
154
276
  jFileSize(fileId: number, pSize64: DataView): number {
155
- const file = this.#mapIdToFile.get(fileId)!
156
- const size = file.accessHandle.getSize() - HEADER_OFFSET_DATA
157
- pSize64.setBigInt64(0, BigInt(size), true)
158
- return VFS.SQLITE_OK
277
+ return Effect.gen(this, function* () {
278
+ const file = this.#mapIdToFile.get(fileId)!
279
+ const opfsFileSize = yield* Opfs.Opfs.syncGetSize(file.accessHandle)
280
+ const size = opfsFileSize - HEADER_OFFSET_DATA
281
+ pSize64.setBigInt64(0, BigInt(size), true)
282
+ return VFS.SQLITE_OK
283
+ }).pipe(
284
+ Effect.tapCauseLogPretty,
285
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_FSTAT)),
286
+ Runtime.runSync(this.#runtime),
287
+ )
159
288
  }
160
289
 
161
290
  jSectorSize(_fileId: number): number {
@@ -167,38 +296,46 @@ export class AccessHandlePoolVFS extends FacadeVFS {
167
296
  }
168
297
 
169
298
  jAccess(zName: string, _flags: number, pResOut: DataView): number {
170
- const path = this.#getPath(zName)
171
- pResOut.setInt32(0, this.#mapPathToAccessHandle.has(path) ? 1 : 0, true)
172
- return VFS.SQLITE_OK
299
+ return Effect.gen(this, function* () {
300
+ const path = this.#getPath(zName)
301
+ pResOut.setInt32(0, this.#mapPathToAccessHandle.has(path) ? 1 : 0, true)
302
+ return VFS.SQLITE_OK
303
+ }).pipe(
304
+ Effect.tapCauseLogPretty,
305
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_ACCESS)),
306
+ Runtime.runSync(this.#runtime),
307
+ )
173
308
  }
174
309
 
175
310
  jDelete(zName: string, _syncDir: number): number {
176
- const path = this.#getPath(zName)
177
- this.#deletePath(path)
178
- return VFS.SQLITE_OK
311
+ return Effect.gen(this, function* () {
312
+ const path = this.#getPath(zName)
313
+ yield* this.#deletePath(path)
314
+ return VFS.SQLITE_OK
315
+ }).pipe(
316
+ Effect.tapCauseLogPretty,
317
+ Effect.catchAllCause(() => Effect.succeed(VFS.SQLITE_IOERR_DELETE)),
318
+ Runtime.runSync(this.#runtime),
319
+ )
179
320
  }
180
321
 
181
- async close() {
182
- this.#releaseAccessHandles()
322
+ close() {
323
+ this.#releaseAccessHandles().pipe(Runtime.runPromise(this.#runtime))
183
324
  }
184
325
 
185
326
  async isReady() {
186
- if (!this.#directoryHandle) {
187
- // All files are stored in a single directory.
188
- let handle = await navigator.storage.getDirectory()
189
- for (const d of this.#directoryPath.split('/')) {
190
- if (d) {
191
- handle = await handle.getDirectoryHandle(d, { create: true })
327
+ return Effect.gen(this, function* () {
328
+ if (!this.#directoryHandle) {
329
+ // All files are stored in a single directory.
330
+ this.#directoryHandle = yield* Opfs.getDirectoryHandleByPath(this.#directoryPath, { create: true })
331
+
332
+ yield* this.#acquireAccessHandles()
333
+ if (this.getCapacity() === 0) {
334
+ yield* this.addCapacity(DEFAULT_CAPACITY)
192
335
  }
193
336
  }
194
- this.#directoryHandle = handle
195
-
196
- await this.#acquireAccessHandles()
197
- if (this.getCapacity() === 0) {
198
- await this.addCapacity(DEFAULT_CAPACITY)
199
- }
200
- }
201
- return true
337
+ return true
338
+ }).pipe(Runtime.runPromise(this.#runtime))
202
339
  }
203
340
 
204
341
  /**
@@ -216,156 +353,184 @@ export class AccessHandlePoolVFS extends FacadeVFS {
216
353
  }
217
354
 
218
355
  /**
219
- * Increase the capacity of the file system by n.
356
+ * Get all currently tracked SQLite file paths.
357
+ * This can be used by higher-level components for file management operations.
358
+ *
359
+ * @returns Array of currently active SQLite file paths
220
360
  */
221
- async addCapacity(n: number): Promise<number> {
222
- for (let i = 0; i < n; ++i) {
223
- const name = Math.random().toString(36).replace('0.', '')
224
- const handle = await this.#directoryHandle!.getFileHandle(name, {
225
- create: true,
226
- })
361
+ getTrackedFilePaths(): string[] {
362
+ return Array.from(this.#mapPathToAccessHandle.keys())
363
+ }
227
364
 
228
- const accessHandle = await Effect.tryPromise({
229
- try: () => handle.createSyncAccessHandle(),
230
- catch: (cause) => new OpfsError({ cause, path: name }),
231
- }).pipe(Effect.retry(Schedule.exponentialBackoff10Sec), Effect.runPromise)
232
- this.#mapAccessHandleToName.set(accessHandle, name)
365
+ /**
366
+ * Increase the capacity of the file system by n.
367
+ */
368
+ addCapacity: (n: number) => Effect.Effect<void, WebError.WebError, Opfs.Opfs | Scope.Scope> = Effect.fn((n: number) =>
369
+ Effect.repeatN(
370
+ Effect.gen(this, function* () {
371
+ const name = Math.random().toString(36).replace('0.', '')
372
+ const accessHandle = yield* Opfs.Opfs.getFileHandle(this.#directoryHandle!, name, { create: true }).pipe(
373
+ Effect.andThen((handle) => Opfs.Opfs.createSyncAccessHandle(handle)),
374
+ Effect.retry(Schedule.exponentialBackoff10Sec),
375
+ )
376
+ this.#mapAccessHandleToName.set(accessHandle, name)
233
377
 
234
- this.#setAssociatedPath(accessHandle, '', 0)
235
- }
236
- return n
237
- }
378
+ yield* this.#setAssociatedPath(accessHandle, '', 0)
379
+ }),
380
+ n,
381
+ ),
382
+ )
238
383
 
239
384
  /**
240
385
  * Decrease the capacity of the file system by n. The capacity cannot be
241
386
  * decreased to fewer than the current number of SQLite files in the
242
387
  * file system.
243
388
  */
244
- async removeCapacity(n: number): Promise<number> {
245
- let nRemoved = 0
246
- for (const accessHandle of Array.from(this.#availableAccessHandles)) {
247
- if (nRemoved === n || this.getSize() === this.getCapacity()) return nRemoved
248
-
249
- const name = this.#mapAccessHandleToName.get(accessHandle)!
250
- accessHandle.close()
251
- await this.#directoryHandle!.removeEntry(name)
252
- this.#mapAccessHandleToName.delete(accessHandle)
253
- this.#availableAccessHandles.delete(accessHandle)
254
- ++nRemoved
255
- }
256
- return nRemoved
257
- }
258
-
259
- async #acquireAccessHandles() {
260
- // Enumerate all the files in the directory.
261
- const files = [] as [string, FileSystemFileHandle][]
262
- for await (const [name, handle] of this.#directoryHandle!) {
263
- if (handle.kind === 'file') {
264
- files.push([name, handle as FileSystemFileHandle])
265
- }
266
- }
267
-
268
- // Open access handles in parallel, separating associated and unassociated.
269
- await Promise.all(
270
- files.map(async ([name, handle]) => {
271
- const accessHandle = await Effect.tryPromise({
272
- try: () => handle.createSyncAccessHandle(),
273
- catch: (cause) => new OpfsError({ cause, path: name }),
274
- }).pipe(Effect.retry(Schedule.exponentialBackoff10Sec), Effect.runPromise)
275
- this.#mapAccessHandleToName.set(accessHandle, name)
276
- const path = this.#getAssociatedPath(accessHandle)
277
- if (path) {
278
- this.#mapPathToAccessHandle.set(path, accessHandle)
279
- } else {
280
- this.#availableAccessHandles.add(accessHandle)
281
- }
282
- }),
283
- )
284
- }
285
-
286
- #releaseAccessHandles() {
287
- for (const accessHandle of this.#mapAccessHandleToName.keys()) {
288
- accessHandle.close()
289
- }
290
- this.#mapAccessHandleToName.clear()
291
- this.#mapPathToAccessHandle.clear()
292
- this.#availableAccessHandles.clear()
293
- }
389
+ removeCapacity = Effect.fn((n: number) =>
390
+ Effect.gen(this, function* () {
391
+ let nRemoved = 0
392
+ yield* Effect.forEach(
393
+ this.#availableAccessHandles,
394
+ (accessHandle) =>
395
+ Effect.gen(this, function* () {
396
+ if (nRemoved === n || this.getSize() === this.getCapacity()) return nRemoved
397
+
398
+ const name = this.#mapAccessHandleToName.get(accessHandle)!
399
+ accessHandle.close()
400
+ yield* Opfs.Opfs.removeEntry(this.#directoryHandle!, name)
401
+ this.#mapAccessHandleToName.delete(accessHandle)
402
+ this.#availableAccessHandles.delete(accessHandle)
403
+ ++nRemoved
404
+ }),
405
+ { discard: true },
406
+ )
407
+ return nRemoved
408
+ }),
409
+ )
410
+
411
+ #acquireAccessHandles = Effect.fn(() =>
412
+ Effect.gen(this, function* () {
413
+ const handlesStream = yield* Opfs.Opfs.values(this.#directoryHandle!)
414
+
415
+ yield* handlesStream.pipe(
416
+ Stream.filter((handle): handle is FileSystemFileHandle => handle.kind === 'file'),
417
+ Stream.mapEffect(
418
+ (fileHandle) =>
419
+ Effect.gen(this, function* () {
420
+ const accessHandle = yield* Opfs.Opfs.createSyncAccessHandle(fileHandle)
421
+ return {
422
+ accessHandle,
423
+ opfsFileName: fileHandle.name,
424
+ path: yield* this.#getAssociatedPath(accessHandle),
425
+ } as const
426
+ }),
427
+ { concurrency: 'unbounded' },
428
+ ),
429
+ Stream.runForEach(({ opfsFileName, accessHandle, path }) =>
430
+ Effect.gen(this, function* () {
431
+ this.#mapAccessHandleToName.set(accessHandle, opfsFileName)
432
+ if (path) {
433
+ this.#mapPathToAccessHandle.set(path, accessHandle)
434
+ } else {
435
+ this.#availableAccessHandles.add(accessHandle)
436
+ }
437
+ }),
438
+ ),
439
+ )
440
+ }),
441
+ )
442
+
443
+ #releaseAccessHandles = Effect.fn(() =>
444
+ Effect.gen(this, function* () {
445
+ yield* Effect.forEach(
446
+ this.#mapAccessHandleToName.keys(),
447
+ (accessHandle) => Effect.sync(() => accessHandle.close()),
448
+ { concurrency: 'unbounded', discard: true },
449
+ )
450
+ this.#mapAccessHandleToName.clear()
451
+ this.#mapPathToAccessHandle.clear()
452
+ this.#availableAccessHandles.clear()
453
+ }),
454
+ )
294
455
 
295
456
  /**
296
457
  * Read and return the associated path from an OPFS file header.
297
458
  * Empty string is returned for an unassociated OPFS file.
298
459
  * @returns {string} path or empty string
299
460
  */
300
- #getAssociatedPath(accessHandle: FileSystemSyncAccessHandle): string {
301
- // Read the path and digest of the path from the file.
302
- const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
303
- accessHandle.read(corpus, { at: 0 })
304
-
305
- // Delete files not expected to be present.
306
- const dataView = new DataView(corpus.buffer, corpus.byteOffset)
307
- const flags = dataView.getUint32(HEADER_OFFSET_FLAGS)
308
- if (corpus[0] && (flags & VFS.SQLITE_OPEN_DELETEONCLOSE || (flags & PERSISTENT_FILE_TYPES) === 0)) {
309
- console.warn(`Remove file with unexpected flags ${flags.toString(16)}`)
310
- this.#setAssociatedPath(accessHandle, '', 0)
311
- return ''
312
- }
461
+ #getAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle) =>
462
+ Effect.gen(this, function* () {
463
+ // Read the path and digest of the path from the file.
464
+ const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
465
+ yield* Opfs.Opfs.syncRead(accessHandle, corpus, { at: 0 })
466
+
467
+ // Delete files not expected to be present.
468
+ const dataView = new DataView(corpus.buffer, corpus.byteOffset)
469
+ const flags = dataView.getUint32(HEADER_OFFSET_FLAGS)
470
+ if (corpus[0] && (flags & VFS.SQLITE_OPEN_DELETEONCLOSE || (flags & PERSISTENT_FILE_TYPES) === 0)) {
471
+ yield* Effect.logWarning(`Remove file with unexpected flags ${flags.toString(16)}`)
472
+ yield* this.#setAssociatedPath(accessHandle, '', 0)
473
+ return ''
474
+ }
313
475
 
314
- const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4)
315
- accessHandle.read(fileDigest, { at: HEADER_OFFSET_DIGEST })
316
-
317
- // Verify the digest.
318
- const computedDigest = this.#computeDigest(corpus)
319
- if (fileDigest.every((value, i) => value === computedDigest[i])) {
320
- // Good digest. Decode the null-terminated path string.
321
- const pathBytes = corpus.indexOf(0)
322
- if (pathBytes === 0) {
323
- // Ensure that unassociated files are empty. Unassociated files are
324
- // truncated in #setAssociatedPath after the header is written. If
325
- // an interruption occurs right before the truncation then garbage
326
- // may remain in the file.
327
- accessHandle.truncate(HEADER_OFFSET_DATA)
476
+ const fileDigest = new Uint32Array(HEADER_DIGEST_SIZE / 4)
477
+ yield* Opfs.Opfs.syncRead(accessHandle, fileDigest, { at: HEADER_OFFSET_DIGEST })
478
+
479
+ // Verify the digest.
480
+ const computedDigest = this.#computeDigest(corpus)
481
+ if (fileDigest.every((value, i) => value === computedDigest[i])) {
482
+ // Good digest. Decode the null-terminated path string.
483
+ const pathBytes = corpus.indexOf(0)
484
+ if (pathBytes === 0) {
485
+ // Ensure that unassociated files are empty. Unassociated files are
486
+ // truncated in #setAssociatedPath after the header is written. If
487
+ // an interruption occurs right before the truncation then garbage
488
+ // may remain in the file.
489
+ yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
490
+ }
491
+ return new TextDecoder().decode(corpus.subarray(0, pathBytes))
492
+ } else {
493
+ // Bad digest. Repair this header.
494
+ yield* Effect.logWarning('Disassociating file with bad digest.')
495
+ yield* this.#setAssociatedPath(accessHandle, '', 0)
496
+ return ''
328
497
  }
329
- return new TextDecoder().decode(corpus.subarray(0, pathBytes))
330
- } else {
331
- // Bad digest. Repair this header.
332
- console.warn('Disassociating file with bad digest.')
333
- this.#setAssociatedPath(accessHandle, '', 0)
334
- return ''
335
- }
336
- }
498
+ }),
499
+ )
337
500
 
338
501
  /**
339
502
  * Set the path on an OPFS file header.
340
503
  */
341
- #setAssociatedPath(accessHandle: FileSystemSyncAccessHandle, path: string, flags: number) {
342
- // Convert the path string to UTF-8.
343
- const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
344
- const encodedResult = new TextEncoder().encodeInto(path, corpus)
345
- if (encodedResult.written >= HEADER_MAX_PATH_SIZE) {
346
- throw new Error('path too long')
347
- }
504
+ #setAssociatedPath = Effect.fn((accessHandle: FileSystemSyncAccessHandle, path: string, flags: number) =>
505
+ Effect.gen(this, function* () {
506
+ // Convert the path string to UTF-8.
507
+ const corpus = new Uint8Array(HEADER_CORPUS_SIZE)
508
+ const encodedResult = new TextEncoder().encodeInto(path, corpus)
509
+ if (encodedResult.written >= HEADER_MAX_PATH_SIZE) {
510
+ return yield* Effect.dieMessage('path too long')
511
+ }
348
512
 
349
- // Add the creation flags.
350
- const dataView = new DataView(corpus.buffer, corpus.byteOffset)
351
- dataView.setUint32(HEADER_OFFSET_FLAGS, flags)
352
-
353
- // Write the OPFS file header, including the digest.
354
- const digest = this.#computeDigest(corpus)
355
- accessHandle.write(corpus, { at: 0 })
356
- accessHandle.write(digest, { at: HEADER_OFFSET_DIGEST })
357
- accessHandle.flush()
358
-
359
- if (path) {
360
- this.#mapPathToAccessHandle.set(path, accessHandle)
361
- this.#availableAccessHandles.delete(accessHandle)
362
- } else {
363
- // This OPFS file doesn't represent any SQLite file so it doesn't
364
- // need to keep any data.
365
- accessHandle.truncate(HEADER_OFFSET_DATA)
366
- this.#availableAccessHandles.add(accessHandle)
367
- }
368
- }
513
+ // Add the creation flags.
514
+ const dataView = new DataView(corpus.buffer, corpus.byteOffset)
515
+ dataView.setUint32(HEADER_OFFSET_FLAGS, flags)
516
+
517
+ // Write the OPFS file header, including the digest.
518
+ const digest = this.#computeDigest(corpus)
519
+ yield* Opfs.Opfs.syncWrite(accessHandle, corpus, { at: 0 })
520
+ yield* Opfs.Opfs.syncWrite(accessHandle, digest, { at: HEADER_OFFSET_DIGEST })
521
+ yield* Opfs.Opfs.syncFlush(accessHandle)
522
+
523
+ if (path) {
524
+ this.#mapPathToAccessHandle.set(path, accessHandle)
525
+ this.#availableAccessHandles.delete(accessHandle)
526
+ } else {
527
+ // This OPFS file doesn't represent any SQLite file so it doesn't
528
+ // need to keep any data.
529
+ yield* Opfs.Opfs.syncTruncate(accessHandle, HEADER_OFFSET_DATA)
530
+ this.#availableAccessHandles.add(accessHandle)
531
+ }
532
+ }),
533
+ )
369
534
 
370
535
  /**
371
536
  * We need a synchronous digest function so can't use WebCrypto.
@@ -404,17 +569,14 @@ export class AccessHandlePoolVFS extends FacadeVFS {
404
569
  * Remove the association between a path and an OPFS file.
405
570
  * @param {string} path
406
571
  */
407
- #deletePath(path: string) {
408
- const accessHandle = this.#mapPathToAccessHandle.get(path)
409
- if (accessHandle) {
410
- // Un-associate the SQLite path from the OPFS file.
411
- this.#mapPathToAccessHandle.delete(path)
412
- this.#setAssociatedPath(accessHandle, '', 0)
413
- }
414
- }
572
+ #deletePath = Effect.fn((path: string) =>
573
+ Effect.gen(this, function* () {
574
+ const accessHandle = this.#mapPathToAccessHandle.get(path)
575
+ if (accessHandle) {
576
+ // Un-associate the SQLite path from the OPFS file.
577
+ this.#mapPathToAccessHandle.delete(path)
578
+ yield* this.#setAssociatedPath(accessHandle, '', 0)
579
+ }
580
+ }),
581
+ )
415
582
  }
416
-
417
- export class OpfsError extends Schema.TaggedError<OpfsError>()('OpfsError', {
418
- cause: Schema.Defect,
419
- path: Schema.String,
420
- }) {}